diff options
| author | Méven Car <[email protected]> | 2023-05-26 15:10:09 +0000 |
|---|---|---|
| committer | Méven Car <[email protected]> | 2023-05-26 15:10:09 +0000 |
| commit | a1c5c5cf81b5d1f6b7a0aa10b8a981cb70c5b26c (patch) | |
| tree | 0fcfce01dcef32b1b70b084e9719cf7d9c26418d /src/kitemviews/private | |
| parent | ab13c8881dafbe5beefef0db5891ce9f731ef489 (diff) | |
KDirectoryContentsCounter: show intermediate dir size counting results, improve stopping, improve data caching
Two user visible changes:
* we can see the dir size changing as it is updated.
* This makes the file counting a lot more reactive, when changing directories for instance.
`KDirectoryContentsCounterWorker::walkDir` is not recursive anymore.
The cache is now shared and only a single thread is used to count size recursively.
Diffstat (limited to 'src/kitemviews/private')
4 files changed, 312 insertions, 199 deletions
diff --git a/src/kitemviews/private/kdirectorycontentscounter.cpp b/src/kitemviews/private/kdirectorycontentscounter.cpp index 37e852ab9..e1a47419d 100644 --- a/src/kitemviews/private/kdirectorycontentscounter.cpp +++ b/src/kitemviews/private/kdirectorycontentscounter.cpp @@ -6,6 +6,7 @@ */ #include "kdirectorycontentscounter.h" +#include "dolphin_detailsmodesettings.h" #include "kitemviews/kfileitemmodel.h" #include <KDirWatch> @@ -16,36 +17,102 @@ namespace { + +class LocalCache +{ +public: + struct cacheData { + int count = 0; + long long size = 0; + ushort refCount = 0; + qint64 timestamp = 0; + + inline operator bool() const + { + return timestamp != 0 && count != -1; + } + }; + + LocalCache() + : m_cache() + { + } + + cacheData insert(const QString &key, cacheData data, bool inserted) + { + data.timestamp = QDateTime::currentMSecsSinceEpoch(); + if (inserted) { + data.refCount += 1; + } + m_cache.insert(key, data); + return data; + } + + cacheData value(const QString &key) const + { + return m_cache.value(key); + } + + void unRefAll(const QSet<QString> &keys) + { + for (const auto &key : keys) { + auto entry = m_cache[key]; + entry.refCount -= 1; + if (entry.refCount == 0) { + m_cache.remove(key); + } + } + } + + void removeAll(const QSet<QString> &keys) + { + for (const auto &key : keys) { + m_cache.remove(key); + } + } + +private: + QHash<QString, cacheData> m_cache; +}; + /// cache of directory counting result -static QHash<QString, QPair<int, long>> *s_cache; +static LocalCache *s_cache; +static QThread *s_workerThread; +static KDirectoryContentsCounterWorker *s_worker; } KDirectoryContentsCounter::KDirectoryContentsCounter(KFileItemModel *model, QObject *parent) : QObject(parent) , m_model(model) + , m_priorityQueue() , m_queue() - , m_worker(nullptr) , m_workerIsBusy(false) , m_dirWatcher(nullptr) , m_watchedDirs() { - connect(m_model, &KFileItemModel::itemsRemoved, this, &KDirectoryContentsCounter::slotItemsRemoved); + if (s_cache == nullptr) { + s_cache = new LocalCache(); + } - if (!m_workerThread) { - m_workerThread = new QThread(); - m_workerThread->start(); + if (!s_workerThread) { + s_workerThread = new QThread(); + s_workerThread->setObjectName(QStringLiteral("KDirectoryContentsCounterThread")); + s_workerThread->start(); } - if (s_cache == nullptr) { - s_cache = new QHash<QString, QPair<int, long>>(); + if (!s_worker) { + s_worker = new KDirectoryContentsCounterWorker(); + s_worker->moveToThread(s_workerThread); } - m_worker = new KDirectoryContentsCounterWorker(); - m_worker->moveToThread(m_workerThread); + connect(m_model, &KFileItemModel::itemsRemoved, this, &KDirectoryContentsCounter::slotItemsRemoved); + connect(m_model, &KFileItemModel::directoryRefreshing, this, &KDirectoryContentsCounter::slotDirectoryRefreshing); + + connect(this, &KDirectoryContentsCounter::requestDirectoryContentsCount, s_worker, &KDirectoryContentsCounterWorker::countDirectoryContents); - connect(this, &KDirectoryContentsCounter::requestDirectoryContentsCount, m_worker, &KDirectoryContentsCounterWorker::countDirectoryContents); - connect(this, &KDirectoryContentsCounter::stop, m_worker, &KDirectoryContentsCounterWorker::stop); - connect(m_worker, &KDirectoryContentsCounterWorker::result, this, &KDirectoryContentsCounter::slotResult); + connect(s_worker, &KDirectoryContentsCounterWorker::result, this, &KDirectoryContentsCounter::slotResult); + connect(s_worker, &KDirectoryContentsCounterWorker::intermediateResult, this, &KDirectoryContentsCounter::result); + connect(s_worker, &KDirectoryContentsCounterWorker::finished, this, &KDirectoryContentsCounter::scheduleNext); m_dirWatcher = new KDirWatch(this); connect(m_dirWatcher, &KDirWatch::dirty, this, &KDirectoryContentsCounter::slotDirWatchDirty); @@ -53,60 +120,20 @@ KDirectoryContentsCounter::KDirectoryContentsCounter(KFileItemModel *model, QObj KDirectoryContentsCounter::~KDirectoryContentsCounter() { - if (m_workerThread->isRunning()) { - // The worker thread will continue running. It could even be running - // a method of m_worker at the moment, so we delete it using - // deleteLater() to prevent a crash. - m_worker->deleteLater(); - } else { - // There are no remaining workers -> stop the worker thread. - m_workerThread->quit(); - m_workerThread->wait(); - delete m_workerThread; - m_workerThread = nullptr; - - // The worker thread has finished running now, so it's safe to delete - // m_worker. deleteLater() would not work at all because the event loop - // which would deliver the event to m_worker is not running any more. - delete m_worker; - } + s_cache->unRefAll(m_watchedDirs); } -void KDirectoryContentsCounter::slotResult(const QString &path, int count, long size) +void KDirectoryContentsCounter::slotResult(const QString &path, int count, long long size) { - m_workerIsBusy = false; - - const QFileInfo info = QFileInfo(path); - const QString resolvedPath = info.canonicalFilePath(); - - if (!m_dirWatcher->contains(resolvedPath)) { + const auto fileInfo = QFileInfo(path); + const QString resolvedPath = fileInfo.canonicalFilePath(); + if (fileInfo.isReadable() && !m_watchedDirs.contains(resolvedPath)) { m_dirWatcher->addDir(resolvedPath); - m_watchedDirs.insert(resolvedPath); } + bool inserted = m_watchedDirs.insert(resolvedPath) == m_watchedDirs.end(); - if (!m_priorityQueue.empty()) { - const QString firstPath = m_priorityQueue.front(); - m_priorityQueue.pop_front(); - scanDirectory(firstPath, PathCountPriority::High); - } else if (!m_queue.empty()) { - const QString firstPath = m_queue.front(); - m_queue.pop_front(); - scanDirectory(firstPath, PathCountPriority::Normal); - } - - if (s_cache->contains(resolvedPath)) { - const auto pair = s_cache->value(resolvedPath); - if (pair.first == count && pair.second == size) { - // no change no need to send another result event - return; - } - } - - if (info.dir().path() == m_model->rootItem().url().path()) { - // update cache or overwrite value - // when path is a direct children of the current model root - s_cache->insert(resolvedPath, QPair<int, long>(count, size)); - } + // update cache or overwrite value + s_cache->insert(resolvedPath, {count, size, true}, inserted); // sends the results Q_EMIT result(path, count, size); @@ -131,6 +158,11 @@ void KDirectoryContentsCounter::slotItemsRemoved() { const bool allItemsRemoved = (m_model->count() == 0); + if (allItemsRemoved) { + s_cache->removeAll(m_watchedDirs); + stopWorker(); + } + if (!m_watchedDirs.isEmpty()) { // Don't let KDirWatch watch for removed items if (allItemsRemoved) { @@ -138,7 +170,6 @@ void KDirectoryContentsCounter::slotItemsRemoved() m_dirWatcher->removeDir(path); } m_watchedDirs.clear(); - m_queue.clear(); } else { QMutableSetIterator<QString> it(m_watchedDirs); while (it.hasNext()) { @@ -152,46 +183,102 @@ void KDirectoryContentsCounter::slotItemsRemoved() } } -void KDirectoryContentsCounter::scanDirectory(const QString &path, PathCountPriority priority) +void KDirectoryContentsCounter::slotDirectoryRefreshing() +{ + s_cache->removeAll(m_watchedDirs); +} + +void KDirectoryContentsCounter::scheduleNext() { - const QString resolvedPath = QFileInfo(path).canonicalFilePath(); - const bool alreadyInCache = s_cache->contains(resolvedPath); - if (alreadyInCache) { + if (!m_priorityQueue.empty()) { + m_currentPath = m_priorityQueue.front(); + m_priorityQueue.pop_front(); + } else if (!m_queue.empty()) { + m_currentPath = m_queue.front(); + m_queue.pop_front(); + } else { + m_currentPath.clear(); + m_workerIsBusy = false; + return; + } + + const auto fileInfo = QFileInfo(m_currentPath); + const QString resolvedPath = fileInfo.canonicalFilePath(); + const auto pair = s_cache->value(resolvedPath); + if (pair) { // fast path when in cache // will be updated later if result has changed - const auto pair = s_cache->value(resolvedPath); - Q_EMIT result(path, pair.first, pair.second); + Q_EMIT result(m_currentPath, pair.count, pair.size); } - if (m_workerIsBusy) { - // only enqueue path not yet in queue - if (std::find(m_queue.begin(), m_queue.end(), path) == m_queue.end() - && std::find(m_priorityQueue.begin(), m_priorityQueue.end(), path) == m_priorityQueue.end()) { - if (priority == PathCountPriority::Normal) { - if (alreadyInCache) { - // if we already knew the dir size, it gets lower priority - m_queue.push_back(path); - } else { - m_queue.push_front(path); - } - } else { - // append to priority queue - m_priorityQueue.push_back(path); - } - } - } else { - KDirectoryContentsCounterWorker::Options options; + // if scanned fully recently, skip rescan + if (pair && pair.timestamp >= fileInfo.fileTime(QFile::FileModificationTime).toMSecsSinceEpoch()) { + scheduleNext(); + return; + } + + KDirectoryContentsCounterWorker::Options options; + + if (m_model->showHiddenFiles()) { + options |= KDirectoryContentsCounterWorker::CountHiddenFiles; + } + + m_workerIsBusy = true; + Q_EMIT requestDirectoryContentsCount(m_currentPath, options, DetailsModeSettings::recursiveDirectorySizeLimit()); +} - if (m_model->showHiddenFiles()) { - options |= KDirectoryContentsCounterWorker::CountHiddenFiles; +void KDirectoryContentsCounter::enqueuePathScanning(const QString &path, bool alreadyInCache, PathCountPriority priority) +{ + // ensure to update the entry in the queue + auto it = std::find(m_queue.begin(), m_queue.end(), path); + if (it != m_queue.end()) { + m_queue.erase(it); + } else { + it = std::find(m_priorityQueue.begin(), m_priorityQueue.end(), path); + if (it != m_priorityQueue.end()) { + m_priorityQueue.erase(it); } + } - if (m_model->showDirectoriesOnly()) { - options |= KDirectoryContentsCounterWorker::CountDirectoriesOnly; + if (priority == PathCountPriority::Normal) { + if (alreadyInCache) { + // we already knew the dir size + // otherwise it gets lower priority + m_queue.push_back(path); + } else { + m_queue.push_front(path); } + } else { + // append to priority queue + m_priorityQueue.push_front(path); + } +} - Q_EMIT requestDirectoryContentsCount(path, options); - m_workerIsBusy = true; +void KDirectoryContentsCounter::scanDirectory(const QString &path, PathCountPriority priority) +{ + if (m_workerIsBusy && m_currentPath == path) { + // already listing + return; + } + + const auto fileInfo = QFileInfo(path); + const QString resolvedPath = fileInfo.canonicalFilePath(); + const auto pair = s_cache->value(resolvedPath); + if (pair) { + // fast path when in cache + // will be updated later if result has changed + Q_EMIT result(path, pair.count, pair.size); + } + + // if scanned fully recently, skip rescan + if (pair && pair.timestamp >= fileInfo.fileTime(QFile::FileModificationTime).toMSecsSinceEpoch()) { + return; + } + + enqueuePathScanning(path, pair, priority); + + if (!m_workerIsBusy && !s_worker->stopping()) { + scheduleNext(); } } @@ -199,7 +286,9 @@ void KDirectoryContentsCounter::stopWorker() { m_queue.clear(); m_priorityQueue.clear(); - Q_EMIT stop(); -} -QThread *KDirectoryContentsCounter::m_workerThread = nullptr; + if (m_workerIsBusy && m_currentPath == s_worker->scannedPath()) { + s_worker->stop(); + } + m_currentPath.clear(); +} diff --git a/src/kitemviews/private/kdirectorycontentscounter.h b/src/kitemviews/private/kdirectorycontentscounter.h index 9a5e4a86b..552b5560e 100644 --- a/src/kitemviews/private/kdirectorycontentscounter.h +++ b/src/kitemviews/private/kdirectorycontentscounter.h @@ -49,32 +49,32 @@ Q_SIGNALS: * Signals that the directory \a path contains \a count items of size \a * Size calculation depends on parameter DetailsModeSettings::recursiveDirectorySizeLimit */ - void result(const QString &path, int count, long size); + void result(const QString &path, int count, long long size); - void requestDirectoryContentsCount(const QString &path, KDirectoryContentsCounterWorker::Options options); - - void stop(); + void requestDirectoryContentsCount(const QString &path, KDirectoryContentsCounterWorker::Options options, int maxRecursiveLevel); private Q_SLOTS: - void slotResult(const QString &path, int count, long size); + void slotResult(const QString &path, int count, long long size); void slotDirWatchDirty(const QString &path); void slotItemsRemoved(); + void slotDirectoryRefreshing(); + void scheduleNext(); private: + void enqueuePathScanning(const QString &path, bool alreadyInCache, PathCountPriority priority); + KFileItemModel *m_model; // Used as FIFO queues. std::list<QString> m_priorityQueue; std::list<QString> m_queue; - static QThread *m_workerThread; - - KDirectoryContentsCounterWorker *m_worker; bool m_workerIsBusy; KDirWatch *m_dirWatcher; QSet<QString> m_watchedDirs; // Required as sadly KDirWatch does not offer a getter method // to get all watched directories. + QString m_currentPath; }; #endif diff --git a/src/kitemviews/private/kdirectorycontentscounterworker.cpp b/src/kitemviews/private/kdirectorycontentscounterworker.cpp index 2227ebbc2..eb456da25 100644 --- a/src/kitemviews/private/kdirectorycontentscounterworker.cpp +++ b/src/kitemviews/private/kdirectorycontentscounterworker.cpp @@ -7,16 +7,16 @@ #include "kdirectorycontentscounterworker.h" -// Required includes for subItemsCount(): +// Required includes for countDirectoryContents(): #ifdef Q_OS_WIN #include <QDir> #else -#include <QFile> -#include <qplatformdefs.h> +#include <QElapsedTimer> +#include <fts.h> +#include <sys/stat.h> +#include <sys/types.h> #endif -#include "dolphin_detailsmodesettings.h" - KDirectoryContentsCounterWorker::KDirectoryContentsCounterWorker(QObject *parent) : QObject(parent) { @@ -24,100 +24,135 @@ KDirectoryContentsCounterWorker::KDirectoryContentsCounterWorker(QObject *parent } #ifndef Q_OS_WIN -KDirectoryContentsCounterWorker::CountResult -KDirectoryContentsCounterWorker::walkDir(const QString &dirPath, const bool countHiddenFiles, const bool countDirectoriesOnly, const uint allowedRecursiveLevel) +void KDirectoryContentsCounterWorker::walkDir(const QString &dirPath, bool countHiddenFiles, uint allowedRecursiveLevel) { - int count = -1; - long size = -1; - auto dir = QT_OPENDIR(QFile::encodeName(dirPath)); - if (dir) { - count = 0; - size = 0; - QT_DIRENT *dirEntry; - QT_STATBUF buf; + QByteArray text = dirPath.toLocal8Bit(); + char *rootPath = new char[text.size() + 1]; + ::strncpy(rootPath, text.constData(), text.size() + 1); + char *path[2]{rootPath, nullptr}; + + // follow symlink only for root dir + auto tree = ::fts_open(path, FTS_COMFOLLOW | FTS_PHYSICAL | FTS_XDEV, nullptr); + if (!tree) { + delete[] rootPath; + return; + } + + FTSENT *node; + long long totalSize = -1; + int totalCount = -1; + QElapsedTimer timer; + timer.start(); + + while ((node = fts_read(tree)) && !m_stopping) { + auto info = node->fts_info; + + if (info == FTS_DC) { + // ignore directories clausing cycles + continue; + } + if (info == FTS_DNR) { + // ignore directories that can’t be read + continue; + } + if (info == FTS_ERR) { + // ignore directories causing errors + fts_set(tree, node, FTS_SKIP); + continue; + } + if (info == FTS_DP) { + // ignore end traversal of dir + continue; + } - while (!m_stopping && (dirEntry = QT_READDIR(dir))) { - if (dirEntry->d_name[0] == '.') { - if (dirEntry->d_name[1] == '\0' || !countHiddenFiles) { - // Skip "." or hidden files - continue; - } - if (dirEntry->d_name[1] == '.' && dirEntry->d_name[2] == '\0') { - // Skip ".." - continue; - } + if (!countHiddenFiles && node->fts_name[0] == '.' && strncmp(".git", node->fts_name, 4) != 0) { + // skip hidden files, except .git dirs + if (info == FTS_D) { + fts_set(tree, node, FTS_SKIP); } + continue; + } - // If only directories are counted, consider an unknown file type and links also - // as directory instead of trying to do an expensive stat() - // (see bugs 292642 and 299997). - const bool countEntry = !countDirectoriesOnly || dirEntry->d_type == DT_DIR || dirEntry->d_type == DT_LNK || dirEntry->d_type == DT_UNKNOWN; - if (countEntry) { - ++count; + if (info == FTS_F) { + // only count files that are physical (aka skip /proc/kcore...) + // naive size counting not taking into account effective disk space used (aka size/block_size * block_size) + // skip directory size (usually a 4KB block) + if (node->fts_statp->st_blocks > 0) { + totalSize += node->fts_statp->st_size; } + } - if (allowedRecursiveLevel > 0) { - QString nameBuf = QStringLiteral("%1/%2").arg(dirPath, dirEntry->d_name); + if (info == FTS_D) { + if (node->fts_level == 0) { + // first read was sucessful, we can init counters + totalSize = 0; + totalCount = 0; + } - if (dirEntry->d_type == DT_REG) { - if (QT_STAT(nameBuf.toLocal8Bit(), &buf) == 0) { - size += buf.st_size; - } - } - if (!m_stopping && dirEntry->d_type == DT_DIR) { - // recursion for dirs - auto subdirResult = walkDir(nameBuf, countHiddenFiles, countDirectoriesOnly, allowedRecursiveLevel - 1); - if (subdirResult.size > 0) { - size += subdirResult.size; - } - } + if (node->fts_level > (int)allowedRecursiveLevel) { + // skip too deep nodes + fts_set(tree, node, FTS_SKIP); + continue; } } - QT_CLOSEDIR(dir); + // count first level elements + if (node->fts_level == 1) { + ++totalCount; + } + + // delay intermediate results + if (timer.hasExpired(200) || node->fts_level == 0) { + Q_EMIT intermediateResult(dirPath, totalCount, totalSize); + timer.restart(); + } + } + + delete[] rootPath; + fts_close(tree); + if (errno != 0) { + return; + } + + if (!m_stopping) { + Q_EMIT result(dirPath, totalCount, totalSize); } - return KDirectoryContentsCounterWorker::CountResult{count, size}; } #endif -KDirectoryContentsCounterWorker::CountResult KDirectoryContentsCounterWorker::subItemsCount(const QString &path, Options options) +void KDirectoryContentsCounterWorker::stop() +{ + m_stopping = true; +} + +bool KDirectoryContentsCounterWorker::stopping() const +{ + return m_stopping; +} + +QString KDirectoryContentsCounterWorker::scannedPath() const +{ + return m_scannedPath; +} + +void KDirectoryContentsCounterWorker::countDirectoryContents(const QString &path, Options options, int maxRecursiveLevel) { const bool countHiddenFiles = options & CountHiddenFiles; - const bool countDirectoriesOnly = options & CountDirectoriesOnly; #ifdef Q_OS_WIN QDir dir(path); - QDir::Filters filters = QDir::NoDotAndDotDot | QDir::System; + QDir::Filters filters = QDir::NoDotAndDotDot | QDir::System | QDir::AllEntries; if (countHiddenFiles) { filters |= QDir::Hidden; } - if (countDirectoriesOnly) { - filters |= QDir::Dirs; - } else { - filters |= QDir::AllEntries; - } - return {static_cast<int>(dir.entryList(filters).count()), 0}; -#else - const uint maxRecursiveLevel = DetailsModeSettings::directorySizeCount() ? 1 : DetailsModeSettings::recursiveDirectorySizeLimit(); + Q_EMIT result(path, static_cast<int>(dir.entryList(filters).count()), 0); +#else - auto res = walkDir(path, countHiddenFiles, countDirectoriesOnly, maxRecursiveLevel); + m_scannedPath = path; + walkDir(path, countHiddenFiles, maxRecursiveLevel); - return res; #endif -} - -void KDirectoryContentsCounterWorker::stop() -{ - m_stopping = true; -} -void KDirectoryContentsCounterWorker::countDirectoryContents(const QString &path, Options options) -{ m_stopping = false; - - auto res = subItemsCount(path, options); - - if (!m_stopping) { - Q_EMIT result(path, res.count, res.size); - } + Q_EMIT finished(); } diff --git a/src/kitemviews/private/kdirectorycontentscounterworker.h b/src/kitemviews/private/kdirectorycontentscounterworker.h index 5266960cd..077df9f69 100644 --- a/src/kitemviews/private/kdirectorycontentscounterworker.h +++ b/src/kitemviews/private/kdirectorycontentscounterworker.h @@ -17,32 +17,20 @@ class KDirectoryContentsCounterWorker : public QObject Q_OBJECT public: - enum Option { NoOptions = 0x0, CountHiddenFiles = 0x1, CountDirectoriesOnly = 0x2 }; + enum Option { NoOptions = 0x0, CountHiddenFiles = 0x1 }; Q_DECLARE_FLAGS(Options, Option) - struct CountResult { - /// number of elements in the directory - int count; - /// Recursive sum of the size of the directory content files and folders - /// Calculation depends on DetailsModeSettings::recursiveDirectorySizeLimit - long size; - }; - explicit KDirectoryContentsCounterWorker(QObject *parent = nullptr); - /** - * Counts the items inside the directory \a path using the options - * \a options. - * - * @return The number of items. - */ - CountResult subItemsCount(const QString &path, Options options); - + bool stopping() const; + QString scannedPath() const; Q_SIGNALS: /** * Signals that the directory \a path contains \a count items and optionally the size of its content. */ - void result(const QString &path, int count, long size); + void result(const QString &path, int count, long long size); + void intermediateResult(const QString &path, int count, long long size); + void finished(); public Q_SLOTS: /** @@ -52,16 +40,17 @@ public Q_SLOTS: // Note that the full type name KDirectoryContentsCounterWorker::Options // is needed here. Just using 'Options' is OK for the compiler, but // confuses moc. - void countDirectoryContents(const QString &path, KDirectoryContentsCounterWorker::Options options); + void countDirectoryContents(const QString &path, KDirectoryContentsCounterWorker::Options options, int maxRecursiveLevel); void stop(); private: #ifndef Q_OS_WIN - KDirectoryContentsCounterWorker::CountResult - walkDir(const QString &dirPath, const bool countHiddenFiles, const bool countDirectoriesOnly, const uint allowedRecursiveLevel); + void walkDir(const QString &dirPath, bool countHiddenFiles, uint allowedRecursiveLevel); #endif - bool m_stopping = false; + QString m_scannedPath; + + std::atomic<bool> m_stopping = false; }; Q_DECLARE_METATYPE(KDirectoryContentsCounterWorker::Options) |
