Keep the search index current without re-walking the volume
A search everywhere rebuilt the whole index once it was older than fifteen minutes, so every session paid for a walk of the entire volume - tens of seconds of reading before a single result - even though almost nothing had changed. StorageSource gains names(), which lists a folder's names without reading a size and a date for every child. That is the difference between a walk and a catch-up: list() costs three stat round trips per entry on a phone's mounted storage, and a name index does not need any of them for entries it already knows. SearchIndexer.refresh() compares the names a folder holds now against the names already indexed and reads metadata only for names it has never seen, writing nothing at all for a folder that has not changed. The search itself now answers first and catches up behind the answer: whatever the index knows is shown immediately, and the pass that makes it current runs afterwards and answers again, which is the whole of "I just downloaded it, so why is it not here?". A full walk is now the exception - a volume never indexed, one whose rows were built under the other hidden-files preference, one not walked in full for a week, or the explicit Rebuild - because a catch-up compares names and so cannot see a file swapped for a different file of the same name, nor what a preference change means for rows that were never counted. Two timestamps per volume keep that distinction, which is a schema change: v2 adds full_at and hidden_included with a real migration that keeps the rows, since throwing them away would cost the next search the walk it just avoided. A folder that cannot be read now answers null rather than empty, and a catch-up leaves what it knows about it alone: concluding "everything inside has gone" from a transient failure is how an index loses things that are still there. The tests assert what a pass *reads*, which is the point of the change: a current index makes no call to the storage at all, a stale one reads only the names it has never seen, and a walk followed by a catch-up over a real temporary tree keeps the index level with files and folders added and deleted between them. AllFilesSourceTest covers the new names() against the real filesystem, including the null for an unreadable folder. 288 JVM tests, lint clean. No device pass this time: /dev/kvm disappeared from this environment partway through, so the emulator could not be started again. The device checks for the index itself were made in the previous round.
This commit is contained in:
@@ -37,6 +37,15 @@ class AllFilesSource(
|
||||
if (children == null) emptyList() else children.map { it.toEntry() }
|
||||
}
|
||||
|
||||
/**
|
||||
* One `readdir` and no `stat` per child, which is the whole reason a catch-up pass is
|
||||
* cheap. `File.list()` returns `null` for a folder that cannot be read — exactly the
|
||||
* "leave it alone" answer the caller wants.
|
||||
*/
|
||||
override suspend fun names(directory: StoragePath): List<String>? = withContext(Dispatchers.IO) {
|
||||
File(directory.value).list()?.toList()
|
||||
}
|
||||
|
||||
override suspend fun stat(path: StoragePath): FileEntry? = withContext(Dispatchers.IO) {
|
||||
File(path.value).takeIf { it.exists() }?.toEntry()
|
||||
}
|
||||
|
||||
@@ -62,6 +62,29 @@ class SafSource(
|
||||
entries
|
||||
}
|
||||
|
||||
/**
|
||||
* The same children query as [list], asking the provider for names only.
|
||||
*
|
||||
* A document provider answers a whole query in one round trip, so the saving here is
|
||||
* smaller than on a filesystem — but it is still one cursor instead of one cursor plus a
|
||||
* `FileEntry` per child, and the query is the expensive part either way.
|
||||
*/
|
||||
override suspend fun names(directory: StoragePath): List<String>? = withContext(Dispatchers.IO) {
|
||||
val parentId = SafDocumentIds.documentId(treeDocumentId, directory)
|
||||
val childrenUri = DocumentsContract.buildChildDocumentsUriUsingTree(treeUri, parentId)
|
||||
|
||||
val cursor = resolver.query(childrenUri, NAME_PROJECTION, null, null, null)
|
||||
?: return@withContext null
|
||||
|
||||
val names = ArrayList<String>()
|
||||
cursor.use {
|
||||
while (it.moveToNext()) {
|
||||
it.getString(0)?.let(names::add)
|
||||
}
|
||||
}
|
||||
names
|
||||
}
|
||||
|
||||
override suspend fun stat(path: StoragePath): FileEntry? = withContext(Dispatchers.IO) {
|
||||
resolver.query(documentUri(path), PROJECTION, null, null, null)?.use { cursor ->
|
||||
if (cursor.moveToFirst()) cursor.toEntry(path) else null
|
||||
@@ -157,5 +180,8 @@ class SafSource(
|
||||
DocumentsContract.Document.COLUMN_SIZE,
|
||||
DocumentsContract.Document.COLUMN_LAST_MODIFIED,
|
||||
)
|
||||
|
||||
/** The one column a catch-up needs: what is in the folder, by name. */
|
||||
val NAME_PROJECTION = arrayOf(DocumentsContract.Document.COLUMN_DISPLAY_NAME)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,6 +19,20 @@ data class IndexedFile(
|
||||
val modifiedMillis: Long,
|
||||
)
|
||||
|
||||
/**
|
||||
* What a volume's index is: how current it is, and how it was built.
|
||||
*
|
||||
* The two timestamps are separate on purpose. [indexedAtMillis] answers "may I trust this?";
|
||||
* [fullAtMillis] answers "when was every single entry last looked at?", which is the only
|
||||
* thing that can catch what a catch-up structurally cannot — a file swapped for another of
|
||||
* the same name, or a folder whose timestamps have been restored.
|
||||
*/
|
||||
data class VolumeStamp(
|
||||
val indexedAtMillis: Long,
|
||||
val fullAtMillis: Long,
|
||||
val hiddenIncluded: Boolean,
|
||||
)
|
||||
|
||||
/**
|
||||
* A by-name index of what is on the volumes, so a search does not have to walk them.
|
||||
*
|
||||
@@ -45,8 +59,15 @@ interface SearchIndex {
|
||||
/** Add a batch of rows for the volume being indexed. */
|
||||
suspend fun add(rows: List<IndexedFile>)
|
||||
|
||||
/** Finish the pass, recording when it happened. */
|
||||
suspend fun endVolume(volume: String, indexedAtMillis: Long)
|
||||
/** Finish a pass for [volume]. */
|
||||
suspend fun endVolume(
|
||||
volume: String,
|
||||
indexedAtMillis: Long,
|
||||
/** `true` for a walk that replaced everything, `false` for a catch-up. */
|
||||
full: Boolean = false,
|
||||
/** Whether this pass counted hidden entries, so a preference change can be noticed. */
|
||||
hiddenIncluded: Boolean = false,
|
||||
)
|
||||
|
||||
/**
|
||||
* Rows whose name matches [query], at most [limit].
|
||||
@@ -60,9 +81,24 @@ interface SearchIndex {
|
||||
/** Drop one row, because the file it pointed at has gone. */
|
||||
suspend fun forget(path: StoragePath)
|
||||
|
||||
/** Drop [path] and everything the index knows beneath it. */
|
||||
suspend fun forgetTree(path: StoragePath)
|
||||
|
||||
/**
|
||||
* The rows for the *direct* children of [directory], and no deeper.
|
||||
*
|
||||
* A catch-up compares these against what the folder holds now, so what it needs is exactly
|
||||
* one folder's worth — which is also why this is a query per visited folder rather than
|
||||
* the whole volume held in memory.
|
||||
*/
|
||||
suspend fun children(directory: StoragePath): List<IndexedFile>
|
||||
|
||||
/** When the stalest volume was walked, or `null` when nothing has been indexed. */
|
||||
suspend fun indexedAt(): Long?
|
||||
|
||||
/** What each volume's index is: how current, and how it was built. */
|
||||
suspend fun stamps(): Map<String, VolumeStamp>
|
||||
|
||||
/** Forget everything, for a rebuild from nothing. */
|
||||
suspend fun clear()
|
||||
|
||||
@@ -75,14 +111,25 @@ interface SearchIndex {
|
||||
|
||||
override suspend fun add(rows: List<IndexedFile>) = Unit
|
||||
|
||||
override suspend fun endVolume(volume: String, indexedAtMillis: Long) = Unit
|
||||
override suspend fun endVolume(
|
||||
volume: String,
|
||||
indexedAtMillis: Long,
|
||||
full: Boolean,
|
||||
hiddenIncluded: Boolean,
|
||||
) = Unit
|
||||
|
||||
override suspend fun find(query: String, limit: Int): List<IndexedFile> = emptyList()
|
||||
|
||||
override suspend fun forget(path: StoragePath) = Unit
|
||||
|
||||
override suspend fun forgetTree(path: StoragePath) = Unit
|
||||
|
||||
override suspend fun children(directory: StoragePath): List<IndexedFile> = emptyList()
|
||||
|
||||
override suspend fun indexedAt(): Long? = null
|
||||
|
||||
override suspend fun stamps(): Map<String, VolumeStamp> = emptyMap()
|
||||
|
||||
override suspend fun clear() = Unit
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,19 +46,89 @@ class SqliteSearchIndex(context: Context) : SearchIndex {
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun endVolume(volume: String, indexedAtMillis: Long) = withContext(Dispatchers.IO) {
|
||||
override suspend fun endVolume(
|
||||
volume: String,
|
||||
indexedAtMillis: Long,
|
||||
full: Boolean,
|
||||
hiddenIncluded: Boolean,
|
||||
) = withContext(Dispatchers.IO) {
|
||||
helper.writableDatabase.insertWithOnConflict(
|
||||
TABLE_VOLUMES,
|
||||
null,
|
||||
ContentValues().apply {
|
||||
put(COLUMN_VOLUME, volume)
|
||||
put(COLUMN_INDEXED_AT, indexedAtMillis)
|
||||
// A catch-up keeps the previous full-pass time: no walk has happened, and
|
||||
// pretending otherwise would mean never noticing what only a walk can find.
|
||||
put(COLUMN_FULL_AT, if (full) indexedAtMillis else fullAt(volume))
|
||||
put(COLUMN_HIDDEN_INCLUDED, if (hiddenIncluded) 1 else 0)
|
||||
},
|
||||
SQLiteDatabase.CONFLICT_REPLACE,
|
||||
)
|
||||
Unit
|
||||
}
|
||||
|
||||
override suspend fun stamps(): Map<String, VolumeStamp> = withContext(Dispatchers.IO) {
|
||||
helper.readableDatabase
|
||||
.rawQuery("SELECT $COLUMN_VOLUME, $COLUMN_INDEXED_AT, $COLUMN_FULL_AT, $COLUMN_HIDDEN_INCLUDED FROM $TABLE_VOLUMES", null)
|
||||
.use { cursor ->
|
||||
buildMap {
|
||||
while (cursor.moveToNext()) {
|
||||
put(
|
||||
cursor.getString(0),
|
||||
VolumeStamp(
|
||||
indexedAtMillis = cursor.getLong(1),
|
||||
fullAtMillis = cursor.getLong(2),
|
||||
hiddenIncluded = cursor.getInt(3) != 0,
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun fullAt(volume: String): Long =
|
||||
helper.readableDatabase
|
||||
.rawQuery("SELECT $COLUMN_FULL_AT FROM $TABLE_VOLUMES WHERE $COLUMN_VOLUME = ?", arrayOf(volume))
|
||||
.use { cursor -> if (cursor.moveToFirst()) cursor.getLong(0) else 0L }
|
||||
|
||||
override suspend fun forgetTree(path: StoragePath) = withContext(Dispatchers.IO) {
|
||||
val (lower, upper) = subtreeRange(path)
|
||||
helper.writableDatabase.delete(
|
||||
TABLE_FILES,
|
||||
"$COLUMN_PATH = ? OR ($COLUMN_PATH >= ? AND $COLUMN_PATH < ?)",
|
||||
arrayOf(path.value, lower, upper),
|
||||
)
|
||||
Unit
|
||||
}
|
||||
|
||||
override suspend fun children(directory: StoragePath): List<IndexedFile> = withContext(Dispatchers.IO) {
|
||||
val (lower, upper) = subtreeRange(directory)
|
||||
helper.readableDatabase.query(
|
||||
TABLE_FILES,
|
||||
null,
|
||||
// The range narrows to the subtree through the primary-key index; the `instr` then
|
||||
// keeps only the direct children, which are the ones with no separator left in
|
||||
// what follows the folder's own prefix.
|
||||
"$COLUMN_PATH >= ? AND $COLUMN_PATH < ? AND instr(substr($COLUMN_PATH, ?), '/') = 0",
|
||||
arrayOf(lower, upper, (directory.value.length + 2).toString()),
|
||||
null,
|
||||
null,
|
||||
"$COLUMN_NAME ASC",
|
||||
).use { cursor -> buildList { while (cursor.moveToNext()) add(cursor.toRow()) } }
|
||||
}
|
||||
|
||||
/**
|
||||
* The half-open range that contains everything inside [path] and nothing else.
|
||||
*
|
||||
* Every string beginning with `path/` sorts between `path/` and `path0`, because `0` is the
|
||||
* character immediately after `/`. A range like this uses the primary-key index, where a
|
||||
* `LIKE` would either scan the table or need its wildcards escaped — and a path is allowed
|
||||
* to contain `%` or `_`, which is exactly the sort of thing an unescaped `LIKE` gets wrong.
|
||||
*/
|
||||
private fun subtreeRange(path: StoragePath): Pair<String, String> =
|
||||
"${path.value}/" to "${path.value}0"
|
||||
|
||||
override suspend fun find(query: String, limit: Int): List<IndexedFile> = withContext(Dispatchers.IO) {
|
||||
val (clause, args) = likeClause(query)
|
||||
helper.readableDatabase.query(
|
||||
@@ -173,7 +243,7 @@ class SqliteSearchIndex(context: Context) : SearchIndex {
|
||||
*/
|
||||
internal companion object {
|
||||
const val DATABASE_NAME = "search-index.db"
|
||||
const val DATABASE_VERSION = 1
|
||||
const val DATABASE_VERSION = 2
|
||||
const val TABLE_FILES = "indexed_files"
|
||||
const val TABLE_VOLUMES = "indexed_volumes"
|
||||
const val COLUMN_PATH = "path"
|
||||
@@ -183,6 +253,8 @@ class SqliteSearchIndex(context: Context) : SearchIndex {
|
||||
const val COLUMN_MODIFIED = "modified"
|
||||
const val COLUMN_VOLUME = "volume"
|
||||
const val COLUMN_INDEXED_AT = "indexed_at"
|
||||
const val COLUMN_FULL_AT = "full_at"
|
||||
const val COLUMN_HIDDEN_INCLUDED = "hidden_included"
|
||||
|
||||
private const val LIKE_ESCAPE = '\\'
|
||||
|
||||
@@ -206,8 +278,10 @@ class SqliteSearchIndex(context: Context) : SearchIndex {
|
||||
db.execSQL(
|
||||
"""
|
||||
CREATE TABLE $TABLE_VOLUMES (
|
||||
$COLUMN_VOLUME TEXT NOT NULL PRIMARY KEY,
|
||||
$COLUMN_INDEXED_AT INTEGER NOT NULL
|
||||
$COLUMN_VOLUME TEXT NOT NULL PRIMARY KEY,
|
||||
$COLUMN_INDEXED_AT INTEGER NOT NULL,
|
||||
$COLUMN_FULL_AT INTEGER NOT NULL DEFAULT 0,
|
||||
$COLUMN_HIDDEN_INCLUDED INTEGER NOT NULL DEFAULT 0
|
||||
)
|
||||
""".trimIndent(),
|
||||
)
|
||||
@@ -223,11 +297,28 @@ class SqliteSearchIndex(context: Context) : SearchIndex {
|
||||
*/
|
||||
fun upgrade(db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {
|
||||
for (version in oldVersion until newVersion) {
|
||||
error(
|
||||
"No migration from search-index schema version $version to ${version + 1}. " +
|
||||
"The index can be rebuilt from the device, so dropping and recreating " +
|
||||
"is a valid step here — write it out deliberately and bump DATABASE_VERSION.",
|
||||
)
|
||||
when (version) {
|
||||
// v1 recorded only when a volume was last touched. v2 has to tell a full
|
||||
// walk from a catch-up, and remember which way the hidden-files preference
|
||||
// was set when it ran — both of which decide what the index may be trusted
|
||||
// for. Added rather than recreated: the rows are still good, and a migration
|
||||
// that threw them away would make the next search pay for a walk it need not.
|
||||
1 -> {
|
||||
db.execSQL(
|
||||
"ALTER TABLE $TABLE_VOLUMES ADD COLUMN $COLUMN_FULL_AT INTEGER NOT NULL DEFAULT 0",
|
||||
)
|
||||
db.execSQL(
|
||||
"ALTER TABLE $TABLE_VOLUMES ADD COLUMN $COLUMN_HIDDEN_INCLUDED " +
|
||||
"INTEGER NOT NULL DEFAULT 0",
|
||||
)
|
||||
}
|
||||
|
||||
else -> error(
|
||||
"No migration from search-index schema version $version to ${version + 1}. " +
|
||||
"The index can be rebuilt from the device, so dropping and recreating " +
|
||||
"is a valid step here — write it out deliberately and bump DATABASE_VERSION.",
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,6 +38,24 @@ interface StorageSource {
|
||||
/** Children of [directory], in whatever order the backend returns them (unsorted). */
|
||||
suspend fun list(directory: StoragePath): List<FileEntry>
|
||||
|
||||
/**
|
||||
* The names in [directory], **without reading metadata for each one**.
|
||||
*
|
||||
* The difference is not cosmetic. [list] `stat`s every child for its size and date, and on
|
||||
* a phone's mounted storage a `stat` is a round trip to another process — three of them per
|
||||
* entry, which is what makes a full walk of a large volume take tens of seconds. A catch-up
|
||||
* pass, whose only question is "what is here that I do not already know about?", needs
|
||||
* nothing but the names.
|
||||
*
|
||||
* `null` when the folder could not be read. That is deliberately *not* the same as an empty
|
||||
* list: a caller catching up must leave what it knows about an unreadable folder alone
|
||||
* rather than conclude that everything inside it has gone.
|
||||
*
|
||||
* The default falls back to [list], so a backend that cannot do better still works — it
|
||||
* simply pays the full price.
|
||||
*/
|
||||
suspend fun names(directory: StoragePath): List<String>? = list(directory).map { it.name }
|
||||
|
||||
/** Metadata for [path], or `null` when it does not exist. */
|
||||
suspend fun stat(path: StoragePath): FileEntry?
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package de.softwareapp_hb.filemanager.domain
|
||||
|
||||
import de.softwareapp_hb.filemanager.core.FileEntry
|
||||
import de.softwareapp_hb.filemanager.core.StoragePath
|
||||
import de.softwareapp_hb.filemanager.core.isHiddenName
|
||||
import de.softwareapp_hb.filemanager.data.IndexedFile
|
||||
import de.softwareapp_hb.filemanager.data.SearchIndex
|
||||
import de.softwareapp_hb.filemanager.data.StorageSource
|
||||
@@ -11,15 +12,17 @@ import kotlinx.coroutines.isActive
|
||||
/**
|
||||
* Fills a [SearchIndex] by walking a volume.
|
||||
*
|
||||
* The walk is the same one the live search does — depth-first, hidden entries skipped unless
|
||||
* the preference says otherwise — with two differences that only make sense in the
|
||||
* background: it is not capped at a couple of hundred results, and it hands its rows over in
|
||||
* batches rather than collecting a whole volume's listing in memory.
|
||||
* Two passes, and the difference between them is the whole reason an index can be both instant
|
||||
* and current:
|
||||
*
|
||||
* Cancelling it is safe and leaves the index consistent: rows may have been added, but a
|
||||
* volume's timestamp is only written once the entire pass has finished, so an interrupted
|
||||
* walk leaves the old timestamp in place and reads as *older* than it is. That is the safe
|
||||
* direction to be wrong in.
|
||||
* - [index] walks everything and replaces the volume's rows. It reads the size and date of
|
||||
* every entry, which is what makes it the expensive one.
|
||||
* - [refresh] catches up: it compares the names each folder holds now against the names the
|
||||
* index already has for it, and reads metadata only for what is new.
|
||||
*
|
||||
* Both are safe to cancel: rows may have been written, but a volume's stamp is only written
|
||||
* once a pass has finished, so an interrupted pass leaves the old stamp in place and reads as
|
||||
* *older* than it is. That is the safe direction to be wrong in.
|
||||
*/
|
||||
class SearchIndexer(
|
||||
private val source: StorageSource,
|
||||
@@ -27,7 +30,7 @@ class SearchIndexer(
|
||||
private val clock: () -> Long = System::currentTimeMillis,
|
||||
) {
|
||||
|
||||
/** Walk everything under [volume], replacing what the index knows about it. Returns rows. */
|
||||
/** Walk everything under [volume], replacing what the index knows about it. Returns entries. */
|
||||
suspend fun index(
|
||||
volume: StoragePath,
|
||||
showHidden: Boolean,
|
||||
@@ -72,10 +75,85 @@ class SearchIndexer(
|
||||
|
||||
walk(volume)
|
||||
flush()
|
||||
index.endVolume(key, clock())
|
||||
index.endVolume(key, clock(), full = true, hiddenIncluded = showHidden)
|
||||
return total
|
||||
}
|
||||
|
||||
/**
|
||||
* Bring [volume] up to date without re-reading anything that has not changed. Returns how
|
||||
* many names were looked at — the work done, which for a catch-up is most of the tree.
|
||||
*
|
||||
* A folder's own listing is all a *name* index needs: comparing the names it holds now
|
||||
* against the names already indexed answers "what is new here?" without reading a single
|
||||
* entry the index already knows. That is the difference between a walk and a catch-up —
|
||||
* [StorageSource.list] reads a size and a date for every child, [StorageSource.names] reads
|
||||
* only names, and on a phone's mounted storage each of those is a round trip to another
|
||||
* process.
|
||||
*
|
||||
* Every folder still has to be *visited*: a change deep inside a tree leaves the folders
|
||||
* above it untouched, so no timestamp can say "nothing below here moved", and a pass that
|
||||
* trusted one would miss new files. What this cannot see, it does not invent — a file whose
|
||||
* contents changed is still found by its name, and a periodic full [index] bounds the rest.
|
||||
*/
|
||||
suspend fun refresh(
|
||||
volume: StoragePath,
|
||||
showHidden: Boolean,
|
||||
onProgress: (Int) -> Unit = {},
|
||||
): Int {
|
||||
val key = volume.value
|
||||
var seen = 0
|
||||
var reported = 0
|
||||
|
||||
suspend fun visit(directory: StoragePath) {
|
||||
if (!currentCoroutineContext().isActive) return
|
||||
|
||||
// A folder that cannot be read is left exactly as it was: an empty answer here
|
||||
// would look like "everything inside is gone", and deleting rows on a transient
|
||||
// failure is how an index loses things that are still there.
|
||||
val names = source.names(directory) ?: return
|
||||
seen += names.size
|
||||
if (seen - reported >= REPORT_EVERY) {
|
||||
reported = seen
|
||||
onProgress(seen)
|
||||
}
|
||||
|
||||
val known = index.children(directory).associateBy { it.name }
|
||||
|
||||
val rows = ArrayList<IndexedFile>()
|
||||
val folders = ArrayList<StoragePath>()
|
||||
for (name in names) {
|
||||
if (!showHidden && isHiddenName(name)) continue
|
||||
|
||||
val knownRow = known[name]
|
||||
if (knownRow != null) {
|
||||
// Known by name: nothing to read, and a folder stays a folder.
|
||||
if (knownRow.isDirectory) folders += knownRow.path
|
||||
continue
|
||||
}
|
||||
|
||||
// Only a name the index has never seen needs looking at.
|
||||
val entry = source.stat(directory.child(name)) ?: continue
|
||||
rows += entry.toRow(key)
|
||||
if (entry.isDirectory) folders += entry.path
|
||||
}
|
||||
if (rows.isNotEmpty()) index.add(rows)
|
||||
|
||||
// Whatever the index knew that the folder no longer holds. A folder that has gone
|
||||
// takes its whole subtree with it.
|
||||
for (row in known.values) {
|
||||
if (names.contains(row.name) && (showHidden || !isHiddenName(row.name))) continue
|
||||
if (row.isDirectory) index.forgetTree(row.path) else index.forget(row.path)
|
||||
}
|
||||
|
||||
for (folder in folders) visit(folder)
|
||||
}
|
||||
|
||||
visit(volume)
|
||||
onProgress(seen)
|
||||
index.endVolume(key, clock(), full = false, hiddenIncluded = showHidden)
|
||||
return seen
|
||||
}
|
||||
|
||||
private fun FileEntry.toRow(volume: String) = IndexedFile(
|
||||
volume = volume,
|
||||
path = path,
|
||||
|
||||
@@ -349,11 +349,14 @@ class BrowserViewModel(
|
||||
/**
|
||||
* Search what the index knows, on every volume at once.
|
||||
*
|
||||
* Two things make this honest rather than merely fast. Indexed rows are *candidates*: the
|
||||
* filesystem is the truth, so every hit is checked and the ones whose file has gone are
|
||||
* dropped — and forgotten, so the next search does not pay for them again. And the index
|
||||
* is rebuilt when it is missing or old, because a search that silently omits what arrived
|
||||
* since the last walk is worse than one that spends a minute being sure.
|
||||
* Three rules make this honest rather than merely fast. **Indexed rows are candidates**, not
|
||||
* answers: the filesystem is the truth, so every hit is checked and the ones whose file has
|
||||
* gone are dropped — and forgotten, so the next search does not pay for them again. **The
|
||||
* answer comes first**: whatever the index knows is shown before anything is done about
|
||||
* whether it is current, because a search that waits for a walk before showing a single
|
||||
* result is the thing an index exists to avoid. And **the index is brought up to date behind
|
||||
* that answer**, because a search that silently omits what arrived since the last pass is
|
||||
* worse than one that spends a second being sure.
|
||||
*/
|
||||
private fun searchEverywhere(query: String, source: StorageSource) {
|
||||
// Results can come from anywhere, so the row subtitle shows the whole parent path
|
||||
@@ -363,66 +366,112 @@ class BrowserViewModel(
|
||||
}
|
||||
|
||||
searchJob = viewModelScope.launch {
|
||||
val age = searchIndex.indexedAt()
|
||||
if (age == null || clock() - age > INDEX_FRESHNESS_MILLIS) rebuild(source)
|
||||
val volumes = volumes()
|
||||
val stampAt = clock()
|
||||
val stamps = searchIndex.stamps()
|
||||
val hidden = _state.value.showHidden
|
||||
|
||||
val hits = runCatching { searchIndex.find(query, FileSearch.DEFAULT_LIMIT) }
|
||||
.getOrDefault(emptyList())
|
||||
val confirmed = ArrayList<FileEntry>(hits.size)
|
||||
for (hit in hits) {
|
||||
if (!currentCoroutineContext().isActive) return@launch
|
||||
if (source.exists(hit.path)) {
|
||||
confirmed += hit.toEntry()
|
||||
} else {
|
||||
// An index row whose file has gone is a row that has stopped being useful.
|
||||
searchIndex.forget(hit.path)
|
||||
}
|
||||
if (stamps.isNotEmpty()) showIndexedResults(query, source)
|
||||
|
||||
// A volume whose index was built with the other hidden-files preference, or which
|
||||
// has not been walked end to end for a long time, needs a walk rather than a
|
||||
// catch-up: a catch-up compares names, so it cannot see a name that is there both
|
||||
// times but is not the file it was, and the preference is part of what rows mean.
|
||||
val toWalk = volumes.filter { volume ->
|
||||
val stamp = stamps[volume.value]
|
||||
stamp == null ||
|
||||
stamp.hiddenIncluded != hidden ||
|
||||
stampAt - stamp.fullAtMillis > INDEX_FULL_PASS_AFTER_MILLIS
|
||||
}
|
||||
if (toWalk.isNotEmpty()) pass(source, toWalk, full = true)
|
||||
|
||||
val toCatchUp = volumes.filterNot(toWalk::contains).filter { volume ->
|
||||
val stamp = stamps[volume.value]
|
||||
stamp == null || stampAt - stamp.indexedAtMillis > INDEX_CATCH_UP_AFTER_MILLIS
|
||||
}
|
||||
if (toCatchUp.isNotEmpty()) pass(source, toCatchUp, full = false)
|
||||
|
||||
// Answered again, because the pass may have found the file this search is for —
|
||||
// which is the whole of "I just downloaded it, so why is it not here?".
|
||||
showIndexedResults(query, source)
|
||||
|
||||
_state.update { current ->
|
||||
if (current.query != query) {
|
||||
current
|
||||
} else {
|
||||
current.copy(
|
||||
entries = current.sort.sort(confirmed),
|
||||
searching = false,
|
||||
indexedAt = current.indexedAt ?: age,
|
||||
)
|
||||
current.copy(searching = false, indexedAt = searchIndex.indexedAt())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk every volume and replace what the index knows.
|
||||
*
|
||||
* Deliberately not a foreground service: a scan runs while this screen is up, and stops
|
||||
* when it is not. Nothing is left half-written — an interrupted volume keeps its old
|
||||
* timestamp — so stopping early costs freshness, never correctness.
|
||||
*/
|
||||
fun rebuildSearchIndex() {
|
||||
val source = storage.activeSource() ?: return
|
||||
viewModelScope.launch {
|
||||
rebuild(source)
|
||||
setQuery(_state.value.query)
|
||||
/** Answer [query] from the index, confirming every candidate against the filesystem. */
|
||||
private suspend fun showIndexedResults(query: String, source: StorageSource) {
|
||||
val hits = runCatching { searchIndex.find(query, FileSearch.DEFAULT_LIMIT) }
|
||||
.getOrDefault(emptyList())
|
||||
|
||||
val confirmed = ArrayList<FileEntry>(hits.size)
|
||||
for (hit in hits) {
|
||||
if (!currentCoroutineContext().isActive) return
|
||||
if (source.exists(hit.path)) {
|
||||
confirmed += hit.toEntry()
|
||||
} else {
|
||||
// An index row whose file has gone is a row that has stopped being useful.
|
||||
searchIndex.forget(hit.path)
|
||||
}
|
||||
}
|
||||
|
||||
_state.update { current ->
|
||||
if (current.query != query) current else current.copy(entries = current.sort.sort(confirmed))
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun rebuild(source: StorageSource) {
|
||||
val volumes = storage.volumes().map { it.root }.ifEmpty { listOf(StoragePath.Root) }
|
||||
/**
|
||||
* Bring [volumes] up to date, and say so while it happens.
|
||||
*
|
||||
* A *walk* is for a volume the index has nothing good about — never built, built with a
|
||||
* different hidden-files preference, or not walked in full for a week. Everything else is a
|
||||
* *catch-up*, which reads names and nothing else. Neither is a foreground service: both run
|
||||
* while this screen is up and stop when it is not, and stopping early costs freshness,
|
||||
* never correctness.
|
||||
*/
|
||||
private suspend fun pass(source: StorageSource, volumes: List<StoragePath>, full: Boolean) {
|
||||
val indexer = SearchIndexer(source, searchIndex, clock)
|
||||
|
||||
_state.update { it.copy(indexing = true, indexedCount = 0) }
|
||||
var seen = 0
|
||||
for (volume in volumes) {
|
||||
if (!currentCoroutineContext().isActive) break
|
||||
seen += indexer.index(volume, _state.value.showHidden) { inVolume ->
|
||||
val onProgress: (Int) -> Unit = { inVolume ->
|
||||
_state.update { it.copy(indexedCount = seen + inVolume) }
|
||||
}
|
||||
seen += if (full) {
|
||||
indexer.index(volume, _state.value.showHidden, onProgress)
|
||||
} else {
|
||||
indexer.refresh(volume, _state.value.showHidden, onProgress)
|
||||
}
|
||||
}
|
||||
_state.update { it.copy(indexing = false, indexedAt = searchIndex.indexedAt()) }
|
||||
}
|
||||
|
||||
/** The volumes a pass should cover: every one this app can see. */
|
||||
private suspend fun volumes(): List<StoragePath> =
|
||||
storage.volumes().map { it.root }.ifEmpty { listOf(StoragePath.Root) }
|
||||
|
||||
/**
|
||||
* Walk every volume end to end and replace what the index knows.
|
||||
*
|
||||
* The explicit action, and the only thing that recovers from the index being *wrong* rather
|
||||
* than merely old, which is why it stays one tap away on the search row.
|
||||
*/
|
||||
fun rebuildSearchIndex() {
|
||||
val source = storage.activeSource() ?: return
|
||||
viewModelScope.launch {
|
||||
pass(source, volumes(), full = true)
|
||||
setQuery(_state.value.query)
|
||||
}
|
||||
}
|
||||
|
||||
/** An indexed row as a listing row: the same shape the walk would have produced. */
|
||||
private fun IndexedFile.toEntry() = FileEntry(
|
||||
path = path,
|
||||
@@ -1564,13 +1613,24 @@ class BrowserViewModel(
|
||||
private val SCRATCH_UNSAFE_CHARS = Regex("[^A-Za-z0-9._-]")
|
||||
|
||||
/**
|
||||
* How old the index may be before a search rebuilds it rather than trusting it.
|
||||
* How old the index may be before a search catches it up.
|
||||
*
|
||||
* Fifteen minutes is a judgement, not a measurement: long enough that a hunt through
|
||||
* one folder does not re-walk a volume, short enough that "I just downloaded it"
|
||||
* still finds it. What makes the number safe is that the age is on screen either way.
|
||||
* A catch-up reads names and nothing else, so it is cheap enough to do whenever the
|
||||
* user searches; the window is here only so that a burst of keystrokes does not repeat
|
||||
* it per character. Two minutes is short enough that "I just downloaded it" works, and
|
||||
* long enough that a hunt through one folder costs one pass rather than ten.
|
||||
*/
|
||||
const val INDEX_FRESHNESS_MILLIS = 15L * 60L * 1000L
|
||||
const val INDEX_CATCH_UP_AFTER_MILLIS = 2L * 60L * 1000L
|
||||
|
||||
/**
|
||||
* How long an index may go without a full walk.
|
||||
*
|
||||
* A catch-up compares names, so it cannot notice a file replaced by a different file of
|
||||
* the same name, or a folder whose timestamps were restored — the two things only a walk
|
||||
* sees. A week bounds how long that can go unnoticed, and it is exactly why a volume
|
||||
* carries two timestamps instead of one.
|
||||
*/
|
||||
const val INDEX_FULL_PASS_AFTER_MILLIS = 7L * 24L * 60L * 60L * 1000L
|
||||
|
||||
val Factory: ViewModelProvider.Factory = viewModelFactory {
|
||||
initializer {
|
||||
|
||||
Reference in New Issue
Block a user