diff --git a/AGENTS.md b/AGENTS.md index ca02edd0..59ce405c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,7 +17,7 @@ open the listed source, and run the listed check. Do not read unrelated document | If the task is about | Read | Code | Narrowest check | |---|---|---|---| | Query parsing, results, address box, autocomplete | [design](docs/design.md#navigation-and-search) | [model_search.cpp](src/model_search.cpp), [model_tokenizer.h](src/model_tokenizer.h), [app_search.h](src/app_search.h), [model_index_summary.cpp](src/model_index_summary.cpp) | `/test:*search*` | -| Indexing, the database, thumbnails, hydration | [implementation](docs/implementation.md#index-search-and-database) | [model_index.cpp](src/model_index.cpp), [model_db.cpp](src/model_db.cpp), [model_postings.h](src/model_postings.h) | `/test:*index*`, `/test:*thumbnail*` | +| Indexing, the database, thumbnails, hydration | [implementation](docs/implementation.md#index-search-and-database) | [model_index.cpp](src/model_index.cpp), [model_db.cpp](src/model_db.cpp), [model_postings.h](src/model_postings.h), [model_search_index.cpp](src/model_search_index.cpp) | `/test:*index*`, `/test:*thumbnail*` | | Collection membership, duplicates, presence | [collections](docs/collections.md) | [model_index.cpp](src/model_index.cpp), [model_index_duplicates.cpp](src/model_index_duplicates.cpp), [app_settings.cpp](src/app_settings.cpp), [model_related.h](src/model_related.h) | `/test:*duplicate*`, `/test:*presence*` | | Zoom, pan, the navigator, panorama projection | [zoom](docs/zoom.md) | [model_zoom.h](src/model_zoom.h), [view_media.h](src/view_media.h), [ui_panorama.h](src/ui_panorama.h) | `/test:*zoom*`, `/test:*panorama*` | | Places, distance, map, visits, tiles | [locations](docs/locations.md) | [model_locations.cpp](src/model_locations.cpp), [model_visits.cpp](src/model_visits.cpp), [ui_map_common.h](src/ui_map_common.h), [ui_globe.h](src/ui_globe.h), [model_tile_cache.cpp](src/model_tile_cache.cpp) | `/test:*location*`, `/test:*visit*`, `/test:*globe*` | diff --git a/CMakeLists.txt b/CMakeLists.txt index 3f9555f9..dae3d14d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -228,6 +228,7 @@ set(DIFFRACTOR_PORTABLE_SOURCES src/model_items.cpp src/model_movie.cpp src/model_search.cpp + src/model_search_index.cpp src/model_tile_cache.cpp src/render_charts.cpp src/render_globe.cpp diff --git a/cmake/DiffractorLib8bit.cmake b/cmake/DiffractorLib8bit.cmake index 6be20d3a..4eec189d 100644 --- a/cmake/DiffractorLib8bit.cmake +++ b/cmake/DiffractorLib8bit.cmake @@ -9,7 +9,7 @@ include_guard(GLOBAL) include(FetchContent) -set(DIFFRACTOR_LIB8BIT_REVISION 745ac7111a8e7cc7701ff9f5e77b2e75faaf5085) +set(DIFFRACTOR_LIB8BIT_REVISION db38a1b5c79e64e9f64ad0026c6dddac5ff31fc2) if (NOT DEFINED FETCHCONTENT_SOURCE_DIR_LIB8BIT AND EXISTS "${CMAKE_SOURCE_DIR}/../lib8bit/CMakeLists.txt") set(FETCHCONTENT_SOURCE_DIR_LIB8BIT "${CMAKE_SOURCE_DIR}/../lib8bit" CACHE PATH diff --git a/cmake/DiffractorLibdesktop.cmake b/cmake/DiffractorLibdesktop.cmake index dbffbd5f..864f8d78 100644 --- a/cmake/DiffractorLibdesktop.cmake +++ b/cmake/DiffractorLibdesktop.cmake @@ -11,7 +11,7 @@ include_guard(GLOBAL) include(FetchContent) -set(DIFFRACTOR_LIBDESKTOP_REVISION dff36833d1c7b9207e3bacf8d084933fc44945b3) +set(DIFFRACTOR_LIBDESKTOP_REVISION 38a23099e761082346b3a39978bd3613b2c2aca2) if (NOT DEFINED FETCHCONTENT_SOURCE_DIR_LIBDESKTOP AND EXISTS "${CMAKE_SOURCE_DIR}/../libdesktop/CMakeLists.txt") set(FETCHCONTENT_SOURCE_DIR_LIBDESKTOP "${CMAKE_SOURCE_DIR}/../libdesktop" CACHE PATH diff --git a/cmake/DiffractorLibfiles.cmake b/cmake/DiffractorLibfiles.cmake index a4328d69..b8ef248e 100644 --- a/cmake/DiffractorLibfiles.cmake +++ b/cmake/DiffractorLibfiles.cmake @@ -9,7 +9,7 @@ include_guard(GLOBAL) include(FetchContent) -set(DIFFRACTOR_LIBFILES_REVISION 4481ded34629ec203400fcf1b0ecbdf8510b471c) +set(DIFFRACTOR_LIBFILES_REVISION 27de9d9fb67d7be7fcf9ff8a9f87dad5a91ae3eb) if (NOT DEFINED FETCHCONTENT_SOURCE_DIR_LIBFILES AND EXISTS "${CMAKE_SOURCE_DIR}/../libfiles/CMakeLists.txt") set(FETCHCONTENT_SOURCE_DIR_LIBFILES "${CMAKE_SOURCE_DIR}/../libfiles" CACHE PATH diff --git a/docs/design.md b/docs/design.md index 72c2de07..192e6a7f 100644 --- a/docs/design.md +++ b/docs/design.md @@ -164,6 +164,12 @@ Address editing keeps a committed value, a typed draft, and an optional completi preview completions without replacing the draft. Tab accepts into the draft; Enter commits; Escape first restores the draft from a preview, then restores the address captured when editing began. +Each prediction shows how many items choosing it lists, and of two suggestions of one kind, the one +that finds more ranks higher. A suggestion drawn from the collection's own words, tags, places, or +properties is not offered when it would find nothing. What was typed is always offered first, and +recent searches keep their order. A count of none, or one not ready in time, shows nothing; the list +never waits for a count. + History entries retain query and selection. Back and Forward restore rather than rewrite them, and move the history position only after the target navigation is accepted; a refused or unavailable target leaves the current entry and the next Back/Forward target unchanged. Navigating after Back @@ -174,6 +180,12 @@ Navigation from a map, chart, breakdown, summary, or group header changes the qu sorting, and filters remain user-owned. Filtering removes hidden items from selection; removing the filter does not reselect them. +A sidebar row's total is the number of items clicking it lists. Hovering a row also says how many of +the items shown it holds, when the index can say. A tag or label row that would list nothing is not +shown; rating rows always are, because together they are one scale. Predictions and sidebar rows +have no item target, as the owner decided: they navigate, and their counts describe what navigating +lists. + ## Selection And Viewing Plain click selects one item; Ctrl toggles; Shift extends from the focus anchor; Ctrl+Shift combines diff --git a/docs/file-io.md b/docs/file-io.md index 230c2504..9f0d9f75 100644 --- a/docs/file-io.md +++ b/docs/file-io.md @@ -80,7 +80,7 @@ rather than copied and then rewritten. Other formats are copied, then updated. I replacements. Temporary paths are cleaned on every bounded failure path. A JPEG XL's metadata is updated by libfiles like any other format's, its compressed boxes read with -Diffractor's Brotli; one made from a JPEG keeps the boxes it gives that JPEG back from at their size. +libfiles' bounded Brotli decoder; one made from a JPEG keeps the boxes it gives that JPEG back from at their size. Its picture is never rewritten in place: rotating or editing one is not offered, since a re-encode would lose what it was encoded with. A pixel write that produces one - Convert, Save As, a shared copy - reads the packet as an update would, applies the edit and, from another format, what the source shows @@ -88,6 +88,9 @@ that its packet lacks, and encodes it into the file's own boxes beside the sourc follows. The codestream holds the orientation the file ends with, which the EXIF is made to agree with. A JPEG XL moved unchanged is copied as it is. +Encoded JPEG, PNG, WebP and JPEG XL buffers transfer ownership from libfiles +into the displayed image's blob without a second encoded-buffer copy. + Pasted clipboard bitmaps follow the same publication rule: the encoder writes an owned temporary file off the UI thread, then publishes it to the chosen auto-numbered destination only after the stream and encoder commit successfully. diff --git a/docs/implementation.md b/docs/implementation.md index 91e27492..3b26048f 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -112,7 +112,11 @@ through `index_items`, scan/database workers update per-file atomics in place, a lets folder replacement reject a node built before an in-place update. The revision protects the folder contents as a set of per-file metadata/search-presence/cache fields; immutable replacement alone is insufficient because scanning a whole folder for each individual metadata update would -clone large folder vectors and move file scanning back under publication contention. +clone large folder vectors and move file scanning back under publication contention. A second +counter, the search revision, rises with every in-place update except those to what is derived from an +item's bytes - duplicates, checksums and picture hashes - so the search index can tell a folder whose +captured contents changed from one that only gained a derived value it reads live; one counter cannot +serve both, because replacement must still reject a node built before a derived update. Indexing progress is an immutable snapshot owned by the index worker and atomically published for UI/sidebar readers; paired total/remaining counters must be read together, and handing every paint call through the index queue or posting a UI callback for each scanned file would either block Paint @@ -120,7 +124,15 @@ on Source work or create unbounded invalidation traffic. `view_state` owns the visit timeline on the UI thread, while `_visits_generation` is atomic only so the location worker can observe superseded visit derivations through `df::cancel_token`; the worker still publishes a complete detached `visit_timeline` back to the UI thread before the UI-owned value -is replaced. +is replaced. The address box's `_count_version` is atomic for the same reason: each keystroke raises +it on the UI thread, and the auto-complete worker observes it through `df::cancel_token` to stop +counting a superseded pass, which then publishes nothing. + +The search index is an index cache published as one immutable snapshot. Its own queue builds it with +no lock held, `index_state` swaps the pointer under the summary lock, and readers copy the pointer +under the same lock and evaluate with none held. An index updated in place would need a lock across +every evaluation that indexing writers then wait on; a snapshot plus the per-folder revisions it +records keeps readers and indexing apart. Three process-wide values are also published to every thread. A translation - and the extra plural forms of a plural text - is an atomic pointer into storage `app_text_t` only ever appends to: the UI @@ -143,8 +155,8 @@ publish before extraction completes. Search parses into exact matchers and conservative candidate prefilters. Negation and OR never create a prefilter that could omit an exact result. Query generations prevent stale result publication. Folder-scoped searches validate their filesystem scope; index-wide searches answer from cached -records. In-app file operations report every changed source and destination folder so search refresh -does not depend on a live watcher. +records, through the [search index](#search-index) where it is current. In-app file operations report +every changed source and destination folder so search refresh does not depend on a live watcher. The summary keeps authoritative tag histograms and word counts for every indexed tag. Tag-companion recommendations are auxiliary typeahead hints and are bounded per pathological item: when one item @@ -160,6 +172,50 @@ items, and presence consume the same duplicate relation so they cannot disagree Related-item collection scores the index against one anchor and keeps bounded best results per relation. Counting and display share the same collector and stable path tie-breaks. +### Search Index + +The search index is an immutable in-memory reverse index of the collection, read from the cached +records rather than the database, so it needs no schema change and no re-index. Items are numbered in +path order. Each distinct name, free-text value, formatted value, split part, and tag part maps to the +items holding it, and a trigram index over the distinct text values suggests which of them hold a +free-text term. A search's terms become item sets combined by `combine_search_terms`, the steps the +matcher applies to one item. The matcher's own predicate confirms every suggestion. Rating and label +terms read nothing of an item but that property, so items are grouped by its value and `match_term` +answers once per group. Date terms compare only the day each of an item's date keys falls on, so +`date_term_keys_of` and `date_term_found`, which the matcher uses too, decide them with each day +compared once. A term no table can answer, such as a size comparison, is answered by `match_term` over +each item's captured metadata. An answer from the index is therefore the answer a scan gives, and the +index tests compare the two over about 130 searches. + +Each folder entry keeps the folder node and search revision it was read from. A folder changed since +is scanned on top of the snapshot. Duplicates, checksums, picture hashes and presence are always read +live, so recording them leaves a folder current. A search uses the snapshot while at least half the +collection is unchanged and otherwise scans, as it would without one. Formatted values depend on the +language and number format, and date keys on the time zone and, through what counts as a valid date, +the day; when one changes, those terms are answered item by item until the next read, and maintenance +reads the collection again on a new day. + +Reading never delays indexing or a search. It runs on its own `search_index` queue, never +`index_compute`, a debounced second after the summary and duplicate-prediction passes, and only once +changes since the last read pass a tolerance that grows with the collection, or leave the snapshot one +no search would use. It does not start while indexing or scanning runs and abandons when either +starts; a snapshot too stale to use is then dropped so it stops holding replaced records. Nothing is +locked while reading or evaluating: publication swaps a pointer under the summary lock and releases the +old snapshot after it. A collection whose snapshot would pass a 1 GiB ceiling keeps none and scans. +Each snapshot published asks the sidebar to count again, since it may have counted by scanning. The +performance report's Search index rows show how many searches it answered and how long reads took and +how much memory they hold. + +Sidebar rows and address-box predictions are counted through `search_counter`, which lists the +collection once for a whole pass. Its evaluator keeps each term's items and the items through each +presence requirement for the pass, so the searches of a pass share them, and a count reads only the +folders the snapshot cannot answer for. A sidebar pass counts from the snapshot only while few items +have changed since it was read, since every row would read each of them; past that it counts as it did +without one. Prediction counts are budgeted per keystroke and stopped by the next one. When the +snapshot cannot answer cheaply - there is none yet, too many items changed, the search walks a folder +tree, or it is a related search - the count is left out rather than scanned, and a folder search with a +term is never counted, since that would read the folder's unread files on every keystroke. + ### Thumbnail Pipeline Layout dimensions, encoded thumbnail, decoded near-viewport surface, and GPU texture are separate @@ -246,7 +302,8 @@ repository lint, build, embedded tests, and translation validation. Focused test state. - [model_index.cpp](../src/model_index.cpp), [model_search.cpp](../src/model_search.cpp), [model_postings.h](../src/model_postings.h), and [model_db.cpp](../src/model_db.cpp): index, search, - candidate sets, and persistence. + candidate sets, and persistence; [model_search_index.cpp](../src/model_search_index.cpp): the + search index snapshot and its evaluation. - [model_index_duplicates.cpp](../src/model_index_duplicates.cpp): duplicate prediction and presence; [model_index_summary.cpp](../src/model_index_summary.cpp): the summary, its vocabulary, and auto-complete. diff --git a/docs/testing.md b/docs/testing.md index 7e7160e4..0e320c87 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -40,7 +40,7 @@ One file per subject, matching the `src/` prefixes in [implementation](implement | [test_metadata.cpp](../src/test_metadata.cpp) | `register_metadata_tests` | `metadata*`, `model_property`, `model_dates`, `model_tags`: EXIF, IPTC, XMP, ICC, per-format metadata scanning, the date pack and its authority table, the panorama declaration, sidecars, tag sets, property presentation | | [test_media_edit.cpp](../src/test_media_edit.cpp) | `register_media_edit_tests` | The metadata write path and the image editing model: in-place and staged edits, Windows shell tags, orientation, crop, perspective, temperature | | [test_av.cpp](../src/test_av.cpp) | `register_av_tests` | `av*`: probing, seeking, read-ahead bounds, hover preview, audio buffers, the visualizer, SID tunes, subtitles, session lifetime | -| [test_index.cpp](../src/test_index.cpp) | `register_index_tests` | `model_index`, `model_db`, `model_postings`: indexing, the inverted and trigram indexes, the database, the thumbnail pipeline, duplicates and presence, stale-result discarding, hydration | +| [test_index.cpp](../src/test_index.cpp) | `register_index_tests` | `model_index`, `model_db`, `model_postings`, `model_search_index`: indexing, the inverted and trigram indexes, the search index and its agreement with a scan, the database, the thumbnail pipeline, duplicates and presence, stale-result discarding, hydration | | [test_search.cpp](../src/test_search.cpp) | `register_search_tests` | `model_search`, `model_related`, `app_match`: query parsing, term matching, scopes, negation, exclusion, related items, prediction | | [test_locations.cpp](../src/test_locations.cpp) | `register_location_tests` | `model_locations`, `model_visits`, `ui_map*`, `ui_globe`, `model_tile_cache`: the gazetteer, place naming, map areas and geometry, the globe's projection and framing, visits, the timeline, the map tile cache. See [locations](locations.md) | | [test_movie.cpp](../src/test_movie.cpp) | `register_movie_tests` | `model_movie`: the Movie document -- derived output geometry, transition and duration arithmetic, timeline edits and undo, what entering Movie does with the timeline it holds, the OpenTimelineIO project file, the Movie Maker reader, finding a moved source among same-named files. See [movie](movie.md) | @@ -94,7 +94,7 @@ Where a decision is trapped inside a method that also performs it, extract the d | `should_resume_at` / `position_to_save` ([av_player.h](../src/av_player.h)) | the resume window and what a close saves | | `calc_playback_advance` ([util_interfaces.h](../src/util_interfaces.h)) | slideshow, repeat and continue-with-next, and how they interact | | `search_edit_session` ([app_search.h](../src/app_search.h)) | the address box editing session: draft, preview, two-stage Escape, Tab, Enter | -| `make_search_auto_complete` ([app_search.h](../src/app_search.h)) | the address bar's completion results: content, ranking, de-duplication, the cap, latest-wins | +| `make_search_auto_complete` ([app_search.h](../src/app_search.h)) | the address bar's completion results: content, counts, ranking, de-duplication, the cap, latest-wins | | `default_keyboard_accelerators` ([app_commands.h](../src/app_commands.h)) | the shipped key bindings, so a collision can be seen | A test that drives the completion strategy must run the pass on a real worker thread: `index_state::auto_complete_words` asserts it is not on the UI thread, which is the contract the shipped code keeps. `run_completion_worker` in [test_app.cpp](../src/test_app.cpp) models the hop with a `deferred_async_strategy`; running it inline trips the assertion rather than passing. That is also what makes the latest-wins test possible, because it can publish two passes out of order. diff --git a/docs/third-party.md b/docs/third-party.md index 41a8045d..3330e976 100644 --- a/docs/third-party.md +++ b/docs/third-party.md @@ -42,7 +42,7 @@ branch or the API version left in its headers. | Library | Imported release or upstream snapshot | Location | Type | Upstream / update source | | --- | --- | --- | --- | --- | -| Brotli | 1.2.1 | Fetched by libfiles | Release archive by SHA-256 | [google/brotli releases](https://github.com/google/brotli/releases). libfiles pins the archive in its `cmake/dd-dependencies.json` and builds its C decoder and encoder, every source this tree's vendored copy built, as `libfiles::brotli`; [CMakeLists.txt](../CMakeLists.txt) names it `diffractor::brotli`, which libjxl and [files_jxl.cpp](../src/files_jxl.cpp), for a JPEG XL's compressed metadata boxes, link | +| Brotli | 1.2.1 | Fetched by libfiles | Release archive by SHA-256 | [google/brotli releases](https://github.com/google/brotli/releases). libfiles pins the archive in its `cmake/dd-dependencies.json` and builds its C decoder and encoder as `libfiles::brotli`; libfiles uses its bounded decoder for JPEG XL compressed metadata boxes. [CMakeLists.txt](../CMakeLists.txt) also names the target `diffractor::brotli` for libjxl's codec dependency | | bzip2 | 1.0.8 | Fetched | Release archive by SHA-256 | [Sourceware](https://sourceware.org/bzip2/downloads.html). Pinned in [dd-dependencies.json](../cmake/dd-dependencies.json) and built unchanged by [bzip2.cmake](../cmake/vendored/bzip2.cmake): the copy once vendored here spelled four POSIX names the MSVC way, which the vendored policy's `_CRT_NONSTDC_NO_WARNINGS` makes needless | | dav1d | 1.5.4 | Fetched by libfiles | Release archive by SHA-256 | [VideoLAN](https://code.videolan.org/videolan/dav1d). libfiles pins the release archive in its `cmake/dd-dependencies.json` and builds it with the source list, configuration and bitdepth-template wrappers this tree's vendored copy used, as `libfiles::dav1d`; [CMakeLists.txt](../CMakeLists.txt) names it `diffractor::dav1d`, which libheif and FFmpeg's AV1 decoder link | | Adobe DNG SDK | 1.7.1 build 2768 | Fetched by libfiles | Release archive by SHA-256 | [Adobe source archive](https://download.adobe.com/pub/adobe/dng/dng_sdk_1_7_1_2768_20261013.zip). libfiles pins the archive in its `cmake/dd-dependencies.json` and builds `dng_sdk/source/` - every source but the SDK's XMP backend, which its [dng_xmp_sdk.cpp](https://github.com/ZacWalk/libfiles/blob/main/src/dng_xmp_sdk.cpp) replaces - with the `RawEnvironment.h` this tree's copy carried, as `libfiles::dng`; [CMakeLists.txt](../CMakeLists.txt) names it `diffractor::dng`, which LibRaw hands DNGs to | diff --git a/exe/languages/cs.po b/exe/languages/cs.po index 01b5d1fb..1416b02d 100644 --- a/exe/languages/cs.po +++ b/exe/languages/cs.po @@ -4090,3 +4090,9 @@ msgstr "Titulky {}" msgid "Subtitles" msgstr "Titulky" + +msgid "1 of the items shown" +msgid_plural "{count} of the items shown" +msgstr[0] "1 ze zobrazených položek" +msgstr[1] "{count} ze zobrazených položek" +msgstr[2] "{count} ze zobrazených položek" diff --git a/exe/languages/de.po b/exe/languages/de.po index 3e5317f5..1c531bba 100644 --- a/exe/languages/de.po +++ b/exe/languages/de.po @@ -3932,3 +3932,8 @@ msgstr "Untertitel {}" msgid "Subtitles" msgstr "Untertitel" + +msgid "1 of the items shown" +msgid_plural "{count} of the items shown" +msgstr[0] "1 der angezeigten Elemente" +msgstr[1] "{count} der angezeigten Elemente" diff --git a/exe/languages/es.po b/exe/languages/es.po index c6f42b9b..4a747684 100644 --- a/exe/languages/es.po +++ b/exe/languages/es.po @@ -3932,3 +3932,8 @@ msgstr "Subtítulos {}" msgid "Subtitles" msgstr "Subtítulos" + +msgid "1 of the items shown" +msgid_plural "{count} of the items shown" +msgstr[0] "1 de los elementos mostrados" +msgstr[1] "{count} de los elementos mostrados" diff --git a/exe/languages/fr.po b/exe/languages/fr.po index 62f99d3e..e9af9e82 100644 --- a/exe/languages/fr.po +++ b/exe/languages/fr.po @@ -3928,3 +3928,8 @@ msgstr "Sous-titres {}" msgid "Subtitles" msgstr "Sous-titres" + +msgid "1 of the items shown" +msgid_plural "{count} of the items shown" +msgstr[0] "1 des éléments affichés" +msgstr[1] "{count} des éléments affichés" diff --git a/exe/languages/it.po b/exe/languages/it.po index c671536c..c446fdca 100644 --- a/exe/languages/it.po +++ b/exe/languages/it.po @@ -3932,3 +3932,8 @@ msgstr "Sottotitoli {}" msgid "Subtitles" msgstr "Sottotitoli" + +msgid "1 of the items shown" +msgid_plural "{count} of the items shown" +msgstr[0] "1 degli elementi mostrati" +msgstr[1] "{count} degli elementi mostrati" diff --git a/exe/languages/ja.po b/exe/languages/ja.po index 624c7801..8344c54b 100644 --- a/exe/languages/ja.po +++ b/exe/languages/ja.po @@ -3928,3 +3928,8 @@ msgstr "字幕 {}" msgid "Subtitles" msgstr "字幕" + +msgid "1 of the items shown" +msgid_plural "{count} of the items shown" +msgstr[0] "表示中の項目のうち 1 件" +msgstr[1] "表示中の項目のうち {count} 件" diff --git a/exe/languages/ko.po b/exe/languages/ko.po index df73ca03..a7d1c7e8 100644 --- a/exe/languages/ko.po +++ b/exe/languages/ko.po @@ -3926,3 +3926,8 @@ msgstr "자막 {}" msgid "Subtitles" msgstr "자막" + +msgid "1 of the items shown" +msgid_plural "{count} of the items shown" +msgstr[0] "표시된 항목 중 1개" +msgstr[1] "표시된 항목 중 {count}개" diff --git a/exe/languages/pl.po b/exe/languages/pl.po index f447cce3..44f97f1f 100644 --- a/exe/languages/pl.po +++ b/exe/languages/pl.po @@ -3967,3 +3967,9 @@ msgstr "Napisy {}" msgid "Subtitles" msgstr "Napisy" + +msgid "1 of the items shown" +msgid_plural "{count} of the items shown" +msgstr[0] "1 z wyświetlanych elementów" +msgstr[1] "{count} z wyświetlanych elementów" +msgstr[2] "{count} z wyświetlanych elementów" diff --git a/exe/languages/pt.po b/exe/languages/pt.po index 46adbd6f..65200f4a 100644 --- a/exe/languages/pt.po +++ b/exe/languages/pt.po @@ -3927,3 +3927,8 @@ msgstr "Legendas {}" msgid "Subtitles" msgstr "Legendas" + +msgid "1 of the items shown" +msgid_plural "{count} of the items shown" +msgstr[0] "1 dos itens mostrados" +msgstr[1] "{count} dos itens mostrados" diff --git a/exe/languages/ru.po b/exe/languages/ru.po index d14a8d8f..8b5592f0 100644 --- a/exe/languages/ru.po +++ b/exe/languages/ru.po @@ -3968,3 +3968,9 @@ msgstr "Субтитры {}" msgid "Subtitles" msgstr "Субтитры" + +msgid "1 of the items shown" +msgid_plural "{count} of the items shown" +msgstr[0] "1 из показанных элементов" +msgstr[1] "{count} из показанных элементов" +msgstr[2] "{count} из показанных элементов" diff --git a/exe/languages/tr.po b/exe/languages/tr.po index 644a624b..f9dd2646 100644 --- a/exe/languages/tr.po +++ b/exe/languages/tr.po @@ -3928,3 +3928,8 @@ msgstr "Altyazı {}" msgid "Subtitles" msgstr "Altyazılar" + +msgid "1 of the items shown" +msgid_plural "{count} of the items shown" +msgstr[0] "Gösterilen öğelerden 1" +msgstr[1] "Gösterilen öğelerden {count}" diff --git a/exe/languages/uk.po b/exe/languages/uk.po index 44186aa5..5affa198 100644 --- a/exe/languages/uk.po +++ b/exe/languages/uk.po @@ -3995,3 +3995,9 @@ msgstr "Субтитри {}" msgid "Subtitles" msgstr "Субтитри" + +msgid "1 of the items shown" +msgid_plural "{count} of the items shown" +msgstr[0] "1 з показаних елементів" +msgstr[1] "{count} з показаних елементів" +msgstr[2] "{count} з показаних елементів" diff --git a/exe/languages/zh.po b/exe/languages/zh.po index 09cd37bf..5895b20b 100644 --- a/exe/languages/zh.po +++ b/exe/languages/zh.po @@ -3926,3 +3926,8 @@ msgstr "字幕 {}" msgid "Subtitles" msgstr "字幕" + +msgid "1 of the items shown" +msgid_plural "{count} of the items shown" +msgstr[0] "显示的项目中有 1 项" +msgstr[1] "显示的项目中有 {count} 项" diff --git a/src/app.h b/src/app.h index 657d3b70..a426add8 100644 --- a/src/app.h +++ b/src/app.h @@ -599,6 +599,7 @@ class app_frame final : platform::task_queue predictions_task_queue; platform::task_queue summary_task_queue; platform::task_queue presence_task_queue; + platform::task_queue search_index_task_queue; platform::task_queue auto_complete_task_queue; platform::task_queue query_task_queue; platform::task_queue render_task_queue; @@ -622,6 +623,7 @@ class app_frame final : std::atomic_bool _web_worker_started = false; std::atomic_bool _cloud_worker_started = false; std::atomic_bool _auto_complete_worker_started = false; + std::atomic_bool _search_index_worker_started = false; std::atomic_bool _map_tile_workers_started = false; std::atomic_bool _tile_db_worker_started = false; std::atomic_bool _media_preview_worker_started = false; diff --git a/src/app_match.h b/src/app_match.h index 42180ac2..afe988d8 100644 --- a/src/app_match.h +++ b/src/app_match.h @@ -7,7 +7,8 @@ // This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY // Purpose: Auto-complete matching logic for search. Implements folder and text matching -// with highlight support for the search box dropdown suggestions. +// with highlight support for the search box dropdown suggestions, and draws the count of +// items each suggestion finds. #pragma once #include "ui_controllers.h" @@ -99,6 +100,22 @@ static std::string auto_complete_lead(std::string lead) return lead; } +// The count a prediction carries, right-aligned and quieter than its text. Answers the width it took so +// the text stops short of it. None is drawn as nothing, as the sidebar draws it. +static int draw_match_count(ui::draw_context& dc, const std::optional& count, const recti bounds) +{ + if (!count || *count == 0) return 0; + + const auto text = str::format_count(*count); + const auto extent = dc.measure_text(text, ui::style::font_face::dialog, ui::style::text_style::single_line, + bounds.width()); + auto count_bounds = bounds; + count_bounds.left = std::max(bounds.left, bounds.right - extent.cx); + dc.draw_text(text, {}, count_bounds, ui::style::font_face::dialog, ui::style::text_style::single_line, + ui::color(dc.colors.foreground, dc.colors.alpha * 0.6f), {}); + return extent.cx + dc.padding2; +} + static icon_index search_icon(const std::string_view text) { if (df::is_path(str::trim(text))) return icon_index::folder; @@ -180,6 +197,7 @@ class folder_match final : public ui::auto_complete_match, public std::enable_sh icon_bounds.right = icon_bounds.left + icon_width; xdraw_icon(dc, icon_index::folder, icon_bounds, clr, {}); rr.left = icon_bounds.right + dc.padding2; + rr.right -= draw_match_count(dc, count, rr); if (!str::is_empty(lead)) { @@ -260,6 +278,7 @@ class text_match final : public ui::auto_complete_match, public std::enable_shar icon_bounds.right = icon_bounds.left + icon_width; xdraw_icon(dc, icon, icon_bounds, clr, {}); rr.left = icon_bounds.right + dc.padding2; + rr.right -= draw_match_count(dc, count, rr); if (!str::is_empty(lead)) { diff --git a/src/app_perf.cpp b/src/app_perf.cpp index 374add21..a813fc61 100644 --- a/src/app_perf.cpp +++ b/src/app_perf.cpp @@ -252,6 +252,22 @@ void df::log_perf_summary() format_average(load(q.count_us), load(q.counts)), "-"); } + if (load(q.index_answers) != 0) + { + append_perf_row(report, "Query", "Search index", format_count(load(q.index_answers)), "-", "-", "-", + std::format("changed-folders-scanned={}", format_count(load(q.index_scanned_folders)))); + } + + if (load(x.search_index_builds) != 0) + { + append_perf_row(report, "Index", "Search index build", format_count(load(x.search_index_builds)), + format_us(load(x.search_index_build_us)), + format_average(load(x.search_index_build_us), load(x.search_index_builds)), + format_us(load(x.search_index_build_max_us)), + std::format("items={}; memory={}", format_count(load(x.search_index_docs)), + file_size(load(x.search_index_bytes)).str())); + } + if (load(f.scans) != 0) { append_perf_row(report, "Files", "Scan", format_count(load(f.scans)), format_us(load(f.scan_us)), diff --git a/src/app_perf.h b/src/app_perf.h index a43de3c2..2b4040f1 100644 --- a/src/app_perf.h +++ b/src/app_perf.h @@ -62,6 +62,10 @@ namespace df std::atomic_uint64_t materialize_items = 0; std::atomic_uint64_t counts = 0; std::atomic_uint64_t count_us = 0; + // Searches and counts the search index answered, and the folders changed since it was read that + // they scanned on top of it - which is what the index costs to keep current between rebuilds. + std::atomic_uint64_t index_answers = 0; + std::atomic_uint64_t index_scanned_folders = 0; }; struct file_counters @@ -123,6 +127,13 @@ namespace df std::atomic_uint32_t pass_aspect_solo = 0; std::atomic_uint32_t pass_aspect_solo_swap = 0; std::atomic_uint32_t pass_matched_cross_aspect = 0; + + // Reading the collection into the search index. The gauges hold what the latest one holds. + std::atomic_uint64_t search_index_builds = 0; + std::atomic_uint64_t search_index_build_us = 0; + std::atomic_uint32_t search_index_build_max_us = 0; + std::atomic_uint64_t search_index_docs = 0; + std::atomic_uint64_t search_index_bytes = 0; }; // One slot per worker queue, claimed once by its worker thread at startup so the dispatch loop diff --git a/src/app_search.cpp b/src/app_search.cpp index df53e9f8..a01bdd3c 100644 --- a/src/app_search.cpp +++ b/src/app_search.cpp @@ -8,7 +8,8 @@ // Purpose: The address box. Search editing, focus, Enter, Cancel and accepting a prediction, and // the auto-complete that ranks history, recent folders, tags, index vocabulary and scoped values -// (tag:, with:, ...) on the auto-complete worker before publishing them to the prediction list. +// (tag:, with:, ...) on the auto-complete worker, counting what each finds through the search index, +// before publishing them to the prediction list. #include "pch.h" @@ -38,6 +39,12 @@ class search_auto_complete final : public std::enable_shared_from_this vocabulary_match(std::string text, std::string lead, ui::match_highlights highlights, + const int weight, const icon_index icon) + { + auto result = std::make_shared(*this, std::move(text), std::move(lead), std::move(highlights), + weight, icon); + result->drop_when_empty = true; + return result; + } + + // What choosing a prediction searches for. Read without asking the file system, because a path + // on a share that has gone away would hold the list until the network gave up. + static std::optional prediction_search(const ui::auto_complete_match& match) + { + if (const auto* const folder = dynamic_cast(&match); folder && str::is_empty(folder->lead)) + { + df::search_t result; + result.add_selector(df::item_selector(folder->folder)); + return result; + } + + const auto result = df::search_t::parse_without_probing(match.edit_text()); + if (result.is_empty()) return std::nullopt; + return result; + } + + // Ten times as many items is worth the same step in rank wherever it starts. + static int count_rank(const uint64_t count) + { + return static_cast(std::lround(8.0 * std::log10(1.0 + static_cast(count)))); + } + + // Each prediction says how many items it finds. One that finds more ranks above one of its kind that + // finds fewer, and one drawn from the collection's vocabulary that finds nothing is left out. Only + // the search index is asked, within a budget, and a newer keystroke stops the pass, so a prediction + // left uncounted keeps its place rather than waiting on a scan. Answers false for a superseded pass. + bool count_predictions(ui::auto_complete_results& found, const uint64_t search_generation, const bool rank, + const df::cancel_token& token) + { + // Likeliest first, so the budget is spent where it decides the most, and each text counted once. + std::ranges::stable_sort(found, [](const auto& a, const auto& b) { return a->weight > b->weight; }); + + df::hash_set seen; + std::erase_if(found, [&seen](const auto& candidate) + { + return !seen.emplace(candidate->edit_text()).second; + }); + + search_counter counter(_state.item_index); + const auto deadline = df::now_us() + prediction_count_budget_us; + + for (const auto& match : found) + { + if (_search_generation != search_generation || token.is_cancelled()) return false; + if (df::now_us() > deadline) break; + + if (const auto search = prediction_search(*match)) + { + match->count = counter.try_count_items(*search, token); + } + } + + if (_search_generation != search_generation || token.is_cancelled()) return false; + + std::erase_if(found, [](const auto& match) { return match->drop_when_empty && match->count == 0u; }); + + if (rank) + { + for (const auto& match : found) + { + if (match->count) match->weight += count_rank(*match->count); + } + + std::ranges::stable_sort(found, [](const auto& a, const auto& b) { return a->weight > b->weight; }); + } + + return true; + } + // Issue #157: complete the value part of a scoped token (tag: / # / @ / with: / without:) // from the matching vocabulary, so e.g. "tag:do" suggests tags starting with "do" and // "with:ex" suggests property scopes. Runs on the auto_complete worker thread. Returns @@ -119,9 +206,8 @@ class search_auto_complete final : public std::enable_shared_from_this(*this, word.text, scope.lead, word.highlights, - scope_weight + std::min(word.occurrences, 20), - icon_index::tag)); + found.emplace_back(vocabulary_match(word.text, scope.lead, word.highlights, + scope_weight + std::min(word.occurrences, 20), icon_index::tag)); } } @@ -133,9 +219,8 @@ class search_auto_complete final : public std::enable_shared_from_this= max_predictions) break; if (scope.value.empty() || str::starts(ks.scope, scope.value)) { - found.emplace_back(std::make_shared(*this, scope.format(ks.scope), scope.lead, - ui::match_highlights{}, scope_weight, - search_icon(scope.format(ks.scope)))); + found.emplace_back(vocabulary_match(scope.format(ks.scope), scope.lead, ui::match_highlights{}, + scope_weight, search_icon(scope.format(ks.scope)))); } } } @@ -147,10 +232,9 @@ class search_auto_complete final : public std::enable_shared_from_this= max_predictions) break; - found.emplace_back(std::make_shared(*this, scope.format(loc.text), scope.lead, - loc.highlights, - scope_weight + std::min(loc.occurrences, 20), - icon_index::location)); + found.emplace_back(vocabulary_match(scope.format(loc.text), scope.lead, loc.highlights, + scope_weight + std::min(loc.occurrences, 20), + icon_index::location)); } } else @@ -159,11 +243,10 @@ class search_auto_complete final : public std::enable_shared_from_this= max_predictions) break; - found.emplace_back(std::make_shared(*this, scope.format(w.text), scope.lead, w.highlights, - scope_weight, - scope.kind == df::search_scope_kind::tag - ? icon_index::tag - : search_icon(scope.format(w.text)))); + found.emplace_back(vocabulary_match(scope.format(w.text), scope.lead, w.highlights, scope_weight, + scope.kind == df::search_scope_kind::tag + ? icon_index::tag + : search_icon(scope.format(w.text)))); } } @@ -174,6 +257,8 @@ class search_auto_complete final : public std::enable_shared_from_this(_known); + // Taken here, on the UI thread, so this keystroke stops any counting the last one is still doing. + const df::cancel_token count_token(_count_version); // The active search and the recent-search list are UI-thread owned and are mutated by // view_state::open while a prediction is still running, so snapshot them here. @@ -184,7 +269,7 @@ class search_auto_complete final : public std::enable_shared_from_this( - *this, word.text, std::string{}, - word.highlights, - calc_auto_complete_word_weight(word, *known))); + found.emplace_back(vocabulary_match( + word.text, std::string{}, word.highlights, + calc_auto_complete_word_weight(word, *known), icon_index::search)); } } @@ -285,11 +370,10 @@ class search_auto_complete final : public std::enable_shared_from_this( - *this, word.text, std::string{}, - word.highlights, + found.emplace_back(vocabulary_match( + word.text, std::string{}, word.highlights, 15 + std::min(word.occurrences, 10), icon_index::tag)); } } @@ -306,11 +390,9 @@ class search_auto_complete final : public std::enable_shared_from_this( - *this, location.text, std::string{}, - location.highlights, - 12 + std::min(location.occurrences, 10), - icon_index::location)); + found.emplace_back(vocabulary_match( + location.text, std::string{}, location.highlights, + 12 + std::min(location.occurrences, 10), icon_index::location)); } } } @@ -322,11 +404,13 @@ class search_auto_complete final : public std::enable_shared_from_this( - *this, word.text, lead_text, word.highlights, - calc_auto_complete_word_weight(word, *known))); + found.emplace_back(vocabulary_match( + word.text, lead_text, word.highlights, + calc_auto_complete_word_weight(word, *known), icon_index::search)); } if (found.size() < max_predictions) @@ -356,8 +440,8 @@ class search_auto_complete final : public std::enable_shared_from_this( - *this, word.text, query, word.highlights, + found.emplace_back(vocabulary_match( + word.text, query, word.highlights, 30 + std::min(word.occurrences, 20), icon_index::tag)); } @@ -378,9 +462,9 @@ class search_auto_complete final : public std::enable_shared_from_this( - *this, word.text, query, word.highlights, - calc_auto_complete_word_weight(word, *known))); + found.emplace_back(vocabulary_match( + word.text, query, word.highlights, + calc_auto_complete_word_weight(word, *known), icon_index::search)); } } @@ -391,9 +475,9 @@ class search_auto_complete final : public std::enable_shared_from_this( - *this, word.text, query, word.highlights, - calc_auto_complete_word_weight(word, *known))); + found.emplace_back(vocabulary_match( + word.text, query, word.highlights, + calc_auto_complete_word_weight(word, *known), icon_index::tag)); } } } @@ -412,18 +496,16 @@ class search_auto_complete final : public std::enable_shared_from_thisweight > b->weight; - }); + // Rank higher-weight suggestions first (literal query, contextual/scoped values, exact/recent + // matches), then by how many items each finds, preserving discovery order for equal weights. + // Recent searches keep their order: how recent a search is, is why it is offered. + const auto showing_recents = query_parts.empty() || + str::icmp(query, current_search_text) == 0; - df::hash_set seen; - std::erase_if(found, [&seen](const auto& candidate) + if (!count_predictions(found, search_generation, !showing_recents, count_token)) { - return !seen.emplace(candidate->edit_text()).second; - }); + return; + } if (found.size() > max_predictions) found.resize(max_predictions); _state.queue_ui([keep_alive, complete, found, search_generation] diff --git a/src/app_sidebar.h b/src/app_sidebar.h index b2ff127f..44d6cebc 100644 --- a/src/app_sidebar.h +++ b/src/app_sidebar.h @@ -20,7 +20,8 @@ // searches, the tag vocabulary or the sidebar's own settings. // sidebar_file_types_and_dates projection - the type and history charts, read straight from the // index histograms. No queries, no element churn. -// sidebar_counts the per-row sums. One index query per row, so it never rides +// sidebar_counts the per-row sums. Read from the search index in one pass when it +// can answer, otherwise one index query per row, so it never rides // along with a structural rebuild it did not need. // sidebar_drives volume enumeration, which blocks on unreachable network mappings. // @@ -174,6 +175,11 @@ class sidebar_element final : public view_element, public std::enable_shared_fro df::file_group_histogram summary; // Distinguishes "not counted yet" from "counted, and the answer is none" - both render blank. bool summary_known = false; + // calc_sum reads a summary the index already holds rather than searching, so a structural rebuild + // may ask it when the search index cannot answer. + bool sum_is_cheap = false; + // How many of the items the view shows this row's search keeps, when the search index could say. + std::optional scoped_count; ui::color32 clr = 0; int icon_repeat = 1; mutable sidebar_tooltip_thumbnail _tooltip_thumbnail; @@ -303,6 +309,15 @@ class sidebar_element final : public view_element, public std::enable_shared_fro } result.elements->add(std::make_shared(summary, flex_item::new_line)); + + if (scoped_count) + { + result.elements->add(std::make_shared(format_plural_text(tt.sidebar_kept_fmt, static_cast(*scoped_count)), + ui::style::font_face::dialog, + ui::style::text_style::multiline, + flex_item::new_line)); + } + result.active_bounds = result.window_bounds = bounds.offset(element_offset); result.horizontal = true; } @@ -611,6 +626,7 @@ class search_item_factory { return s.item_index.tag_summary(tag); }; + i->sum_is_cheap = true; i->summary = t.second; results.emplace_back(i); } @@ -630,6 +646,7 @@ class search_item_factory { return s.item_index.rating_summary(rating); }; + i->sum_is_cheap = true; i->summary = s.item_index.rating_summary(rating); return i; } @@ -697,6 +714,7 @@ class search_item_factory { return s.item_index.label_summary(label); }; + i->sum_is_cheap = true; i->summary = lab.second; results.emplace_back(i); } @@ -721,6 +739,45 @@ class search_item_factory return result; } + + // Rows arrive counted, so a rebuild never shows them loading once the index can answer: from the + // search index, which makes every total what clicking its row finds, or - until the index has been + // read - from the summaries the cheap rows already hold. A tag or label row that finds nothing leads + // nowhere and is left out, but only once counts can be trusted. + static std::vector count_rows(view_state& s, search_counter& counter, + std::vector rows, const bool hide_empty) + { + if (!s.item_index.is_init_complete()) return rows; + + const df::cancel_token token; + const auto indexed = counter.is_indexed(); + + for (const auto& row : rows) + { + // A favorite scoped to a folder is answered by walking that tree, which waits for the counts pass + // that follows rather than holding up the rows. + if (indexed && !row->search.has_selector() && !row->search.has_related()) + { + row->summary = counter.count(row->search, token); + row->summary_known = true; + } + else if (!indexed && row->sum_is_cheap && row->calc_sum) + { + row->summary = row->calc_sum(s, token); + row->summary_known = true; + } + } + + if (hide_empty) + { + std::erase_if(rows, [](const search_item_ptr& row) + { + return row->summary_known && row->summary.total_items().count == 0; + }); + } + + return rows; + } }; static int calc_indexing_perc(const index_progress_snapshot& stats) @@ -3045,6 +3102,8 @@ class sidebar_host final : public std::enable_shared_from_this, pu drive_info = _drive_info, config, request_id] { constexpr search_item_factory f; + // One listing of the index answers every row of this rebuild. + search_counter counter(s.item_index); std::vector items; std::vector drives; std::vector item_elements; @@ -3070,7 +3129,8 @@ class sidebar_host final : public std::enable_shared_from_this, pu if (config.sidebar.show_favorite_searches) { - add_elements(f.create_search_items(s, existing, config)); + add_elements(search_item_factory::count_rows( + s, counter, f.create_search_items(s, existing, config), false)); } if (config.sidebar.show_drives) @@ -3084,17 +3144,18 @@ class sidebar_host final : public std::enable_shared_from_this, pu if (config.sidebar.show_ratings) { - add_elements(f.create_ratings(s, existing)); + add_elements(search_item_factory::count_rows(s, counter, f.create_ratings(s, existing), false)); } if (config.sidebar.show_labels) { - add_elements(f.create_labels(s, existing)); + add_elements(search_item_factory::count_rows(s, counter, f.create_labels(s, existing), true)); } if (config.sidebar.show_tags) { - add_elements(f.create_tags(s, existing, config)); + add_elements(search_item_factory::count_rows(s, counter, f.create_tags(s, existing, config), + true)); } @@ -3222,42 +3283,107 @@ class sidebar_host final : public std::enable_shared_from_this, pu // calc_sum only queries item_index, which is synchronized, so it is safe off the UI thread. The // elements themselves are not: they own text layouts, so the list is guarded rather than copied // loose into the worker, and the view is reached through a weak reference locked on the UI thread. + // A row's search is set when the row is made and never changes, so the worker reads it as safely. auto elements = ui_owned(_state._async, std::make_shared>(_items)); - _state.queue_async(async_queue::sidebar, [weak = weak_from_this(), &s = _state, elements, token] + // What the view shows, by path, so each row can say how many of those items it keeps. Folders are + // left out: no row counts them. + auto shown = std::make_shared>(); + shown->reserve(_state.display_items().size()); + + for (const auto& i : _state.display_items()._items) { - df::scope_locked_inc slc(df::jobs_running); + if (!i->is_folder()) shown->emplace_back(i->path()); + } - int update_count = 0; + _state.queue_async(async_queue::sidebar, [weak = weak_from_this(), &s = _state, elements, token, shown, + current_search = _state.search()] + { + df::scope_locked_inc slc(df::jobs_running); + search_counter counter(s.item_index); - for (const auto& i : *elements) - { - if (i->calc_sum) - { - const auto sum = i->calc_sum(s, token); + if (counter.is_indexed()) + { + // Every row from the search index, so each total is what clicking the row finds, + // published together because together they take a few milliseconds. + struct row_count + { + search_item_ptr row; + df::file_group_histogram sum; + std::optional kept; + }; + + std::vector counts; + counts.reserve(elements->size()); + const auto kept_items = shown->empty() + ? search_kept_items{} + : counter.make_kept_items(*shown); + + for (const auto& i : *elements) + { + if (token.is_cancelled()) return; - if (!token.is_cancelled()) - { - update_count += 1; + auto& c = counts.emplace_back(); + c.row = i; + c.sum = counter.count(i->search, token); - //if (i->summary != sum) - { - s.queue_ui([weak, i, sum, token] - { - if (token.is_cancelled()) return; - const auto t = weak.lock(); - if (!t) return; - i->summary_known = true; - i->summary = sum; - t->invalidate(); - }); - } - } - } - } + // The row the view shows keeps everything it shows, which is not worth saying. + if (!shown->empty() && i->search != current_search) + { + c.kept = counter.count_kept(i->search, kept_items, token); + } + } - df::trace(std::format("Sidebar update {} predictions", update_count)); - }); + if (token.is_cancelled()) return; + + s.queue_ui([weak, counts = std::move(counts), token] + { + if (token.is_cancelled()) return; + const auto t = weak.lock(); + if (!t) return; + + for (const auto& c : counts) + { + c.row->summary_known = true; + c.row->summary = c.sum; + c.row->scoped_count = c.kept; + } + + t->invalidate(); + }); + + df::trace(std::format("Sidebar update {} predictions from the search index", counts.size())); + return; + } + + int update_count = 0; + + for (const auto& i : *elements) + { + if (i->calc_sum) + { + const auto sum = i->calc_sum(s, token); + + if (!token.is_cancelled()) + { + update_count += 1; + + s.queue_ui([weak, i, sum, token] + { + if (token.is_cancelled()) return; + const auto t = weak.lock(); + if (!t) return; + i->summary_known = true; + i->summary = sum; + i->scoped_count.reset(); + t->invalidate(); + }); + } + } + } + + df::trace(std::format("Sidebar update {} predictions", update_count)); + }); } view_controller_ptr controller_from_location(const pointi loc, hit_test_context& ctx) override diff --git a/src/app_text.cpp b/src/app_text.cpp index 7ad4bcb5..653cfbd2 100644 --- a/src/app_text.cpp +++ b/src/app_text.cpp @@ -1936,6 +1936,7 @@ void app_text_t::calc_text_mapping() movie_ignored_fmt, movie_missing_fmt, movie_relink_fmt, + sidebar_kept_fmt, }; text_mapping mapping; diff --git a/src/app_text.h b/src/app_text.h index e8556e1d..9ea05396 100644 --- a/src/app_text.h +++ b/src/app_text.h @@ -1555,6 +1555,8 @@ struct app_text_t "1 clip cannot find its source file.", "{count} clips cannot find their source files." }; plural_text movie_relink_fmt = {"1 clip was relinked.", "{count} clips were relinked."}; + // A sidebar row's tooltip: how many of the items the view shows the row's search would keep. + plural_text sidebar_kept_fmt = {"1 of the items shown", "{count} of the items shown"}; }; extern app_text_t tt; diff --git a/src/app_workers.cpp b/src/app_workers.cpp index a90c6d45..0b1312d4 100644 --- a/src/app_workers.cpp +++ b/src/app_workers.cpp @@ -467,6 +467,10 @@ void app_frame::queue_async(const async_queue q, std::function f) case async_queue::index_presence_single: presence_task_queue.reset_and_enqueue(std::move(f)); break; + case async_queue::index_search_single: + ensure_worker(_search_index_worker_started, search_index_task_queue, "search_index"); + search_index_task_queue.reset_and_enqueue(std::move(f)); + break; case async_queue::auto_complete: ensure_worker(_auto_complete_worker_started, auto_complete_task_queue, "auto_complete"); auto_complete_task_queue.reset_and_enqueue(std::move(f)); @@ -515,6 +519,10 @@ void app_frame::queue_async_after(const async_queue q, const uint32_t delay_ms, case async_queue::index_summary_single: summary_task_queue.enqueue_after(delay_ms, std::move(f)); break; + case async_queue::index_search_single: + ensure_worker(_search_index_worker_started, search_index_task_queue, "search_index"); + search_index_task_queue.enqueue_after(delay_ms, std::move(f)); + break; default: df::assert_true(false); queue_async(q, std::move(f)); diff --git a/src/files.cpp b/src/files.cpp index 0b1c4f7b..2931f172 100644 --- a/src/files.cpp +++ b/src/files.cpp @@ -5167,7 +5167,7 @@ df::blob turn_jpeg_losslessly(const libfiles::jpeg_transformer& transformer, lib // simple_transform counts from none at 0, where the EXIF orientation libfiles takes counts from 1. const auto orientation = static_cast(static_cast(transform) + 1); const auto budget = static_cast(std::max(df::max_decode_bytes, 0)); - const auto turned = transformer.turn(src, encoder, orientation, budget); + auto turned = transformer.turn(src, encoder, orientation, budget); if (turned.over_budget) { @@ -5177,7 +5177,7 @@ df::blob turn_jpeg_losslessly(const libfiles::jpeg_transformer& transformer, lib if (over_budget) *over_budget = true; } - return df::blob(turned.data.begin(), turned.data.end()); + return df::blob(std::move(turned.data)); } catch (const libfiles::encode_error& e) { @@ -5200,11 +5200,11 @@ ui::image_ptr save_jpeg(const ui::const_surface_ptr& surface_in, const libfiles: try { - const auto data = e.encode(dimensions_in.cx, dimensions_in.cy, surface_in->pixels(), surface_in->stride(), + auto data = e.encode(dimensions_in.cx, dimensions_in.cy, surface_in->pixels(), surface_in->stride(), static_cast(orientation_in), metadata, options); log_notes(e); - return std::make_shared(df::blob(data.begin(), data.end()), + return std::make_shared(df::blob(std::move(data)), sizei{static_cast(e.width()), static_cast(e.height())}, ui::image_format::JPEG, orientation_in); } @@ -5242,7 +5242,7 @@ ui::image_ptr save_png(const ui::const_surface_ptr& surface_in, const libfiles:: // libpng's warnings go to the quiet debug log rather than the console and the test output. for (const auto& note : encoded.notes) df::log_once("libpng", note); - return std::make_shared(df::blob(encoded.data.begin(), encoded.data.end()), dims, + return std::make_shared(df::blob(std::move(encoded.data)), dims, ui::image_format::PNG, surface_in->orientation()); } catch (const libfiles::encode_error& e) @@ -5422,12 +5422,12 @@ ui::image_ptr save_webp(const ui::const_surface_ptr& surface_in, const libfiles: options.lossy_alpha = params.webp_lossy_alpha; options.fast = params.webp_fast; - const auto encoded = libfiles::encode_webp(view, static_cast(surface_in->orientation()), + auto encoded = libfiles::encode_webp(view, static_cast(surface_in->orientation()), metadata, options); for (const auto& note : encoded.notes) df::log(__FUNCTION__, note); if (encoded.data.empty()) return {}; - return std::make_shared(df::cspan(encoded.data.data(), encoded.data.size()), dimensions, + return std::make_shared(df::blob(std::move(encoded.data)), dimensions, ui::image_format::WEBP, surface_in->orientation()); } diff --git a/src/files.h b/src/files.h index 2d950b01..631a54e6 100644 --- a/src/files.h +++ b/src/files.h @@ -1485,15 +1485,9 @@ struct still_scan_extras file_scan_result scan_still_libfiles(read_stream& s, libfiles::scan_intent intent, bool want_thumbnail, files* decoder, const still_scan_extras& extras); -// HEIF/AVIF and JPEG XL through libfiles. libfiles' decoder decodes a HEIF thumbnail item for a caller -// that wants one, and the Brotli a JPEG XL's compressed boxes need is supplied here. +// HEIF/AVIF and JPEG XL through libfiles, including a HEIF thumbnail item and compressed JPEG XL boxes. file_scan_result scan_heif_libfiles(read_stream& s, libfiles::scan_intent intent, bool want_thumbnail, files* decoder); file_scan_result scan_jxl_libfiles(read_stream& s, libfiles::scan_intent intent, bool want_thumbnail, files* decoder); -// Decompresses a JPEG XL's Brotli-compressed metadata box into at most max_size bytes, or answers nothing: -// the Brotli libfiles asks for to scan one, read its packet or update it. -std::optional> decompress_jxl_box(std::span compressed, std::string_view box_type, - size_t max_size); - // An audio or video file as libfiles reads it, with its cover art's encoded bytes when asked for. struct media_scan_read { diff --git a/src/files_jxl.cpp b/src/files_jxl.cpp index 60090bae..f9fe5152 100644 --- a/src/files_jxl.cpp +++ b/src/files_jxl.cpp @@ -8,58 +8,15 @@ // Purpose: JPEG XL decoding and encoding through libfiles: decoding into a surface judged against the // decode budget first, and encoding a surface with its metadata for a save. libfiles scans these files -// too; this supplies the Brotli decoder its scan asks for when a container keeps its metadata boxes -// compressed. +// too, including Brotli-compressed metadata boxes through its own decoder. #include "pch.h" #include "files.h" #include "libfiles/encode.h" -#include - -// The first buffer a compressed metadata box is decompressed into, grown as far as libfiles allows. -constexpr size_t box_chunk = 64u * 1024u; - -// What libfiles asks for when a JPEG XL keeps a metadata box compressed ('brob'): the box's one whole -// Brotli stream, decompressed into at most max_size bytes. A stream that would need more is abandoned -// there rather than decompressed to the end. -std::optional> decompress_jxl_box(const std::span compressed, std::string_view, - const size_t max_size) -{ - const df::releaser dec(BrotliDecoderCreateInstance(nullptr, nullptr, nullptr), - [](auto* d) { BrotliDecoderDestroyInstance(d); }); - if (!dec.get()) return {}; - - std::vector out(std::min(max_size, box_chunk)); - auto available_in = compressed.size(); - const auto* next_in = compressed.data(); - size_t produced = 0; - - for (;;) - { - auto available_out = out.size() - produced; - auto* next_out = out.data() + produced; - const auto status = BrotliDecoderDecompressStream(dec.get(), &available_in, &next_in, &available_out, - &next_out, nullptr); - produced = out.size() - available_out; - - if (status == BROTLI_DECODER_RESULT_SUCCESS) - { - // One whole stream and nothing after it, as libjxl requires of a 'brob' box. - if (available_in != 0) return {}; - out.resize(produced); - return out; - } - - if (status != BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT || out.size() >= max_size) return {}; - out.resize(std::min(max_size, out.size() * 2u)); - } -} - file_scan_result scan_jxl_libfiles(read_stream& s, libfiles::scan_intent, const bool want_thumbnail, files* const decoder) { still_scan_extras extras; - extras.brotli = decompress_jxl_box; // Scanned for indexing whatever the intent: libjxl's scan gave JPEG XL no structure report. auto result = scan_still_libfiles(s, libfiles::scan_intent::index, want_thumbnail, decoder, extras); @@ -156,11 +113,11 @@ ui::image_ptr save_jxl(const ui::const_surface_ptr& surface_in, const libfiles:: options.lossless = params.jxl_lossless; options.quality = params.jxl_quality; - const auto encoded = libfiles::encode_jxl(view, static_cast(turn), metadata, + auto encoded = libfiles::encode_jxl(view, static_cast(turn), metadata, options); for (const auto& note : encoded.notes) df::log(__FUNCTION__, note); if (encoded.data.empty()) return {}; - return std::make_shared(df::cspan(encoded.data.data(), encoded.data.size()), dimensions, + return std::make_shared(df::blob(std::move(encoded.data)), dimensions, ui::image_format::JXL, turn); } diff --git a/src/metadata.cpp b/src/metadata.cpp index a02c04ad..37e49503 100644 --- a/src/metadata.cpp +++ b/src/metadata.cpp @@ -833,8 +833,8 @@ static void require_updatable(const df::file_path path) if (!libfiles::can_update(libfiles_update_format(path))) throw app_exception("this file format cannot store the metadata"); } -// The packet a file holds, where libfiles reads and writes it - a JPEG XL's with the Brotli its scan is -// given. Nothing when libfiles does not read the format or the file cannot be opened. +// The packet a file holds, where libfiles reads and writes it, including compressed JPEG XL boxes. +// Nothing when libfiles does not read the format or the file cannot be opened. static std::optional read_file_packet(const df::file_path path) { const auto f = open_file(path, platform::file_open_mode::read); @@ -844,7 +844,7 @@ static std::optional read_file_packet(const df::file_p libfiles::reader r(stream); if (r.size() < 16 || !libfiles::can_update(libfiles_update_format(r))) return {}; - return libfiles::read_packet(r, decompress_jxl_box); + return libfiles::read_packet(r); } // What the item shows, written into the packet wherever the packet lacks it. @@ -955,13 +955,12 @@ static libfiles::legacy_edits legacy_edits_of(const metadata_edits& edits, const return legacy; } -// What an edit writes: the packet, the fields it sets wherever the format keeps them besides, and the -// Brotli a JPEG XL's compressed boxes are read with, so the update reads the file as its scan did. +// What an edit writes: the packet and the fields it sets wherever the format keeps them besides. +// libfiles reads compressed metadata with the same decoder as its scan. static libfiles::metadata_update update_of(const libfiles::xmp::meta& xmp, const metadata_edits& edits) { libfiles::metadata_update update{&xmp}; update.legacy = legacy_edits_of(edits, xmp); - update.brotli = decompress_jxl_box; return update; } diff --git a/src/model_index.cpp b/src/model_index.cpp index 2ba053ff..c55911ab 100644 --- a/src/model_index.cpp +++ b/src/model_index.cpp @@ -7,9 +7,11 @@ // This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY // Purpose: File indexing engine. Scans folders, extracts metadata, keeps the in-memory item -// collection and answers queries over it. Reads films, episodes and shows from file and folder names, -// and fills what a video's tags leave out from them as items load. Duplicate prediction and presence live -// in model_index_duplicates.cpp; the summary and its vocabulary in model_index_summary.cpp. +// collection and answers queries over it - from the search index (model_search_index.cpp) where it is +// current, scanning only folders changed since it was read - and reads that index on its own queue. +// Reads films, episodes and shows from file and folder names, and fills what a video's tags leave out +// from them as items load. Duplicate prediction and presence live in model_index_duplicates.cpp; the +// summary and its vocabulary in model_index_summary.cpp. #include "pch.h" @@ -73,6 +75,9 @@ std::function test_after_validate_fol struct query_items_result { + // Folders whose names match are listed beside the items. + static constexpr bool lists_folders = true; + index_state::query_item_results results; void match_item(const df::file_path id, const df::index_file_item& file, const df::search_result& match) @@ -84,10 +89,31 @@ struct query_items_result { results.push_back({index_state::query_item_kind::folder, {}, {}, folder_path, folder, {}}); } + + // A document the search index matched. One term answers with what it matched in, as the matcher does; + // several answer as their combination, which is all the matcher says of them either. + void match_doc(const df::search_index& index, const uint32_t doc, const df::search_matcher& matcher) + { + const auto& file = index.file(doc); + const auto& folder = index.folders()[index.doc_folder(doc)]; + const auto& terms = matcher.search().terms(); + const auto match = terms.size() == 1 + ? matcher.match_term(folder.path.text(), file, index.metadata(doc), terms.front()) + : df::search_result{df::search_result_type::match_multiple}; + + df::assert_true(match.is_match()); + + if (match.is_match()) + { + results.push_back({index_state::query_item_kind::file, folder.path.combine_file(file.name), file, {}, {}, match}); + } + } }; struct count_items_result { + static constexpr bool lists_folders = false; + df::file_group_histogram summary; void match_item(const df::file_path id, const df::index_file_item& file, const df::search_result& match) @@ -98,6 +124,34 @@ struct count_items_result static void match_folder(const df::folder_path folder_path, const df::index_folder_item_ptr& folder) { } + + void match_doc(const df::search_index& index, const uint32_t doc, const df::search_matcher&) + { + const auto& file = index.file(doc); + summary.record(file, index.path(doc), df::thumbnail_representative_rank(file, index.metadata(doc))); + } +}; + +// How many, and nothing more: no sizes, no representative picture. +struct tally_items_result +{ + static constexpr bool lists_folders = false; + + df::file_group_histogram summary; + + void match_item(const df::file_path, const df::index_file_item& file, const df::search_result&) + { + summary.record(file); + } + + static void match_folder(const df::folder_path, const df::index_folder_item_ptr&) + { + } + + void match_doc(const df::search_index& index, const uint32_t doc, const df::search_matcher&) + { + summary.record(index.file(doc)); + } }; // A related search written as text - typed, or restored from a favorite or saved search - @@ -195,7 +249,8 @@ static void iterate_items(const df::search_t& search_in, df::cancel_token token, index_items& index, bool refresh_from_file_system, - bool show_sidecars) + bool show_sidecars, + index_state::search_context* context = nullptr) { df::search_t resolved_related; const auto& search = resolve_related(search_in, state, resolved_related); @@ -363,43 +418,86 @@ static void iterate_items(const df::search_t& search_in, } else { - const auto folders = index.all_folders(); + index_folders_t listed; + const auto& folders = context ? context->folders : (listed = index.all_folders()); + auto* const evaluator = context && context->evaluator && !has_related ? &*context->evaluator : nullptr; - for (const auto& folder_node : folders) + // Asks a folder's items one by one, as every folder was asked before there was a search index. + // Folder masks are an early rejection only; each surviving item is checked against its own mask + // and then by the authoritative exact matcher. + const auto scan_folder = [&](const index_folders_t::value_type& folder_node) { - if (token.is_cancelled()) break; + if (!has_related && !matcher.can_match_folder && + !matcher.can_contain(folder_node.second->search_presence_summary)) + { + return; + } - if (folder_node.second->is_in_collection) + for (const auto& file_node : folder_node.second->files) { - // Folder masks are an early rejection only; each surviving item is checked - // against its own mask and then by the authoritative exact matcher. - if (has_related || - matcher.can_match_folder || - matcher.can_contain(folder_node.second->search_presence_summary)) + if (token.is_cancelled()) break; + + if (!(file_node.flags && df::index_item_flags::is_sidecar) || show_sidecars) { - for (const auto& file_node : folder_node.second->files) + const auto path = folder_node.first.combine_file(file_node.name); + const auto match = matcher.match_item(path, file_node); + + if (match.is_match()) { - if (token.is_cancelled()) break; + report(path, file_node, match); + } + } + } + }; - if (!(file_node.flags && df::index_item_flags::is_sidecar) || show_sidecars) - { - const auto path = folder_node.first.combine_file(file_node.name); - const auto match = matcher.match_item(path, file_node); + if (evaluator) + { + // The search index answers below for every folder it read and has not seen change; only the + // rest are asked item by item. + for (const auto position : evaluator->view().unclean_folders()) + { + if (token.is_cancelled()) break; - if (match.is_match()) - { - report(path, file_node, match); - } - } + const auto& folder_node = folders[position]; + if (!folder_node.second->is_in_collection) continue; + + df::bump(df::query_perf.index_scanned_folders); + scan_folder(folder_node); + } + } + + // A count asks for items alone, so only results that list folders read every folder's name. + if (!evaluator || T::lists_folders) + { + for (const auto& folder_node : folders) + { + if (token.is_cancelled()) break; + if (!folder_node.second->is_in_collection) continue; + + if (!evaluator) scan_folder(folder_node); + + if constexpr (T::lists_folders) + { + if (matcher.match_folder(folder_node.first.text(), folder_node.first.name()).is_match()) + { + results.match_folder(folder_node.first, folder_node.second); } } + } + } - const auto match = matcher.match_folder(folder_node.first.text(), folder_node.first.name()); + if (evaluator && matcher.has_terms && !token.is_cancelled()) + { + df::bump(df::query_perf.index_answers); + const auto docs = evaluator->evaluate(matcher, token); - if (match.is_match()) + if (!token.is_cancelled()) + { + const auto& snapshot = evaluator->index(); + docs.for_each([&results, &snapshot, &matcher](const uint32_t doc) { - results.match_folder(folder_node.first, folder_node.second); - } + results.match_doc(snapshot, doc, matcher); + }); } } } @@ -415,7 +513,8 @@ static void iterate_items(const df::search_t& search_in, void index_state::query_items(const df::search_t& search, const std::function& found_callback, - const df::cancel_token& token) + const df::cancel_token& token, + const search_execution execution) { df::scope_locked_inc l(searching); @@ -452,7 +551,9 @@ void index_state::query_items(const df::search_t& search, { df::bump(df::query_perf.queries); df::perf_timer timer(df::query_perf.query_us, &df::query_perf.query_max_us); - iterate_items(search, results, *this, token, _items, true, search.has_related()); + std::optional context; + if (!has_selector && !search.has_related()) context = make_search_context(execution); + iterate_items(search, results, *this, token, _items, true, search.has_related(), context ? &*context : nullptr); } df::bump(df::query_perf.query_items, results.results.size()); @@ -538,7 +639,8 @@ df::item_set index_state::materialize_query_items(query_item_results items, cons return results; } -df::file_group_histogram index_state::count_matches(const df::search_t& search, const df::cancel_token& token) +df::file_group_histogram index_state::count_matches(const df::search_t& search, const df::cancel_token& token, + const search_execution execution) { count_items_result result; @@ -547,12 +649,291 @@ df::file_group_histogram index_state::count_matches(const df::search_t& search, df::measure_ms ms(stats.count_matches_ms); df::bump(df::query_perf.counts); df::perf_timer timer(df::query_perf.count_us); - iterate_items(search, result, *this, token, _items, false, search.has_related()); + std::optional context; + if (!search.has_selector() && !search.has_related()) context = make_search_context(execution); + iterate_items(search, result, *this, token, _items, false, search.has_related(), context ? &*context : nullptr); } return result.summary; } +index_state::search_context index_state::make_search_context(const search_execution execution) const +{ + search_context result; + result.folders = _items.all_folders(); + + if (execution == search_execution::automatic) + { + if (auto snapshot = search_index()) + { + result.evaluator.emplace(std::move(snapshot), result.folders); + if (!result.evaluator->is_useful()) result.evaluator.reset(); + } + } + + return result; +} + +bool index_state::matches_collection_search(const df::search_matcher& matcher, const df::file_path path) const +{ + const auto folder = _items.find(path.folder()); + if (!folder || !folder->is_in_collection) return false; + if (!matcher.can_match_folder && !matcher.can_contain(folder->search_presence_summary)) return false; + + const auto found = std::lower_bound(folder->files.begin(), folder->files.end(), path.name()); + if (found == folder->files.end() || icmp(found->name, path.name()) != 0) return false; + if (found->flags && df::index_item_flags::is_sidecar) return false; + + return matcher.match_item(path, *found).is_match(); +} + +// Up to this many changed items, scanning them on top of the search index costs a query less than +// reading the collection again costs. +static uint64_t search_index_stale_tolerance(const uint64_t docs) +{ + return 1024 + docs / 32; +} + +// Far beyond what any real collection needs, so it only stops a pathological one from holding memory +// that scanning does without. +constexpr size_t max_search_index_bytes = size_t{1} << 30; +// The least a document costs, so a collection certain to pass the ceiling is not read to find out. +constexpr size_t min_search_index_bytes_per_doc = 48; +// A burst of index changes settles before the collection is read again. +constexpr uint32_t search_index_debounce_ms = 1000; +// A prediction's count scans this many changed items at most on top of the search index. +constexpr uint64_t max_predicted_stale_docs = 20000; +// A pass counting hundreds of searches - a sidebar's rows - scans this many changed items at most per +// search; past it the pass counts as it did without the index. +constexpr uint64_t max_counted_stale_docs = 4096; + +void index_state::publish_search_index(std::shared_ptr index) +{ + std::shared_ptr previous; + const auto publishing = index != nullptr; + + { + platform::exclusive_lock lock(_summary_rw); + previous = std::move(_summary._search_index); + _summary._search_index = std::move(index); + } + + // The sidebar may have counted without this snapshot - from summaries before any was read, or by + // scanning while the last was too stale to use - so it counts again from this one. Only counts: a + // snapshot changes no row. + if (publishing) _async.invalidate_view(view_invalid::sidebar_counts); + + // The snapshot replaced here, with the folder nodes and metadata it held, is released after the lock. +} + +void index_state::update_search_index() +{ + if (df::is_closing || !is_init_complete()) return; + + // Indexing rewrites folders faster than a snapshot could follow and needs the CPU a build would take. + // Searches scan meanwhile, as they always did, and the pass that ends indexing asks again. + const auto indexing_active = [this] + { + return df::is_closing || indexing.load() > 0 || scanning_items.load() > 0; + }; + + const auto folders = _items.all_folders(); + + if (const auto current = search_index()) + { + const df::search_index_view view(*current, folders); + // Date terms are answered from the snapshot only on the day it was read, so the first pass of a new + // day reads the collection again. + const auto formats_current = current->format_signature() == df::search_index::current_format_signature() && + current->date_day() == platform::now().to_days(); + + // A small collection's tolerance can pass the point where no search would use the snapshot, so one + // no search would use is read again however few items changed. + if (formats_current && view.is_useful() && + view.stale_doc_count(*current) <= search_index_stale_tolerance(view.collection_doc_count())) + { + return; + } + + if (indexing_active()) + { + // Past use to any search, while still holding the nodes and metadata indexing has replaced. + if (!view.is_useful()) publish_search_index(nullptr); + return; + } + } + else if (indexing_active()) + { + return; + } + + size_t docs = 0; + + for (const auto& folder : folders) + { + if (folder.second && folder.second->is_in_collection) docs += folder.second->files.size(); + } + + if (docs * min_search_index_bytes_per_doc > max_search_index_bytes) + { + publish_search_index(nullptr); + return; + } + + const auto start_us = df::now_us(); + auto built = df::search_index::build(folders, ++_search_index_generation, indexing_active); + if (!built) return; + + if (built->byte_size() > max_search_index_bytes) + { + publish_search_index(nullptr); + return; + } + + const auto elapsed_us = static_cast(df::now_us() - start_us); + stats.search_index_docs = static_cast(built->doc_count()); + stats.search_index_bytes = built->byte_size(); + stats.search_index_build_ms = static_cast(elapsed_us / 1000); + df::bump(df::index_perf.search_index_builds); + df::bump(df::index_perf.search_index_build_us, elapsed_us); + df::record_peak(df::index_perf.search_index_build_max_us, + static_cast(std::min(elapsed_us, UINT32_MAX))); + df::index_perf.search_index_docs.store(built->doc_count(), std::memory_order_relaxed); + df::index_perf.search_index_bytes.store(built->byte_size(), std::memory_order_relaxed); + + df::trace(std::format("Search index read {} items in {} ms, {}", built->doc_count(), elapsed_us / 1000, + df::file_size(static_cast(built->byte_size())).str())); + + publish_search_index(std::move(built)); +} + +void index_state::queue_update_search_index() +{ + _async.queue_async_after(async_queue::index_search_single, search_index_debounce_ms, [this] + { + if (df::is_closing) return; + update_search_index(); + }); +} + +index_state::search_context& search_counter::context() +{ + if (!_context) _context = _index.make_search_context(); + return *_context; +} + +bool search_counter::is_indexed() +{ + const auto& ctx = context(); + return ctx.evaluator && ctx.evaluator->view().stale_doc_count(ctx.evaluator->index()) <= max_counted_stale_docs; +} + +df::file_group_histogram search_counter::count(const df::search_t& search, const df::cancel_token& token) +{ + if (search.has_selector() || search.has_related()) return _index.count_matches(search, token); + + count_items_result result; + df::bump(df::query_perf.counts); + df::perf_timer timer(df::query_perf.count_us); + iterate_items(search, result, _index, token, _index._items, false, false, &context()); + return result.summary; +} + +std::optional search_counter::try_count_items(const df::search_t& search, const df::cancel_token& token) +{ + if (search.is_empty() || search.has_related()) return std::nullopt; + + if (search.has_selector()) + { + // One folder the index holds is read without asking the file system; a tree may be the whole + // collection, and a folder it does not hold would be enumerated. A term needing metadata would + // have each item of the folder not yet read opened and read here, keystroke after keystroke. + const auto& selectors = search.selectors(); + if (selectors.size() != 1 || selectors.front().is_recursive() || !_index.is_collection_search(search)) return std::nullopt; + if (search.needs_metadata() || !_index._items.find(selectors.front().folder())) return std::nullopt; + + tally_items_result result; + iterate_items(search, result, _index, token, _index._items, false, false, nullptr); + if (token.is_cancelled()) return std::nullopt; + return result.summary.total_items().count; + } + + auto& ctx = context(); + if (!ctx.evaluator) return std::nullopt; + + // Every changed folder is scanned on top of the index, so only a nearly current one answers here. + if (ctx.evaluator->view().stale_doc_count(ctx.evaluator->index()) > max_predicted_stale_docs) return std::nullopt; + + tally_items_result result; + iterate_items(search, result, _index, token, _index._items, false, false, &ctx); + if (token.is_cancelled()) return std::nullopt; + return result.summary.total_items().count; +} + +search_kept_items search_counter::make_kept_items(const std::vector& paths) +{ + search_kept_items result; + + auto& ctx = context(); + const auto* const evaluator = ctx.evaluator ? &*ctx.evaluator : nullptr; + if (evaluator) result.docs = df::doc_set(evaluator->index().doc_count()); + + // A view lists a folder's items together more often than not, so a folder is looked up once a run. + df::folder_path last_folder; + std::optional last_entry; + + for (const auto& path : paths) + { + if (evaluator) + { + const auto& snapshot = evaluator->index(); + + if (path.folder() != last_folder || last_folder.is_empty()) + { + last_folder = path.folder(); + last_entry = snapshot.find_folder(last_folder); + if (last_entry && !evaluator->view().is_clean_entry(*last_entry)) last_entry.reset(); + } + + if (last_entry) + { + if (const auto doc = snapshot.find_doc(*last_entry, path.name())) + { + result.docs.set(*doc); + continue; + } + } + } + + result.loose.emplace_back(path); + } + + return result; +} + +std::optional search_counter::count_kept(const df::search_t& search, const search_kept_items& items, + const df::cancel_token& token) +{ + auto& ctx = context(); + if (search.has_selector() || search.has_related() || !ctx.evaluator) return std::nullopt; + + const df::search_matcher matcher(search, platform::now().to_days(), &_index.locations()); + const auto docs = ctx.evaluator->evaluate(matcher, token); + + // Documents prepared against another snapshot number other items. + if (items.docs.size() != docs.size()) return std::nullopt; + + auto result = docs.count_and(items.docs); + + for (const auto& path : items.loose) + { + if (token.is_cancelled()) break; + if (_index.matches_collection_search(matcher, path)) ++result; + } + + return result; +} + ///////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////// @@ -2285,7 +2666,7 @@ df::index_file_item index_state::find_item(const df::file_path id) const } -inline bool index_state::is_collection_search(const df::search_t& search) const +bool index_state::is_collection_search(const df::search_t& search) const { for (const auto& sel : search.selectors()) { @@ -2351,7 +2732,7 @@ void index_state::save_crc(const df::file_path id, const index_file_revision rev file.calc_search_presence(); folder->update_search_presence(file); return true; - }); + }, index_items::file_change::derived); if (!published) return; item_db_write write; @@ -2391,7 +2772,7 @@ void index_state::save_phashes(std::vector hashes, const bool cont // orientations filled in. file.phash = picture_hashes; return true; - }); + }, index_items::file_change::derived); if (found) { @@ -4629,6 +5010,9 @@ void index_state::queue_update_predictions() // only: nothing about which rows exist has changed, so rebuilding them would discard every // sidebar text layout to publish a number. _async.invalidate_view(view_invalid::sidebar_counts); + // Hashes saved since the last pass changed folders the search index read; it decides on its own + // queue whether that is worth reading the collection again. + queue_update_search_index(); }); } @@ -4656,6 +5040,7 @@ void index_state::queue_update_summary() // update_summary already asked for the rebuild its new vocabulary needs; this only has to // re-earn the counts that vocabulary changed. _async.invalidate_view(view_invalid::sidebar_counts); + queue_update_search_index(); }); } diff --git a/src/model_index.h b/src/model_index.h index 86bd7d87..ff26d405 100644 --- a/src/model_index.h +++ b/src/model_index.h @@ -7,12 +7,14 @@ // This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY // Purpose: File indexing engine and duplicate detection. Scans folders, maintains item index, -// calculates summaries, and identifies duplicate files by name, date, and CRC. +// calculates summaries, and identifies duplicate files by name, date, and CRC. search_counter counts +// many searches - sidebar rows, address-box predictions - against one listing of the index. #pragma once #include "model_items.h" #include "model_postings.h" +#include "model_search_index.h" struct search_part; class location_cache; @@ -166,6 +168,11 @@ struct index_statistic int predictions_ms = 0; int count_matches_ms = 0; + // The search index as last published: what it holds and what reading it cost. + int search_index_docs = 0; + int search_index_build_ms = 0; + size_t search_index_bytes = 0; + int scan_items_ms = 0; int update_presence_ms = 0; @@ -210,6 +217,15 @@ class index_items }; public: + // What an in-place update changes. Either kind stops a folder rebuilt from an older copy replacing the + // node; only a searchable change stales the search index's read of the folder, which reads what is + // derived from an item's bytes - its duplicates, checksum and picture hashes - live. + enum class file_change + { + searchable, + derived + }; + index_folder_snapshot find_snapshot(const df::folder_path folder) const { platform::shared_lock lock(_rw); @@ -299,7 +315,7 @@ class index_items } template - bool update_file(const df::file_path path, Func func) const + bool update_file(const df::file_path path, Func func, const file_change change = file_change::searchable) const { platform::shared_lock lock(_rw); const auto found_folder = _index.find(path.folder()); @@ -312,7 +328,7 @@ class index_items if (found_file == folder->files.end()) return false; if (icmp(found_file->name, path.name()) != 0) return false; if (!func(folder, *found_file)) return false; - folder->mark_content_changed(); + folder->mark_content_changed(change == file_change::searchable); return true; } @@ -742,6 +758,8 @@ struct index_summary std::shared_ptr _distinct_prime_folders = std::make_shared(); index_metadata_summary_const_ptr _metadata = std::make_shared(); index_histograms_const_ptr _histograms = std::make_shared(); + // Null until the collection has been read once, and while indexing has left it too stale to help. + std::shared_ptr _search_index; }; enum class location_matrix_projection @@ -908,6 +926,8 @@ struct folder_scan_item class index_state final : public df::no_copy { + friend class search_counter; + async_strategy& _async; platform::mutex _summary_rw; @@ -916,6 +936,9 @@ class index_state final : public df::no_copy _Guarded_by_(_summary_rw) index_summary _summary; _Guarded_by_(_summary_rw) uint64_t _summary_generation = 0; std::atomic_uint64_t _predictions_generation = 0; + // Numbers each published search index. Written only by update_search_index, which runs on the + // search index queue alone. + uint64_t _search_index_generation = 0; // Written only on the UI thread by queue_scan_offline_thumbnails, read by the cloud worker so a // batch queued for a screenful that has since scrolled away can stop issuing network thumbnail // fetches. Atomic because the two contexts are unavoidably different: the worker cannot read the @@ -955,11 +978,47 @@ class index_state final : public df::no_copy void add_distinct_other_folders(df::unique_folders folders); void enqueue_db_write(item_db_write write); void enqueue_db_writes(std::vector writes); - bool is_collection_search(const df::search_t& search) const; bool is_excluded_from_collection(df::folder_path folder) const; void queue_deferred_validation(df::folder_path folder_path, bool rescan); + void publish_search_index(std::shared_ptr index); public: + bool is_collection_search(const df::search_t& search) const; + + // How a search is answered. A scan asks every item; automatic answers a selector-free search from the + // search index where it is current and scans only what changed since. Both find the same items. + enum class search_execution + { + automatic, + scan, + }; + + // One listing of the index, and the search index evaluator classified against it when the index + // answers. Built once and reused, so many searches share the listing and each other's terms. + struct search_context + { + index_folders_t folders; + std::optional evaluator; + }; + + search_context make_search_context(search_execution execution = search_execution::automatic) const; + + std::shared_ptr search_index() const + { + platform::shared_lock lock(_summary_rw); + return _summary._search_index; + } + + // Reads the collection into a new search index when the published one has gone stale, drops it when + // indexing has left it too stale to help, and does nothing while indexing is running. Runs on the + // search index queue, never on a queue that indexing, searching or the sidebar waits for. + void update_search_index(); + void queue_update_search_index(); + + // Whether one item, by path, is among the results of a selector-free collection search: what the + // scan of its folder would answer for it. + bool matches_collection_search(const df::search_matcher& matcher, df::file_path path) const; + enum class query_item_kind { file, @@ -1069,7 +1128,8 @@ class index_state final : public df::no_copy df::index_file_item find_item(df::file_path id) const; - df::file_group_histogram count_matches(const df::search_t& a, const df::cancel_token& token); + df::file_group_histogram count_matches(const df::search_t& a, const df::cancel_token& token, + search_execution execution = search_execution::automatic); df::file_group_histogram label_summary(const std::string_view label) const { @@ -1116,7 +1176,8 @@ class index_state final : public df::no_copy void query_items(const df::search_t& search, const std::function& found_callback, - const df::cancel_token& token); + const df::cancel_token& token, + search_execution execution = search_execution::automatic); df::item_set materialize_query_items(query_item_results items, const df::unique_items& existing) const; struct validate_folder_result @@ -1423,3 +1484,48 @@ class index_state final : public df::no_copy bool rebuild_sorted_words(index_metadata_summary& summary, uint64_t generation = 0) const; std::vector auto_complete_text(prop::key_ref key); }; + +// Items from a current result set, held the way the search index can intersect them: the documents of +// those it read and has not seen change, and the paths of the rest. +struct search_kept_items +{ + df::doc_set docs; + std::vector loose; +}; + +// Counts many searches against one listing of the index, so a sidebar's rows or one keystroke's +// predictions share the listing and the answers to the terms they have in common. Used by one worker at +// a time; reads the index as any search does and holds no lock between calls. +class search_counter final : public df::no_copy +{ + index_state& _index; + std::optional _context; + + index_state::search_context& context(); + +public: + explicit search_counter(index_state& index) : _index(index) + { + } + + // Whether selector-free searches are answered by the search index rather than by a scan. Every answer + // also asks each item the index has not read since it changed, so a pass counting many searches - a + // sidebar's rows - uses the index only while those are few, and otherwise counts as it did without one. + bool is_indexed(); + + // What count_matches answers. + df::file_group_histogram count(const df::search_t& search, const df::cancel_token& token); + + // The number of items a search finds, or nothing when only scanning many folders, or reading files, + // could say: a prediction's count must never cost what running the search costs. + std::optional try_count_items(const df::search_t& search, const df::cancel_token& token); + + // Prepares a current result set for count_kept. Folders are skipped: no search's count holds them. + search_kept_items make_kept_items(const std::vector& paths); + + // How many of `items` the search finds, when the search index can say. A search scoped to folders or + // related to an item has no answer here: only walking those folders, and reading what has not been + // read, could give one. + std::optional count_kept(const df::search_t& search, const search_kept_items& items, + const df::cancel_token& token); +}; diff --git a/src/model_index_duplicates.cpp b/src/model_index_duplicates.cpp index 5b4a1d0c..aeae876b 100644 --- a/src/model_index_duplicates.cpp +++ b/src/model_index_duplicates.cpp @@ -214,7 +214,7 @@ void index_state::update_predictions() if (current_file.exact_duplicate_group.load() == 0) return false; current_file.exact_duplicate_group = 0; return true; - }); + }, index_items::file_change::derived); } } } @@ -311,7 +311,7 @@ void index_state::update_predictions() if (file.exact_duplicate_group.load() == group) return false; file.exact_duplicate_group = group; return true; - }); + }, index_items::file_change::derived); } std::vector crowded(files.size(), 0); @@ -596,7 +596,7 @@ void index_state::update_predictions() } file.update_duplicates(folder, dup_info); return true; - }); + }, index_items::file_change::derived); } stats.indexed_dup_folder_count = static_cast(component_groups.size()); diff --git a/src/model_index_summary.cpp b/src/model_index_summary.cpp index 7b40c611..b8c95418 100644 --- a/src/model_index_summary.cpp +++ b/src/model_index_summary.cpp @@ -723,14 +723,33 @@ std::vector index_state::auto_complete_words( // Prefix matches first: a binary search over the sorted snapshot (O(log N + k)) rather // than a full scan of the vocabulary. Prefix hits are the strongest predictions and are - // returned ahead of interior substring matches. + // returned ahead of interior substring matches. The range is in alphabetical order, so the + // words the collection uses most are taken from all of it rather than the first few. const auto [prefix_first, prefix_last] = word_prefix_range(summary->_sorted_words, query); - for (auto it = prefix_first; it != prefix_last && result.size() < max_results; ++it) + std::vector> prefix_words; + prefix_words.reserve(static_cast(std::distance(prefix_first, prefix_last))); + + for (auto it = prefix_first; it != prefix_last; ++it) { const auto count = summary->_distinct_words.find(*it); - result.emplace_back(std::string(*it), std::vector{{0, str::matched_text_byte_length(*it, 0, query)}}, - count == summary->_distinct_words.end() ? 0 : count->second.i); + prefix_words.emplace_back(*it, count == summary->_distinct_words.end() ? 0 : count->second.i); + } + + const auto prefix_taken = std::min(prefix_words.size(), max_results); + std::ranges::partial_sort(prefix_words, prefix_words.begin() + static_cast(prefix_taken), + [](const auto& left, const auto& right) + { + return left.second != right.second + ? left.second > right.second + : str::icmp(left.first, right.first) < 0; + }); + + for (size_t i = 0; i < prefix_taken; ++i) + { + const auto word = prefix_words[i].first; + result.emplace_back(std::string(word), std::vector{{0, str::matched_text_byte_length(word, 0, query)}}, + prefix_words[i].second); } // For longer queries, top up with interior substring matches the prefix pass cannot find, diff --git a/src/model_items.cpp b/src/model_items.cpp index 93b5f199..d1450b5f 100644 --- a/src/model_items.cpp +++ b/src/model_items.cpp @@ -1626,6 +1626,13 @@ void view_state::update_item_groups() invalidate_view(view_invalid::app_layout); } + // Each sidebar row's tooltip says how many of the items shown it keeps, so a different set is + // counted again. Regrouping or resorting the same items rebuilds this without changing it. + if (_display_items != new_display_items) + { + invalidate_view(view_invalid::sidebar_counts); + } + _display_items = std::move(new_display_items); _item_groups = std::move(new_item_groups); diff --git a/src/model_items.h b/src/model_items.h index 5094ce24..df912fba 100644 --- a/src/model_items.h +++ b/src/model_items.h @@ -658,6 +658,10 @@ namespace df // extra stale bits are safe because item masks and exact matching follow. std::atomic search_presence_summary; std::atomic_uint64_t content_revision = 0; + // Raised with content_revision by every in-place change except to what is derived from an item's + // bytes - its duplicates, checksum and picture hashes - which the search index reads live, so a + // snapshot of the folder stays current through them. + std::atomic_uint64_t search_revision = 0; // is_in_collection / is_excluded are flipped by the indexing thread (index_folders) // while UI and database threads read them through shared node pointers. They are @@ -691,8 +695,14 @@ namespace df return content_revision.load(std::memory_order_acquire); } - void mark_content_changed() + uint64_t search_revision_snapshot() const { + return search_revision.load(std::memory_order_acquire); + } + + void mark_content_changed(const bool searchable = true) + { + if (searchable) search_revision.fetch_add(1, std::memory_order_release); content_revision.fetch_add(1, std::memory_order_release); } @@ -781,15 +791,22 @@ namespace df int created = 0; }; - inline uint8_t thumbnail_representative_rank(const index_file_item& file) + // The metadata already read for this file, null when it has none, so a caller holding it does not read + // the file's published pointer again. + inline uint8_t thumbnail_representative_rank(const index_file_item& file, const prop::item_metadata* metadata) { - const auto metadata = file.metadata.load(); const auto visual_media = file.ft && (file.ft->has_trait(file_traits::bitmap) || file.ft->has_trait(file_traits::video_metadata)); if (!visual_media || !file.ft->has_trait(file_traits::thumbnail)) return 0; return metadata && metadata->rating >= 4 ? 2 : 1; } + inline uint8_t thumbnail_representative_rank(const index_file_item& file) + { + const auto metadata = file.metadata.load(); + return thumbnail_representative_rank(file, metadata.get()); + } + inline bool should_replace_thumbnail_representative(const file_path& candidate, const uint8_t candidate_rank, const file_path& current, const uint8_t current_rank) { @@ -912,9 +929,13 @@ namespace df } void record(const index_file_item& info, const file_path& path) + { + record(info, path, thumbnail_representative_rank(info)); + } + + void record(const index_file_item& info, const file_path& path, const uint8_t rank) { record(info); - const auto rank = thumbnail_representative_rank(info); if (should_replace_thumbnail_representative(path, rank, representative_path, _representative_rank)) { representative_path = path; diff --git a/src/model_postings.h b/src/model_postings.h index 1dbfad48..b1062676 100644 --- a/src/model_postings.h +++ b/src/model_postings.h @@ -6,10 +6,11 @@ // License details are available at https://www.gnu.org/licenses/lgpl-2.1.html // This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY -// Purpose: Compressed posting lists for the search inverted index. Encodes a strictly -// increasing list of document (item) ids as delta + variable-byte (VByte) bytes. The -// trigram index built on top of it accelerates substring prediction in the address box; -// the boolean set operations and the inverted index above them are implemented and tested +// Purpose: Compressed posting lists and the trigram index. Encodes a strictly increasing list of +// document (item) ids as delta + variable-byte (VByte) bytes; the trigram index built on top of it +// narrows a substring to the few values that could contain it, for prediction in the address box and +// for the search index's free-text terms (model_search_index.h); the boolean set operations and the +// term inverted index above them are implemented and tested #pragma once @@ -129,17 +130,12 @@ namespace df } }; - // Boolean composition over sorted id lists. These map directly onto Diffractor's query - // semantics: AND of terms (intersect), OR / "with:" (union), and negation / "without:" - // (difference). Inputs must be sorted ascending; outputs are sorted ascending. + // Boolean composition over sorted id lists: AND of terms (intersect), OR / "with:" (union), and + // negation / "without:" (difference). Inputs must be sorted ascending; outputs are sorted ascending. // - // NOTE on what is live vs. foundation: `postings_intersect` is used in production by the - // trigram substring-prediction path. `postings_union` and `postings_difference` are NOT yet - // wired into production - they are the retained, unit-tested foundation for the deferred - // term->items search-matching reverse index. Live search *matching* still uses the existing - // folder/item presence-mask prefilter, so composing OR/AND-NOT over item postings is not - // needed until (and if) that matcher is profiled against real libraries and migrated. They - // are kept (rather than removed and re-added) so the composition stays complete and tested. + // `postings_intersect` is used by the trigram index's substring candidates. Live search combines + // whole sets of items as bitsets instead (df::doc_set, model_search_index.h), which suits sets that + // cover much of the collection; these are kept, tested, for the term inverted index below. inline std::vector postings_intersect(const std::vector& a, const std::vector& b) { std::vector result; @@ -168,16 +164,10 @@ namespace df // contain that term. Documents must be added in ascending id order so each term's postings // stay sorted. Term lookup is case-insensitive (matching search semantics). // - // STATUS: foundation, not yet wired into production. This is the *exact*-term reverse index - // intended for a future term->items search-matching path. It is deliberately retained and - // unit-tested (see should_query_inverted_index) but is NOT used by live search, which keeps - // its folder/item presence-mask prefilter. Two reasons it is not wired yet: - // 1. The live text matcher is substring + wildcard across many fields (see compare_text), - // which an exact-token index cannot reproduce - substring acceleration is instead done - // by trigram_index (below), which IS live for typeahead prediction. - // 2. Flipping item matching onto an inverted index is a core-search rewrite that overlaps - // the existing presence-mask prefilter and should be justified by profiling on a real library. - // It is kept as a validated building block so that migration is a small, guarded step. + // Not what live search uses: an exact-token index cannot answer the free-text matcher, which looks + // for a term inside any of many fields (see compare_text). The search index answers that by asking + // each distinct field value once and mapping the values that match to their items. This class is + // kept, tested (see should_query_inverted_index), as the building block for whole-word terms. class inverted_index { df::hash_map _terms; @@ -360,6 +350,13 @@ namespace df _doc_count = std::max(_doc_count, id == UINT32_MAX ? id : id + 1); } + // What the index holds, for sizing and diagnostics. + size_t byte_size() const + { + return _pending.capacity() * sizeof(pending_t) + _grams.capacity() * sizeof(gram_entry) + + _postings.capacity(); + } + // Collapse everything added so far into the queryable form. Call once, after the last add(). void freeze() { diff --git a/src/model_search.cpp b/src/model_search.cpp index 7909a689..5837d17f 100644 --- a/src/model_search.cpp +++ b/src/model_search.cpp @@ -310,6 +310,18 @@ df::search_t df::search_t::parse(const std::string_view text) return result; } +df::search_t df::search_t::parse_without_probing(const std::string_view text) +{ + if (unc_root_of(str::trim(text)) == unc_root::nothing) + { + search_t result; + result.raw_text(text); + return result; + } + + return parse_parts(text, false); +} + bool df::search_t::can_be_read_from(const std::string_view text) const { const auto trimmed = str::trim(text); @@ -1948,107 +1960,206 @@ inline bool same_term(const std::string_view text, const df::search_term& term) return str::same(t, q); } -df::search_result compare_text(const df::search_term& term, const df::index_file_item& file) +static uint64_t pack_dimensions(const prop::item_metadata& md) +{ + return static_cast(md.width) << 16 | md.height; +} + +// The metadata text fields a free-text term searches inside, in the order compare_text reports them. +static const df::text_contains_field text_contains_table[] = { + {&prop::item_metadata::album, &prop::album}, + {&prop::item_metadata::audio_codec, &prop::audio_codec}, + {&prop::item_metadata::bitrate, &prop::bitrate}, + {&prop::item_metadata::camera_manufacturer, &prop::camera_manufacturer}, + {&prop::item_metadata::camera_model, &prop::camera_model}, + {&prop::item_metadata::comment, &prop::comment}, + {&prop::item_metadata::composer, &prop::composer}, + {&prop::item_metadata::copyright_creator, &prop::copyright_creator}, + {&prop::item_metadata::copyright_credit, &prop::copyright_credit}, + {&prop::item_metadata::copyright_notice, &prop::copyright_notice}, + {&prop::item_metadata::copyright_source, &prop::copyright_source}, + {&prop::item_metadata::copyright_url, &prop::copyright_url}, + {&prop::item_metadata::description, &prop::description}, + {&prop::item_metadata::encoder, &prop::encoder}, + {&prop::item_metadata::file_name, &prop::file_name}, + {&prop::item_metadata::genre, &prop::genre}, + {&prop::item_metadata::lens, &prop::lens}, + {&prop::item_metadata::location_place, &prop::location_place}, + {&prop::item_metadata::location_country, &prop::location_country}, + {&prop::item_metadata::location_state, &prop::location_state}, + {&prop::item_metadata::performer, &prop::performer}, + {&prop::item_metadata::pixel_format, &prop::pixel_format}, + {&prop::item_metadata::publisher, &prop::publisher}, + {&prop::item_metadata::show, &prop::show}, + {&prop::item_metadata::synopsis, &prop::synopsis}, + {&prop::item_metadata::title, &prop::title}, + {&prop::item_metadata::label, &prop::label}, + {&prop::item_metadata::video_codec, &prop::video_codec}, + {&prop::item_metadata::raw_file_name, &prop::raw_file_name}, +}; + +// The numbers a free-text term matches whole, as they are displayed, in the order compare_text reports them. +static const df::text_format_field text_format_table[] = { + { + &prop::dimensions, + [](const prop::item_metadata& md, file_type_ref) { return prop::format_dimensions(md.dimensions()); }, + [](const prop::item_metadata& md, file_type_ref) { return pack_dimensions(md); } + }, + { + &prop::dimensions, + [](const prop::item_metadata& md, const file_type_ref ft) { return prop::format_pixels(md.dimensions(), ft); }, + [](const prop::item_metadata& md, const file_type_ref ft) + { + return pack_dimensions(md) | + static_cast(ft->has_trait(file_traits::av)) << 32 | + static_cast(ft->has_trait(file_traits::bitmap)) << 33; + } + }, + { + &prop::exposure_time, + [](const prop::item_metadata& md, file_type_ref) { return prop::format_exposure(md.exposure_time); }, + [](const prop::item_metadata& md, file_type_ref) { return uint64_t{std::bit_cast(md.exposure_time)}; } + }, + { + &prop::f_number, + [](const prop::item_metadata& md, file_type_ref) { return prop::format_f_num(md.f_number); }, + [](const prop::item_metadata& md, file_type_ref) { return uint64_t{std::bit_cast(md.f_number)}; } + }, + { + &prop::focal_length, + [](const prop::item_metadata& md, file_type_ref) + { + return prop::format_focal_length(md.focal_length, md.focal_length_35mm_equivalent); + }, + [](const prop::item_metadata& md, file_type_ref) + { + return uint64_t{std::bit_cast(md.focal_length)} | + static_cast(md.focal_length_35mm_equivalent) << 32; + } + }, + { + &prop::duration, + [](const prop::item_metadata& md, file_type_ref) { return prop::format_duration(md.duration); }, + [](const prop::item_metadata& md, file_type_ref) { return uint64_t{md.duration}; } + }, + { + &prop::iso_speed, + [](const prop::item_metadata& md, file_type_ref) { return prop::format_iso(md.iso_speed); }, + [](const prop::item_metadata& md, file_type_ref) { return uint64_t{md.iso_speed}; } + }, + { + &prop::audio_channels, + [](const prop::item_metadata& md, file_type_ref) { return prop::format_audio_channels(md.audio_channels); }, + [](const prop::item_metadata& md, file_type_ref) { return uint64_t{md.audio_channels}; } + }, + { + &prop::audio_sample_rate, + [](const prop::item_metadata& md, file_type_ref) { return prop::format_audio_sample_rate(md.audio_sample_rate); }, + [](const prop::item_metadata& md, file_type_ref) { return uint64_t{md.audio_sample_rate}; } + }, + { + &prop::audio_sample_type, + [](const prop::item_metadata& md, file_type_ref) + { + return prop::format_audio_sample_type(static_cast(md.audio_sample_type)); + }, + [](const prop::item_metadata& md, file_type_ref) { return uint64_t{md.audio_sample_type}; } + }, + { + &prop::year, + [](const prop::item_metadata& md, file_type_ref) { return str::to_string(md.year); }, + [](const prop::item_metadata& md, file_type_ref) { return uint64_t{md.year}; } + }, +}; + +static bool split_on_separator(const char c) +{ + return str::is_separator(c); +} + +static bool split_on_artist_separator(const char c) +{ + return str::is_artist_separator(c); +} + +// The lists a free-text term matches one entry of exactly, in the order compare_text reports them. +static const df::text_split_field text_split_table[] = { + {&prop::item_metadata::tags, &prop::tag, split_on_separator}, + {&prop::item_metadata::artist, &prop::artist, split_on_artist_separator}, + {&prop::item_metadata::album_artist, &prop::album_artist, split_on_artist_separator}, +}; + +std::span df::text_contains_fields() +{ + return text_contains_table; +} + +std::span df::text_format_fields() +{ + return text_format_table; +} + +std::span df::text_split_fields() +{ + return text_split_table; +} + +bool df::text_term_contains(const search_term& term, const std::string_view value) +{ + return contains_term(value, term); +} + +bool df::text_term_same(const search_term& term, const std::string_view value) +{ + return same_term(value, term); +} + +bool df::text_term_equals_part(const search_term& term, const std::string_view part) +{ + return compare_term(term, part).match; +} + +std::string df::search_fold_key(const std::string_view text) +{ + std::string buf; + const auto normalized = nfc_view(text, buf); + std::string result; + result.reserve(normalized.size() * 4); + + auto p = normalized.begin(); + const auto end = normalized.end(); + + while (p < end) + { + const auto c = static_cast(str::to_lower(str::pop_utf8_char(p, end))); + result.push_back(static_cast(c & 0xff)); + result.push_back(static_cast(c >> 8 & 0xff)); + result.push_back(static_cast(c >> 16 & 0xff)); + result.push_back(static_cast(c >> 24 & 0xff)); + } + + return result; +} + +df::search_result compare_text(const df::search_term& term, const df::index_file_item& file, + const prop::item_metadata* const md) { if (contains_term(file.name, term)) return {df::search_result_type::match_prop, prop::file_name}; if (same_term(df::format_size(file.size), term)) return {df::search_result_type::match_prop, prop::file_size}; - const auto md = file.metadata.load(); - if (md) { - if (contains_term(md->album, term)) return {df::search_result_type::match_prop, prop::album}; - if (contains_term(md->audio_codec, term)) return {df::search_result_type::match_prop, prop::audio_codec}; - if (contains_term(md->bitrate, term)) return {df::search_result_type::match_prop, prop::bitrate}; - if (contains_term(md->camera_manufacturer, term)) - return { - df::search_result_type::match_prop, prop::camera_manufacturer - }; - if (contains_term(md->camera_model, term)) return {df::search_result_type::match_prop, prop::camera_model}; - if (contains_term(md->comment, term)) return {df::search_result_type::match_prop, prop::comment}; - if (contains_term(md->composer, term)) return {df::search_result_type::match_prop, prop::composer}; - if (contains_term(md->copyright_creator, term)) - return { - df::search_result_type::match_prop, prop::copyright_creator - }; - if (contains_term(md->copyright_credit, term)) - return { - df::search_result_type::match_prop, prop::copyright_credit - }; - if (contains_term(md->copyright_notice, term)) - return { - df::search_result_type::match_prop, prop::copyright_notice - }; - if (contains_term(md->copyright_source, term)) - return { - df::search_result_type::match_prop, prop::copyright_source - }; - if (contains_term(md->copyright_url, term)) return {df::search_result_type::match_prop, prop::copyright_url}; - if (contains_term(md->description, term)) return {df::search_result_type::match_prop, prop::description}; - if (contains_term(md->encoder, term)) return {df::search_result_type::match_prop, prop::encoder}; - if (contains_term(md->file_name, term)) return {df::search_result_type::match_prop, prop::file_name}; - if (contains_term(md->genre, term)) return {df::search_result_type::match_prop, prop::genre}; - if (contains_term(md->lens, term)) return {df::search_result_type::match_prop, prop::lens}; - if (contains_term(md->location_place, term)) return {df::search_result_type::match_prop, prop::location_place}; - if (contains_term(md->location_country, term)) - return { - df::search_result_type::match_prop, prop::location_country - }; - if (contains_term(md->location_state, term)) return {df::search_result_type::match_prop, prop::location_state}; - if (contains_term(md->performer, term)) return {df::search_result_type::match_prop, prop::performer}; - if (contains_term(md->pixel_format, term)) return {df::search_result_type::match_prop, prop::pixel_format}; - if (contains_term(md->publisher, term)) return {df::search_result_type::match_prop, prop::publisher}; - if (contains_term(md->show, term)) return {df::search_result_type::match_prop, prop::show}; - if (contains_term(md->synopsis, term)) return {df::search_result_type::match_prop, prop::synopsis}; - if (contains_term(md->title, term)) return {df::search_result_type::match_prop, prop::title}; - if (contains_term(md->label, term)) return {df::search_result_type::match_prop, prop::label}; - if (contains_term(md->video_codec, term)) return {df::search_result_type::match_prop, prop::video_codec}; - if (contains_term(md->raw_file_name, term)) return {df::search_result_type::match_prop, prop::raw_file_name}; - - if (same_term(prop::format_dimensions(md->dimensions()), term)) - return { - df::search_result_type::match_prop, prop::dimensions - }; - if (same_term(prop::format_pixels(md->dimensions(), file.ft), term)) - return { - df::search_result_type::match_prop, prop::dimensions - }; - if (same_term(prop::format_exposure(md->exposure_time), term)) - return { - df::search_result_type::match_prop, prop::exposure_time - }; - if (same_term(prop::format_f_num(md->f_number), term)) - return { - df::search_result_type::match_prop, prop::f_number - }; - if (same_term(prop::format_focal_length(md->focal_length, md->focal_length_35mm_equivalent), term)) - return { - df::search_result_type::match_prop, prop::focal_length - }; + for (const auto& field : text_contains_table) + { + if (contains_term(md->*field.member, term)) return {df::search_result_type::match_prop, field.key}; + } - if (same_term(prop::format_duration(md->duration), term)) - return { - df::search_result_type::match_prop, prop::duration - }; - if (same_term(prop::format_iso(md->iso_speed), term)) - return { - df::search_result_type::match_prop, prop::iso_speed - }; - if (same_term(prop::format_audio_channels(md->audio_channels), term)) - return { - df::search_result_type::match_prop, prop::audio_channels - }; - if (same_term(prop::format_audio_sample_rate(md->audio_sample_rate), term)) - return { - df::search_result_type::match_prop, prop::audio_sample_rate - }; - if (same_term(format_audio_sample_type(static_cast(md->audio_sample_type)), term)) - return - {df::search_result_type::match_prop, prop::audio_sample_type}; - if (same_term(str::to_string(md->year), term)) return {df::search_result_type::match_prop, prop::year}; + for (const auto& field : text_format_table) + { + if (same_term(field.format(*md, file.ft), term)) return {df::search_result_type::match_prop, field.key}; + } compare_result comp_result; - prop::key_ref key = prop::null; auto cmp = [&comp_result, &term](const std::string_view part) { @@ -2058,49 +2169,29 @@ df::search_result compare_text(const df::search_term& term, const df::index_file } }; - if (!is_empty(md->tags)) split2(md->tags, true, cmp); - - if (comp_result.match) - { - key = prop::tag; - } - else + for (const auto& field : text_split_table) { - if (!is_empty(md->artist)) split2(md->artist, true, cmp, str::is_artist_separator); + const auto value = md->*field.member; + if (!is_empty(value)) split2(value, true, cmp, field.separator); if (comp_result.match) { - key = prop::artist; + return {df::search_result_type::match_prop, field.key, comp_result.val_matched}; } - else - { - if (!is_empty(md->album_artist)) split2(md->album_artist, true, cmp, str::is_artist_separator); - - if (comp_result.match) - { - key = prop::album_artist; - } - } - } - - if (comp_result.match) - { - return {df::search_result_type::match_prop, key, comp_result.val_matched}; } } return {df::search_result_type::no_match}; } -static compare_result compare_val(const df::search_term& term, const df::index_file_item& file) +static compare_result compare_val(const df::search_term& term, const df::index_file_item& file, + const prop::item_metadata* const md) { auto&& t = term.key; if (t == prop::modified && !prop::is_null(file.file_modified)) return compare_term(term, file.file_modified); if (t == prop::file_size && !file.size.is_empty()) return compare_file_size(term, file.size.to_int64()); - const auto md = file.metadata.load(); - if (md) { if (term.key == prop::tag || term.key == prop::artist || term.key == prop::album_artist || @@ -2215,13 +2306,11 @@ static compare_result compare_val(const df::search_term& term, const df::index_f return {}; } -bool has_type(const prop::key_ref t, const df::index_file_item& file) +static bool has_type(const prop::key_ref t, const df::index_file_item& file, const prop::item_metadata* const md) { if (t == prop::modified) return !prop::is_null(file.file_modified); if (t == prop::file_size) return !file.size.is_empty(); - const auto md = file.metadata.load(); - if (md) { if (t == prop::title) return !prop::is_null(md->title); @@ -2355,59 +2444,56 @@ static bool is_date_match(const df::date_parts& term, const df::date_t d, const return false; } -static bool is_date_match(const df::search_term& term, const df::index_file_item& file, const uint32_t now_days) +// A header is only clickable if the matcher answers the key it was drawn from, so each key is reproduced +// here whole - mirroring one rung of one of them is #184 again on whichever order the missing rung belongs +// to. Each stamp is converted the way the group key converts it; a UTC stamp against a local key misses +// by a day. +df::date_term_keys df::date_term_keys_of(const date_parts_prop target, const index_file_item& file, + const prop::item_metadata* const md) { - const bool is_any = term.date_val.target == df::date_parts_prop::any; - const auto target = term.date_val.target; + date_term_keys result; + const bool is_any = target == date_parts_prop::any; - if (target == df::date_parts_prop::original || target == df::date_parts_prop::created || is_any) + if (target == date_parts_prop::original || target == date_parts_prop::created || is_any) { - const auto md = file.metadata.load(); - - // A header is only clickable if the matcher answers the key it was drawn from, so each key - // is reproduced here whole - mirroring one rung of one of them is #184 again on whichever - // order the missing rung belongs to. The stamp is converted the way the group key converts - // it; a UTC stamp against a local key misses by a day. const auto file_stamp = file.file_created.system_to_local(); - auto original_key = df::date_t::null; - if (target == df::date_parts_prop::original || is_any) + if (target == date_parts_prop::original || is_any) { - const auto resolved = md ? md->created() : df::date_t::null; - original_key = resolved.is_valid() ? resolved : file_stamp; - - if (original_key.is_valid() && - is_date_match(term.date_val, original_key, term.modifiers, now_days)) - { - return true; - } + const auto resolved = md ? md->created() : date_t::null; + result.original = resolved.is_valid() ? resolved : file_stamp; } - if (target == df::date_parts_prop::created || is_any) + if (target == date_parts_prop::created || is_any) { - const auto created = md ? md->dates.created() : df::date_t::null; - const auto created_key = created.is_valid() ? created : file_stamp; - - // On the `any` path the two keys usually agree, and the first has already answered. - if (created_key.is_valid() && created_key != original_key && - is_date_match(term.date_val, created_key, term.modifiers, now_days)) - { - return true; - } + const auto created = md ? md->dates.created() : date_t::null; + result.created = created.is_valid() ? created : file_stamp; } } - if (target == df::date_parts_prop::modified || is_any) + if (target == date_parts_prop::modified || is_any) { - // Same rule as the created axis: Group by Modified keys on the local stamp, so the term has - // to compare against the local stamp or the header it drew answers with nothing. - if (is_date_match(term.date_val, file.file_modified.load().system_to_local(), term.modifiers, now_days)) - { - return true; - } + result.modified = file.file_modified.load().system_to_local(); } - return false; + return result; +} + +static bool is_date_match(const df::search_term& term, const df::index_file_item& file, + const prop::item_metadata* const md, const uint32_t now_days) +{ + const auto target = term.date_val.target; + const auto keys = df::date_term_keys_of(target, file, md); + + return df::date_term_found(target, keys.original, keys.created, keys.modified, [&term, now_days](const df::date_t key) + { + return is_date_match(term.date_val, key, term.modifiers, now_days); + }); +} + +bool df::search_matcher::date_key_matches(const search_term& term, const date_t key) const +{ + return is_date_match(term.date_val, key, term.modifiers, _now_days); } @@ -2528,6 +2614,20 @@ bool df::match_volume_label(const std::string_view folder_name, const hash_map 0 && md && md->has_gps() && map_location_area{ .cell = term.location_cell, .cell_span = term.location_cell_span }.contains(location_heat_map::calc_map_loc(md->coordinate)); @@ -2577,7 +2676,6 @@ df::search_result df::search_matcher::match_term(const str::cached folder_name, { // locations.md 3.6: no location can be determined at all, which is distinct // from `without:place` asking whether the stored field is populated. - const auto md = file.metadata.load(); const bool has_location = md && (md->coordinate.is_valid() || !str::is_empty(md->location_place) || !str::is_empty(md->location_state) || @@ -2590,7 +2688,6 @@ df::search_result df::search_matcher::match_term(const str::cached folder_name, } else if (term.type == search_term_type::location) { - const auto md = file.metadata.load(); auto match_location = false; if (term.coord_val.is_valid()) { @@ -2681,7 +2778,6 @@ df::search_result df::search_matcher::match_term(const str::cached folder_name, { // locations.md 2.5 step 5: nothing significant enough is close enough to name. A stored // place name is the user's own answer, so an item that carries one is never remote. - const auto md = file.metadata.load(); const bool is_remote = _locations && md && md->coordinate.is_valid() && str::is_empty(md->location_place) && str::is_empty(md->location_state) && @@ -2696,7 +2792,6 @@ df::search_result df::search_matcher::match_term(const str::cached folder_name, { // The file's own declaration. An image merely shaped like a panorama does not match, so a // wide crop of a landscape is not swept in with the photo spheres. - const auto md = file.metadata.load(); const bool is_panorama = md && md->is_panorama(); if (is_panorama == term.modifiers.positive) @@ -2707,7 +2802,6 @@ df::search_result df::search_matcher::match_term(const str::cached folder_name, else if (term.type == search_term_type::film || term.type == search_term_type::episode) { // What the item's name, tags and folder's library kind make it; never a guess from its length. - const auto md = file.metadata.load(); const auto wanted = term.type == search_term_type::film ? prop::media_kind::film : prop::media_kind::episode; const bool is_kind = md && md->kind == wanted; @@ -2718,7 +2812,6 @@ df::search_result df::search_matcher::match_term(const str::cached folder_name, } else if (term.type == search_term_type::in_progress) { - const auto md = file.metadata.load(); const bool started = md && md->is_in_progress(); if (started == term.modifiers.positive) @@ -2728,9 +2821,7 @@ df::search_result df::search_matcher::match_term(const str::cached folder_name, } else if (term.type == search_term_type::text) { - const auto match_folder = can_match_folder && - str::contains(term.text, "**") && - str::wildcard_icmp(folder_name.sv(), term.text); + const auto match_folder = text_term_matches_folder(term, folder_name); // special case - exclusions can match folder name if (!term.modifiers.positive) @@ -2745,7 +2836,7 @@ df::search_result df::search_matcher::match_term(const str::cached folder_name, } else { - const auto match_text = compare_text(term, file); + const auto match_text = compare_text(term, file, md); if (match_text.is_match() && term.modifiers.positive) { @@ -2761,7 +2852,7 @@ df::search_result df::search_matcher::match_term(const str::cached folder_name, } else if (term.is_date()) { - const bool match_date = is_date_match(term, file, _now_days); + const bool match_date = is_date_match(term, file, md, _now_days); if (match_date == term.modifiers.positive) { @@ -2772,7 +2863,7 @@ df::search_result df::search_matcher::match_term(const str::cached folder_name, { if (file.ft != file_type::folder) { - const bool match_has_type = has_type(term.key, file); + const bool match_has_type = has_type(term.key, file, md); if (match_has_type == term.modifiers.positive) { @@ -2782,7 +2873,7 @@ df::search_result df::search_matcher::match_term(const str::cached folder_name, } else if (term.type == search_term_type::value) { - const auto cmp = compare_val(term, file); + const auto cmp = compare_val(term, file, md); const bool match_value = modifier_match(cmp, term.modifiers); if (match_value == term.modifiers.positive) @@ -2951,91 +3042,28 @@ df::search_result df::search_matcher::match_item(const file_path path, const ind } df::search_result df::search_matcher::match_all_terms(const str::cached folder_name, const index_file_item& file) const +{ + const auto md = file.metadata.load(); + return match_all_terms(folder_name, file, md.get()); +} + +df::search_result df::search_matcher::match_all_terms(const str::cached folder_name, const index_file_item& file, + const prop::item_metadata* const md) const { if (_search._terms.size() == 1) { // optimisation for single term - return match_term(folder_name, file, _search._terms[0]); + return match_term(folder_name, file, md, _search._terms[0]); } - struct level - { - bool logical_and = true; - bool state = true; - // A level starts with no value rather than with an operator's identity. `true` is the - // identity for AND but the annihilator for OR, so an OR level seeded with it stays true - // whatever it later receives - which is what made a nested group after OR match everything. - bool has_value = false; - }; - - constexpr auto max_levels = 32; - level level_results[max_levels]; - auto current_level = 0; - - const auto contribute = [](level& target, const bool value, const bool logical_and) - { - if (!target.has_value) - { - target.state = value; - target.has_value = true; - } - else if (logical_and) - { - target.state &= value; - } - else - { - target.state |= value; - } - }; - - const auto fold_level = [&contribute](level* levels, int& depth) - { - if (depth > 0) - { - contribute(levels[depth - 1], levels[depth].state, levels[depth].logical_and); - depth -= 1; - } - }; - - for (const auto& term : _search._terms) - { - const auto is_match = match_term(folder_name, file, term).is_match(); - - auto opened = 0; - - for (auto i = 0; i < term.modifiers.begin_group; i++) - { - if (current_level < max_levels - 1) - { - current_level += 1; - level_results[current_level].logical_and = term.modifiers.logical_op != - search_term_modifier_bool::m_or; - level_results[current_level].state = true; - level_results[current_level].has_value = false; - opened += 1; - } - } - - // combine into whichever level the term now sits in, so a '(' beyond max_levels still - // contributes its match rather than being discarded - contribute(level_results[current_level], is_match, - opened > 0 || term.modifiers.logical_op != search_term_modifier_bool::m_or); - - for (auto i = 0; i < term.modifiers.end_group; i++) - { - fold_level(level_results, current_level); - } - } - - // an unclosed '(' must still resolve into level 0; otherwise level 0 keeps its initial - // 'true' and the query matches every item - while (current_level > 0) - { - fold_level(level_results, current_level); - } + const auto matched = combine_search_terms( + _search._terms, + [&](const search_term& term) { return match_term(folder_name, file, md, term).is_match(); }, + [](bool& state, const bool value) { state = state && value; }, + [](bool& state, const bool value) { state = state || value; }, + [] { return true; }); - return {level_results[0].state ? search_result_type::match_multiple : search_result_type::no_match}; + return {matched ? search_result_type::match_multiple : search_result_type::no_match}; } df::search_result df::search_matcher::match_folder(const str::cached folder_name, const str::cached name) const diff --git a/src/model_search.h b/src/model_search.h index 9ff23d0e..4d2d3f39 100644 --- a/src/model_search.h +++ b/src/model_search.h @@ -8,7 +8,8 @@ // Purpose: Search query parsing and matching. Parses search expressions into terms, // handles property filters, date ranges, location queries, duplicate and related-item -// matching, and the scope-aware autocompletion classifier. +// matching, and the scope-aware autocompletion classifier. The text predicates, the date keys and the way +// term answers combine are shared with the search index (model_search_index.h), so the two cannot disagree. #pragma once #include "model_property.h" @@ -1367,6 +1368,9 @@ namespace df search_t parse_from_input(std::string_view text) const; static search_t parse_path(std::string_view text); static search_t parse(std::string_view text); + // Reads text as parse does without asking the file system whether a path exists: a path is taken + // as written. For a worker that must never wait on a share, such as one counting predictions. + static search_t parse_without_probing(std::string_view text); // Whether text can be read as this search without asking the file system. Parsing asks it // whether "C:\Pics 2019" is one folder or a folder and a year, and whether a path names a file; @@ -1421,6 +1425,173 @@ namespace df paths selection; }; + // How a search's terms combine: AND by default, OR where written, nested by parentheses. Generic in + // what is combined, so the matcher folds one item's answers and the search index folds whole sets of + // items through exactly the same steps - which is what makes the two agree item for item. + template + Value combine_search_terms(const std::vector& terms, Leaf&& leaf, And&& and_into, Or&& or_into, + Identity&& identity) + { + if (terms.size() == 1) return leaf(terms.front()); + + struct level + { + bool logical_and = true; + // A level starts with no value rather than with an operator's identity. `true` is the + // identity for AND but the annihilator for OR, so an OR level seeded with it stays true + // whatever it later receives - which is what made a nested group after OR match everything. + bool has_value = false; + Value state{}; + }; + + constexpr auto max_levels = 32; + std::array levels{}; + auto current = 0; + + const auto contribute = [&and_into, &or_into](level& target, Value value, const bool logical_and) + { + if (!target.has_value) + { + target.state = std::move(value); + target.has_value = true; + } + else if (logical_and) + { + and_into(target.state, value); + } + else + { + or_into(target.state, value); + } + }; + + const auto fold = [&levels, ¤t, &contribute, &identity] + { + if (current > 0) + { + auto& child = levels[current]; + contribute(levels[current - 1], child.has_value ? std::move(child.state) : identity(), + child.logical_and); + current -= 1; + } + }; + + for (const auto& term : terms) + { + auto value = leaf(term); + auto opened = 0; + + for (auto i = 0; i < term.modifiers.begin_group; i++) + { + if (current < max_levels - 1) + { + current += 1; + levels[current].logical_and = term.modifiers.logical_op != search_term_modifier_bool::m_or; + levels[current].has_value = false; + levels[current].state = Value{}; + opened += 1; + } + } + + // combine into whichever level the term now sits in, so a '(' beyond max_levels still + // contributes its match rather than being discarded + contribute(levels[current], std::move(value), + opened > 0 || term.modifiers.logical_op != search_term_modifier_bool::m_or); + + for (auto i = 0; i < term.modifiers.end_group; i++) + { + fold(); + } + } + + // an unclosed '(' must still resolve into level 0; otherwise level 0 keeps its identity and the + // query matches every item + while (current > 0) + { + fold(); + } + + return levels[0].has_value ? std::move(levels[0].state) : identity(); + } + + // The fields a free-text term reads, in the order compare_text reports the one that matched. The + // search index builds from the same tables, so what it finds cannot drift from what matches. + struct text_contains_field + { + str::cached prop::item_metadata::* member; + prop::key_ref key; + }; + + struct text_format_field + { + prop::key_ref key; + std::string (*format)(const prop::item_metadata& md, file_type_ref ft); + // Every input format reads, packed, so two items with one key always format alike. + uint64_t (*inputs)(const prop::item_metadata& md, file_type_ref ft); + }; + + struct text_split_field + { + str::cached prop::item_metadata::* member; + prop::key_ref key; + bool (*separator)(char); + }; + + // Searched inside by contains_term, after the file name. + std::span text_contains_fields(); + // Formatted for display and compared whole by same_term, after the file size. + std::span text_format_fields(); + // Split into entries, one of which compare_term must equal. + std::span text_split_fields(); + + // Whether a free-text term matches this one value: found in it, equal to it as formatted, or equal + // to it as one entry of a split field. They read nothing but the value, so the search index asks + // once per distinct value what the matcher asks once per item. + bool text_term_contains(const search_term& term, std::string_view value); + bool text_term_same(const search_term& term, std::string_view value); + bool text_term_equals_part(const search_term& term, std::string_view part); + + // The form two texts share exactly when term matching calls them equal - NFC normalised, then each + // code point lower-cased as str::icmp compares it - written four bytes a code point so that no two + // different sequences can share one. + std::string search_fold_key(std::string_view text); + + // The dates a date term compares an item by, as Group by and the history chart key it: the original + // date (the metadata's capture date, else the file's creation stamp), the created date (the + // metadata's, else the same stamp) and the modified stamp, the stamps in local time. Only the keys + // `target` compares are filled; `any` fills all three. + struct date_term_keys + { + date_t original; + date_t created; + date_t modified; + }; + + date_term_keys date_term_keys_of(date_parts_prop target, const index_file_item& file, + const prop::item_metadata* md); + + // Whether a date term finds an item, given the keys it compares the item by and `matches`, which + // decides one key and answers false for one that is not a valid date. On the `any` path the original + // and created keys usually agree, and when they do the first has already answered. The search index + // asks this with its own keys, so it tests the same keys in the same order as the matcher. + template + bool date_term_found(const date_parts_prop target, const Key& original, const Key& created, const Key& modified, + Matches&& matches) + { + const bool is_any = target == date_parts_prop::any; + const bool by_original = target == date_parts_prop::original || is_any; + + if (by_original && matches(original)) return true; + + if ((target == date_parts_prop::created || is_any) && !(by_original && created == original) && + matches(created)) + { + return true; + } + + return (target == date_parts_prop::modified || is_any) && matches(modified); + } + class search_matcher { const search_t& _search; @@ -1512,8 +1683,30 @@ namespace df const bool has_related = false; bool can_contain(const search_presence_mask& available_presence) const; + + // The presence bits every match must carry, or none when the search cannot promise any. + const search_presence_mask& required_presence() const + { + return _required_presence; + } + search_result match_term(str::cached folder_name, const index_file_item& file, const search_term& term) const; + // The same answers for an item whose metadata the caller already holds, null when it has none, so + // a search reads each item's metadata once rather than once per term. + search_result match_term(str::cached folder_name, const index_file_item& file, const prop::item_metadata* md, + const search_term& term) const; search_result match_all_terms(str::cached folder_name, const index_file_item& file) const; + search_result match_all_terms(str::cached folder_name, const index_file_item& file, + const prop::item_metadata* md) const; + // A positive free-text term holding `**` matches every item in a folder its path matches. + bool text_term_matches_folder(const search_term& term, str::cached folder_name) const; + // Whether a date term's date compares true against one date key, today being the matcher's today. + bool date_key_matches(const search_term& term, date_t key) const; + + const search_t& search() const + { + return _search; + } // The strongest relation this item has to the item the search started at, or nothing when it // has none. Axis priority resolves an item that qualifies several ways, so it appears once. diff --git a/src/model_search_index.cpp b/src/model_search_index.cpp new file mode 100644 index 00000000..82897237 --- /dev/null +++ b/src/model_search_index.cpp @@ -0,0 +1,828 @@ +// This file is part of the Diffractor photo and video organizer +// Copyright 2026 Zac Walker +// +// This program is free software; you can redistribute it and / or modify it +// under the terms of the LGPL License either version 2.1 or later. +// License details are available at https://www.gnu.org/licenses/lgpl-2.1.html +// This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY + +// Purpose: Building the search index snapshot and answering searches from it. Every answer either comes +// from a value table read with the very predicate the matcher applies to that value, or from the matcher +// itself - asked once for each value or day a group of documents shares, or about each document - so the +// snapshot finds exactly what a scan finds. + +#include "pch.h" + +#include "model_search_index.h" +#include "files.h" + +namespace +{ + constexpr uint32_t no_value = std::numeric_limits::max(); + + // Enough for every row of a sidebar refresh or the predictions of a keystroke to share their terms, + // and small enough that a large collection's sets cannot accumulate. + constexpr size_t max_cached_leaves = 64; + // The distinct presence requirements of one pass's searches are a handful. + constexpr size_t max_cached_eligible = 16; + + // A file size as format_size reads it: the unit and the size in tenths of it. + struct size_key + { + const char* unit = nullptr; + uint64_t tenths = 0; + + bool operator==(const size_key&) const = default; + }; + + struct size_key_hash + { + size_t operator()(const size_key& k) const + { + return std::hash{}(k.tenths) ^ std::hash{}(k.unit) << 1; + } + }; + + uint32_t intern_key(df::hash_map& ids, std::string key) + { + const auto next = static_cast(ids.size()); + return ids.try_emplace(std::move(key), next).first->second; + } + + size_t key_map_bytes(const df::hash_map& ids) + { + auto result = ids.bucket_count() * sizeof(void*); + + for (const auto& id : ids) + { + result += sizeof(id) + 2 * sizeof(void*); + if (id.first.capacity() > sizeof(std::string)) result += id.first.capacity(); + } + + return result; + } +} + +void df::doc_set::set_range(const uint32_t first, const uint32_t count) +{ + if (count == 0) return; + + const auto last = first + count - 1; + const auto first_word = first >> 6; + const auto last_word = last >> 6; + const auto first_mask = ~uint64_t{0} << (first & 63); + const auto last_mask = ~uint64_t{0} >> (63 - (last & 63)); + + if (first_word == last_word) + { + _words[first_word] |= first_mask & last_mask; + return; + } + + _words[first_word] |= first_mask; + for (auto w = first_word + 1; w < last_word; ++w) _words[w] = ~uint64_t{0}; + _words[last_word] |= last_mask; +} + +void df::doc_set::reset_range(const uint32_t first, const uint32_t count) +{ + if (count == 0) return; + + const auto last = first + count - 1; + const auto first_word = first >> 6; + const auto last_word = last >> 6; + const auto first_mask = ~uint64_t{0} << (first & 63); + const auto last_mask = ~uint64_t{0} >> (63 - (last & 63)); + + if (first_word == last_word) + { + _words[first_word] &= ~(first_mask & last_mask); + return; + } + + _words[first_word] &= ~first_mask; + for (auto w = first_word + 1; w < last_word; ++w) _words[w] = 0; + _words[last_word] &= ~last_mask; +} + +void df::doc_set::intersect(const doc_set& other) +{ + assert_true(_size == other._size); + const auto n = std::min(_words.size(), other._words.size()); + for (size_t i = 0; i < n; ++i) _words[i] &= other._words[i]; +} + +void df::doc_set::unite(const doc_set& other) +{ + assert_true(_size == other._size); + const auto n = std::min(_words.size(), other._words.size()); + for (size_t i = 0; i < n; ++i) _words[i] |= other._words[i]; +} + +void df::doc_set::invert() +{ + for (auto& w : _words) w = ~w; + clear_tail(); +} + +bool df::doc_set::any() const +{ + return std::ranges::any_of(_words, [](const uint64_t w) { return w != 0; }); +} + +uint64_t df::doc_set::count() const +{ + uint64_t result = 0; + for (const auto w : _words) result += std::popcount(w); + return result; +} + +uint64_t df::doc_set::count_and(const doc_set& other) const +{ + assert_true(_size == other._size); + const auto n = std::min(_words.size(), other._words.size()); + uint64_t result = 0; + for (size_t i = 0; i < n; ++i) result += std::popcount(_words[i] & other._words[i]); + return result; +} + +df::value_docs df::value_docs_builder::freeze(const uint32_t value_count) +{ + std::vector offsets(static_cast(value_count) + 1, 0); + + for (const auto& pair : _pairs) + { + ++offsets[pair.first + 1]; + } + + for (size_t i = 1; i < offsets.size(); ++i) + { + offsets[i] += offsets[i - 1]; + } + + std::vector docs(_pairs.size()); + + { + auto cursor = offsets; + + for (const auto& pair : _pairs) + { + docs[cursor[pair.first]++] = pair.second; + } + } + + _pairs.clear(); + _pairs.shrink_to_fit(); + + // Pairs arrive in document order, so each value's documents are already ascending and a value read + // twice from one document - a name that is also its file_name field - sits beside itself. + uint32_t out = 0; + + for (uint32_t value = 0; value < value_count; ++value) + { + const auto begin = offsets[value]; + const auto end = offsets[value + 1]; + offsets[value] = out; + auto last = no_value; + + for (auto i = begin; i < end; ++i) + { + if (docs[i] != last) + { + last = docs[i]; + docs[out++] = last; + } + } + } + + offsets[value_count] = out; + docs.resize(out); + docs.shrink_to_fit(); + return {std::move(offsets), std::move(docs)}; +} + +std::string df::search_index::current_format_signature() +{ + const auto* const picture = files::file_type_from_name(std::string_view("a.jpg")); + const auto* const video = files::file_type_from_name(std::string_view("a.mp4")); + + return std::format("{}|{}|{}|{}|{}|{}|{}|{}", + format_size(file_size(uint64_t{1536})), + prop::format_pixels({}, picture), + prop::format_pixels({100, 100}, picture), + prop::format_pixels({500, 500}, picture), + prop::format_pixels({1920, 1080}, video), + prop::format_audio_sample_rate(44100u), + date_t(2024, 1, 15, 12).system_to_local().to_int64(), + date_t(2024, 7, 15, 12).system_to_local().to_int64()); +} + +std::shared_ptr df::search_index::build(const folder_list& folders, const uint64_t generation, + const std::function& abandon) +{ + auto result = std::make_shared(); + auto& index = *result; + index._generation = generation; + index._format_signature = current_format_signature(); + index._date_day = platform::now().to_days(); + + folder_list members; + members.reserve(folders.size()); + size_t doc_estimate = 0; + + for (const auto& [path, node] : folders) + { + if (node && node->is_in_collection.load()) + { + members.emplace_back(path, node); + doc_estimate += node->files.size(); + } + } + + // Path order, so a folder's documents are contiguous and documents compare in the order paths do. + std::ranges::sort(members, [](const auto& left, const auto& right) { return left.first.compare(right.first) < 0; }); + + index._folders.reserve(members.size()); + index._files.reserve(doc_estimate); + index._doc_folder.reserve(doc_estimate); + index._metadata.reserve(doc_estimate); + index._date_keys.reserve(doc_estimate * 3); + + hash_map text_ids; + value_docs_builder text_docs; + value_docs_builder formatted_docs; + value_docs_builder split_docs; + value_docs_builder tag_docs; + + // What a rating or label value term reads of a document: whether it has metadata, and the value. + constexpr auto no_metadata = std::numeric_limits::max(); + hash_map rating_ids; + hash_map label_ids; + value_docs_builder rating_docs; + value_docs_builder label_docs; + + const auto group = [](hash_map& ids, std::vector& representatives, + value_docs_builder& docs, const uint32_t value, const uint32_t doc) + { + const auto [found, inserted] = ids.try_emplace(value, static_cast(representatives.size())); + if (inserted) representatives.emplace_back(doc); + docs.add(found->second, doc); + }; + + hash_map date_ids; + + const auto date_id = [&index, &date_ids](const date_t date) + { + const auto [found, inserted] = date_ids.try_emplace(date.to_days(), static_cast(index._date_values.size())); + if (inserted) index._date_values.emplace_back(date); + return found->second; + }; + + // Formatting costs far more than a lookup, and a collection holds few distinct sizes once rounded and + // few distinct numbers per field, so each is formatted once for every document sharing it. + const auto format_fields = text_format_fields(); + std::vector> format_memo(format_fields.size()); + hash_map size_memo; + const auto split_fields = text_split_fields(); + std::vector>> split_memo(split_fields.size()); + hash_map> tag_memo; + + const auto add_text = [&index, &text_ids, &text_docs](const str::cached value, const uint32_t doc) + { + const auto next = static_cast(index._text_values.size()); + const auto [found, inserted] = text_ids.try_emplace(value.id, next); + if (inserted) index._text_values.emplace_back(value); + text_docs.add(found->second, doc); + }; + + const auto formatted_id = [&index](const std::string& text) + { + // Empty never equals a term with text, and an empty term is asked of each document instead. + return text.empty() ? no_value : intern_key(index._formatted_ids, search_fold_key(text)); + }; + + for (const auto& [path, node] : members) + { + if (abandon()) return nullptr; + + const auto folder = static_cast(index._folders.size()); + // Read before the files: a change that lands after this read bumps the revision past it, so the + // folder reads as changed even when the change itself was seen below. + const auto revision = node->search_revision_snapshot(); + const auto first_doc = static_cast(index._files.size()); + + for (const auto& file : node->files) + { + const auto doc = static_cast(index._files.size()); + auto md = file.metadata.load(); + + add_text(file.name, doc); + + group(rating_ids, index._ratings.representatives, rating_docs, + md ? static_cast(static_cast(md->rating)) : no_metadata, doc); + group(label_ids, index._labels.representatives, label_docs, md ? md->label.id : no_metadata, doc); + + const auto dates = date_term_keys_of(date_parts_prop::any, file, md.get()); + index._date_keys.push_back(date_id(dates.original)); + index._date_keys.push_back(date_id(dates.created)); + index._date_keys.push_back(date_id(dates.modified)); + + const auto rounded = round_size(static_cast(file.size.to_int64())); + const auto [size_found, size_inserted] = size_memo.try_emplace( + size_key{rounded.unit.data(), rounded.n * 10 + rounded.dec}, no_value); + if (size_inserted) size_found->second = formatted_id(format_size(file.size)); + if (size_found->second != no_value) formatted_docs.add(size_found->second, doc); + + if (md) + { + const auto& metadata = *md; + + for (const auto& field : text_contains_fields()) + { + const auto value = metadata.*field.member; + if (!value.is_empty()) add_text(value, doc); + } + + for (size_t i = 0; i < format_fields.size(); ++i) + { + const auto& field = format_fields[i]; + const auto [found, inserted] = format_memo[i].try_emplace(field.inputs(metadata, file.ft), no_value); + if (inserted) found->second = formatted_id(field.format(metadata, file.ft)); + if (found->second != no_value) formatted_docs.add(found->second, doc); + } + + for (size_t i = 0; i < split_fields.size(); ++i) + { + const auto& field = split_fields[i]; + const auto value = metadata.*field.member; + if (value.is_empty()) continue; + + const auto [found, inserted] = split_memo[i].try_emplace(value.id); + auto& parts = found->second; + + if (inserted) + { + split2(value, true, [&index, &parts](const std::string_view part) + { + parts.emplace_back(intern_key(index._split_ids, search_fold_key(part))); + }, field.separator); + } + + for (const auto id : parts) split_docs.add(id, doc); + } + + if (!metadata.tags.is_empty()) + { + const auto [found, inserted] = tag_memo.try_emplace(metadata.tags.id); + auto& parts = found->second; + + if (inserted) + { + split2(metadata.tags, true, [&index, &parts](const std::string_view part) + { + parts.emplace_back(intern_key(index._tag_ids, search_fold_key(str::trim(part)))); + }); + } + + for (const auto id : parts) tag_docs.add(id, doc); + } + } + + index._files.emplace_back(&file); + index._doc_folder.emplace_back(folder); + index._metadata.emplace_back(std::move(md)); + } + + index._folder_by_node.emplace(node.get(), folder); + index._folder_by_path.emplace(path, folder); + index._folders.push_back({path, node, revision, first_doc, static_cast(index._files.size()) - first_doc}); + } + + index._listed_docs = doc_set(index.doc_count(), true); + + for (uint32_t doc = 0; doc < index.doc_count(); ++doc) + { + if (index._files[doc]->flags && index_item_flags::is_sidecar) index._listed_docs.reset(doc); + } + + index._text_docs = text_docs.freeze(static_cast(index._text_values.size())); + index._formatted_docs = formatted_docs.freeze(static_cast(index._formatted_ids.size())); + index._split_docs = split_docs.freeze(static_cast(index._split_ids.size())); + index._tag_docs = tag_docs.freeze(static_cast(index._tag_ids.size())); + index._ratings.docs = rating_docs.freeze(static_cast(index._ratings.representatives.size())); + index._labels.docs = label_docs.freeze(static_cast(index._labels.representatives.size())); + + // contains_term compares the NFC form of a value, so that is the form whose trigrams are indexed. + for (uint32_t id = 0; id < index._text_values.size(); ++id) + { + if ((id & 0xfff) == 0 && abandon()) return nullptr; + + const auto value = index._text_values[id].sv(); + const auto normalized = nfc_for_match(value); + index._text_trigrams.add(id, normalized.empty() ? value : std::string_view(normalized)); + } + + index._text_trigrams.freeze(); + + index._bytes = sizeof(search_index) + + index._folders.capacity() * sizeof(folder_entry) + + index._folder_by_node.bucket_count() * sizeof(void*) + + index._folder_by_node.size() * (sizeof(std::pair) + 2 * sizeof(void*)) + + index._folder_by_path.bucket_count() * sizeof(void*) + + index._folder_by_path.size() * (sizeof(std::pair) + 2 * sizeof(void*)) + + index._files.capacity() * sizeof(const index_file_item*) + + index._doc_folder.capacity() * sizeof(uint32_t) + + index._metadata.capacity() * sizeof(prop::item_metadata_ptr) + + index._listed_docs.byte_size() + + index._text_values.capacity() * sizeof(str::cached) + + index._text_docs.byte_size() + + index._text_trigrams.byte_size() + + key_map_bytes(index._formatted_ids) + index._formatted_docs.byte_size() + + key_map_bytes(index._split_ids) + index._split_docs.byte_size() + + key_map_bytes(index._tag_ids) + index._tag_docs.byte_size() + + index._ratings.docs.byte_size() + index._ratings.representatives.capacity() * sizeof(uint32_t) + + index._labels.docs.byte_size() + index._labels.representatives.capacity() * sizeof(uint32_t) + + index._date_keys.capacity() * sizeof(uint32_t) + index._date_values.capacity() * sizeof(date_t); + + return result; +} + +std::optional df::search_index::folder_of(const index_folder_item* node) const +{ + const auto found = _folder_by_node.find(node); + if (found == _folder_by_node.end()) return std::nullopt; + return found->second; +} + +std::optional df::search_index::find_folder(const folder_path path) const +{ + const auto found = _folder_by_path.find(path); + if (found == _folder_by_path.end()) return std::nullopt; + return found->second; +} + +std::optional df::search_index::find_doc(const uint32_t folder, const str::cached name) const +{ + const auto& entry = _folders[folder]; + const auto first = _files.begin() + entry.first_doc; + const auto last = first + entry.doc_count; + const auto found = std::lower_bound(first, last, name.sv(), [](const index_file_item* file, const std::string_view n) + { + return icmp(file->name, n) < 0; + }); + + if (found == last || icmp((*found)->name, name) != 0) return std::nullopt; + return static_cast(found - _files.begin()); +} + +bool df::search_index::is_indexed_term(const search_term& term, const bool formats_current, + const bool dates_current) const +{ + if (groups_of(term)) return true; + if (term.is_date()) return dates_current; + + // An empty or wildcard term matches by other rules than equality and containment, so the documents + // are asked one by one. + if (term.text.empty() || term._is_wildcard) return false; + if (term.type == search_term_type::text) return formats_current; + + // A tag: term compares equality, which no ordering modifier changes the meaning of. + return term.type == search_term_type::value && term.key == prop::tag && + !term.modifiers.greater_than && !term.modifiers.less_than; +} + +// compare_val reads a rating or label term's own property and nothing else, so every document holding a +// value answers such a term alike, whatever its modifiers, wildcards or quoting. +const df::search_index::property_groups* df::search_index::groups_of(const search_term& term) const +{ + if (term.type != search_term_type::value) return nullptr; + if (term.key == prop::rating) return &_ratings; + if (term.key == prop::label) return &_labels; + return nullptr; +} + +df::doc_set df::search_index::evaluate_groups(const property_groups& groups, const search_matcher& matcher, + const search_term& term) const +{ + doc_set result(doc_count()); + + for (uint32_t id = 0; id < groups.representatives.size(); ++id) + { + const auto doc = groups.representatives[id]; + const auto& folder = _folders[_doc_folder[doc]]; + + if (matcher.match_term(folder.path.text(), *_files[doc], _metadata[doc].get(), term).is_match()) + { + groups.docs.add_to(result, id); + } + } + + return result; +} + +// What match_term answers for a date term, with each document's keys as days: date_term_found decides +// which keys the term tests, and each day is compared once, however many documents fall on it. +df::doc_set df::search_index::evaluate_dates(const search_matcher& matcher, const search_term& term) const +{ + doc_set result(doc_count()); + std::vector answers(_date_values.size(), -1); + + const auto matches = [this, &matcher, &term, &answers](const uint32_t id) + { + auto& answer = answers[id]; + if (answer < 0) answer = matcher.date_key_matches(term, _date_values[id]) ? 1 : 0; + return answer != 0; + }; + + const auto target = term.date_val.target; + const auto positive = term.modifiers.positive; + + for (uint32_t doc = 0; doc < doc_count(); ++doc) + { + const auto* const keys = _date_keys.data() + static_cast(doc) * 3; + if (date_term_found(target, keys[0], keys[1], keys[2], matches) == positive) result.set(doc); + } + + return result; +} + +df::doc_set df::search_index::evaluate_term(const search_matcher& matcher, const search_term& term, + const doc_set& within, const bool formats_current, + const bool dates_current, const cancel_token& token) const +{ + if (const auto* const groups = groups_of(term)) return evaluate_groups(*groups, matcher, term); + if (term.is_date()) return dates_current ? evaluate_dates(matcher, term) : evaluate_each(matcher, term, within, token); + + if (is_indexed_term(term, formats_current, dates_current)) + { + return term.type == search_term_type::text ? evaluate_text(matcher, term, token) : evaluate_tag(term); + } + + return evaluate_each(matcher, term, within, token); +} + +// What search_matcher::match_term answers for a free-text term, value by value: compare_text matches an +// item when the term is inside its name or a text field, equals its size or a number as displayed, or +// equals one entry of its tags or artists - and a folder's path decides the term for all it holds. +df::doc_set df::search_index::evaluate_text(const search_matcher& matcher, const search_term& term, + const cancel_token& token) const +{ + doc_set result(doc_count()); + + const auto query = term._nfc_text.empty() ? std::string_view(term.text) : std::string_view(term._nfc_text); + const auto candidates = _text_trigrams.candidates(query); + + // A term too short for trigrams is looked for in every distinct value, long enough in a large + // collection for a newer keystroke to want it stopped. A cancelled search's answer is discarded. + uint32_t checked = 0; + const auto add_if_contains = [this, &term, &result, &token, &checked](const uint32_t id) + { + if (text_term_contains(term, _text_values[id])) _text_docs.add_to(result, id); + return (++checked & 0xfff) != 0 || !token.is_cancelled(); + }; + + if (candidates) + { + for (const auto id : *candidates) + { + if (!add_if_contains(id)) return result; + } + } + else + { + for (uint32_t id = 0; id < _text_values.size(); ++id) + { + if (!add_if_contains(id)) return result; + } + } + + const auto key = search_fold_key(term.text); + + if (const auto found = _formatted_ids.find(key); found != _formatted_ids.end()) + { + _formatted_docs.add_to(result, found->second); + } + + if (const auto found = _split_ids.find(key); found != _split_ids.end()) + { + _split_docs.add_to(result, found->second); + } + + for (const auto& folder : _folders) + { + const auto folder_name = folder.path.text(); + const auto decides = matcher.text_term_matches_folder(term, folder_name) || + (!term.modifiers.positive && text_term_contains(term, folder_name)); + + if (decides) result.set_range(folder.first_doc, folder.doc_count); + } + + if (!term.modifiers.positive) result.invert(); + return result; +} + +df::doc_set df::search_index::evaluate_tag(const search_term& term) const +{ + doc_set result(doc_count()); + + if (const auto found = _tag_ids.find(search_fold_key(term.text)); found != _tag_ids.end()) + { + _tag_docs.add_to(result, found->second); + } + + if (!term.modifiers.positive) result.invert(); + return result; +} + +df::doc_set df::search_index::evaluate_each(const search_matcher& matcher, const search_term& term, + const doc_set& within, const cancel_token& token) const +{ + doc_set result(doc_count()); + uint32_t checked = 0; + auto cancelled = false; + + within.for_each([&](const uint32_t doc) + { + if (cancelled) return; + + if ((++checked & 0xfff) == 0 && token.is_cancelled()) + { + cancelled = true; + return; + } + + const auto folder_name = _folders[_doc_folder[doc]].path.text(); + + if (matcher.match_term(folder_name, *_files[doc], _metadata[doc].get(), term).is_match()) + { + result.set(doc); + } + }); + + return result; +} + +df::search_index_view::search_index_view(const search_index& index, const search_index::folder_list& folders) : + _clean(index.folders().size(), 0), _clean_docs(index.doc_count()) +{ + for (uint32_t position = 0; position < folders.size(); ++position) + { + const auto& node = folders[position].second; + if (!node || !node->is_in_collection.load()) continue; + + _collection_doc_count += node->files.size(); + + const auto entry = index.folder_of(node.get()); + + if (!entry || index.folders()[*entry].revision != node->search_revision_snapshot()) + { + _unclean_folders.emplace_back(position); + continue; + } + + const auto& folder = index.folders()[*entry]; + _clean[*entry] = 1; + _clean_docs.set_range(folder.first_doc, folder.doc_count); + _clean_doc_count += folder.doc_count; + } +} + +bool df::search_index_view::is_clean(const search_index& index, const index_folder_item* node) const +{ + const auto entry = index.folder_of(node); + return entry && _clean[*entry] != 0; +} + +df::search_index_evaluator::search_index_evaluator(std::shared_ptr index, + const search_index::folder_list& folders) : + _index(std::move(index)), + _view(*_index, folders), + _formats_current(_index->format_signature() == search_index::current_format_signature()), + _dates_current(_formats_current && _index->date_day() == platform::now().to_days()), + _base(_view.clean_docs()) +{ + _base.intersect(_index->listed_docs()); +} + +df::doc_set df::search_index_evaluator::eligible_docs(const search_matcher& matcher) +{ + // The presence prefilter rejects only what lacks a bit the search requires; with none required, + // every document passes. + const auto required = matcher.required_presence().types; + if (required == 0) return _base; + + const auto can_match_folder = matcher.can_match_folder; + const auto found = std::ranges::find_if(_eligible, [required, can_match_folder](const cached_eligible& eligible) + { + return eligible.required_presence == required && eligible.can_match_folder == can_match_folder; + }); + + if (found != _eligible.end()) return found->docs; + + auto result = _base; + + for (const auto& folder : _index->folders()) + { + if (!can_match_folder && !matcher.can_contain(folder.node->search_presence_summary.load())) + { + result.reset_range(folder.first_doc, folder.doc_count); + } + } + + result.for_each([this, &matcher, &result](const uint32_t doc) + { + if (!matcher.can_contain(_index->file(doc).search_presence.load())) result.reset(doc); + }); + + if (_eligible.size() < max_cached_eligible) _eligible.push_back({required, can_match_folder, result}); + return result; +} + +df::doc_set df::search_index_evaluator::leaf_docs(const search_matcher& matcher, const search_term& term, + const doc_set& within, const bool within_is_eligible, + const cancel_token& token) +{ + // Grouping and the operator joining a term to the last are the combination's business, not the + // term's, so they are not part of what identifies its answer. + auto key = term; + key.modifiers.logical_op = search_term_modifier_bool::none; + key.modifiers.begin_group = 0; + key.modifiers.end_group = 0; + + const auto required = matcher.required_presence().types; + const auto can_match_folder = matcher.can_match_folder; + + const auto found = std::ranges::find_if(_leaves, [&key, required, can_match_folder](const cached_leaf& leaf) + { + return leaf.required_presence == required && leaf.can_match_folder == can_match_folder && leaf.term == key; + }); + + if (found != _leaves.end()) return found->docs; + + const auto indexed = _index->is_indexed_term(term, _formats_current, _dates_current); + auto docs = _index->evaluate_term(matcher, term, within, _formats_current, _dates_current, token); + + // A set asked only of documents an earlier term had already narrowed to is right for this search + // alone, so only answers covering every eligible document are kept for the next. + if ((indexed || within_is_eligible) && !token.is_cancelled()) + { + if (_leaves.size() >= max_cached_leaves) _leaves.erase(_leaves.begin()); + _leaves.push_back({std::move(key), required, can_match_folder, docs}); + } + + return docs; +} + +df::doc_set df::search_index_evaluator::evaluate(const search_matcher& matcher, const cancel_token& token) +{ + const auto& terms = matcher.search().terms(); + if (terms.empty()) return doc_set(_index->doc_count()); + + const auto eligible = eligible_docs(matcher); + + const auto is_conjunction = std::ranges::all_of(terms, [](const search_term& term) + { + return term.modifiers.begin_group == 0 && + term.modifiers.end_group == 0 && + term.modifiers.logical_op != search_term_modifier_bool::m_or; + }); + + if (is_conjunction) + { + // AND gives one answer in any order, so indexed terms go first and a term asked of each document + // is asked only of the documents every earlier term kept. + std::vector ordered; + ordered.reserve(terms.size()); + for (const auto& term : terms) ordered.emplace_back(&term); + + std::ranges::stable_partition(ordered, [this](const search_term* term) + { + return _index->is_indexed_term(*term, _formats_current, _dates_current); + }); + + auto result = eligible; + auto narrowed = false; + + for (const auto* term : ordered) + { + if (token.is_cancelled() || !result.any()) break; + result.intersect(leaf_docs(matcher, *term, result, !narrowed, token)); + narrowed = true; + } + + return result; + } + + auto result = combine_search_terms( + terms, + [this, &matcher, &eligible, &token](const search_term& term) + { + return leaf_docs(matcher, term, eligible, true, token); + }, + [](doc_set& state, const doc_set& value) { state.intersect(value); }, + [](doc_set& state, const doc_set& value) { state.unite(value); }, + [size = _index->doc_count()] { return doc_set(size, true); }); + + result.intersect(eligible); + return result; +} diff --git a/src/model_search_index.h b/src/model_search_index.h new file mode 100644 index 00000000..f23cec6c --- /dev/null +++ b/src/model_search_index.h @@ -0,0 +1,418 @@ +// This file is part of the Diffractor photo and video organizer +// Copyright 2026 Zac Walker +// +// This program is free software; you can redistribute it and / or modify it +// under the terms of the LGPL License either version 2.1 or later. +// License details are available at https://www.gnu.org/licenses/lgpl-2.1.html +// This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY + +// Purpose: The search index - an immutable snapshot of the collection laid out to answer a search a set +// at a time - and its evaluation. Items are numbered in path order, the distinct values free-text and tag +// terms read are mapped to the items holding them, rating and label terms are answered once per value and +// date terms once per day, and a search's terms become sets combined through the same steps the matcher +// applies to one item, so an answer from here is the answer a scan of every item gives. index_state +// builds and publishes it and scans the folders that changed since it was read. + +#pragma once + +#include "model_items.h" +#include "model_postings.h" +#include "model_search.h" + +namespace df +{ + // A set of search index documents, one bit per document. + class doc_set + { + std::vector _words; + uint32_t _size = 0; + + public: + doc_set() noexcept = default; + + explicit doc_set(const uint32_t size, const bool fill = false) : + _words((static_cast(size) + 63) / 64, fill ? ~uint64_t{0} : uint64_t{0}), _size(size) + { + if (fill) clear_tail(); + } + + uint32_t size() const + { + return _size; + } + + void set(const uint32_t doc) + { + _words[doc >> 6] |= uint64_t{1} << (doc & 63); + } + + void reset(const uint32_t doc) + { + _words[doc >> 6] &= ~(uint64_t{1} << (doc & 63)); + } + + bool test(const uint32_t doc) const + { + return (_words[doc >> 6] >> (doc & 63) & 1) != 0; + } + + void set_range(uint32_t first, uint32_t count); + void reset_range(uint32_t first, uint32_t count); + void intersect(const doc_set& other); + void unite(const doc_set& other); + void invert(); + bool any() const; + uint64_t count() const; + // The documents in both, counted without building the intersection. + uint64_t count_and(const doc_set& other) const; + + template + void for_each(Fn&& fn) const + { + for (size_t w = 0; w < _words.size(); ++w) + { + auto bits = _words[w]; + + while (bits != 0) + { + fn(static_cast(w * 64 + std::countr_zero(bits))); + bits &= bits - 1; + } + } + } + + size_t byte_size() const + { + return _words.capacity() * sizeof(uint64_t); + } + + private: + void clear_tail() + { + if (const auto bits = _size % 64; bits != 0) _words.back() &= (uint64_t{1} << bits) - 1; + } + }; + + // Each value's documents, ascending, held contiguously. + class value_docs + { + std::vector _offsets; + std::vector _docs; + + public: + value_docs() noexcept = default; + + value_docs(std::vector offsets, std::vector docs) noexcept : + _offsets(std::move(offsets)), _docs(std::move(docs)) + { + } + + std::span docs(const uint32_t value) const + { + return {_docs.data() + _offsets[value], _docs.data() + _offsets[value + 1]}; + } + + void add_to(doc_set& set, const uint32_t value) const + { + for (const auto doc : docs(value)) set.set(doc); + } + + size_t byte_size() const + { + return (_offsets.capacity() + _docs.capacity()) * sizeof(uint32_t); + } + }; + + // Collects (value, document) pairs in document order and lays them out as value_docs, each value's + // documents ascending and each document once. + class value_docs_builder + { + std::vector> _pairs; + + public: + void add(const uint32_t value, const uint32_t doc) + { + _pairs.emplace_back(value, doc); + } + + value_docs freeze(uint32_t value_count); + }; + + class search_index final + { + public: + using folder_list = std::vector>; + + struct folder_entry + { + folder_path path; + // Holds the node alive, so every document's record stays valid as long as the snapshot is. + index_folder_item_ptr node; + // The node's search revision when it was read: what it held then holds while this does. + uint64_t revision = 0; + uint32_t first_doc = 0; + uint32_t doc_count = 0; + }; + + // Reads every collection folder in `folders`. Answers null when `abandon` turns true part way, so a + // build that indexing has overtaken stops rather than competing with it. + static std::shared_ptr build(const folder_list& folders, uint64_t generation, + const std::function& abandon); + + // What free-text terms compare formatted numbers in - the decimal separator and the translated words + // some formats use - and the time zone date keys are converted in. A snapshot read in another + // answers those terms by asking each document. + static std::string current_format_signature(); + + uint32_t doc_count() const + { + return static_cast(_files.size()); + } + + uint64_t generation() const + { + return _generation; + } + + size_t byte_size() const + { + return _bytes; + } + + const std::string& format_signature() const + { + return _format_signature; + } + + std::span folders() const + { + return _folders; + } + + // Every document but sidecars, which a search lists only beside a related item. + const doc_set& listed_docs() const + { + return _listed_docs; + } + + std::optional folder_of(const index_folder_item* node) const; + std::optional find_folder(folder_path path) const; + // The document holding this file, when its folder is one the snapshot read. + std::optional find_doc(uint32_t folder, str::cached name) const; + + uint32_t doc_folder(const uint32_t doc) const + { + return _doc_folder[doc]; + } + + const index_file_item& file(const uint32_t doc) const + { + return *_files[doc]; + } + + const prop::item_metadata* metadata(const uint32_t doc) const + { + return _metadata[doc].get(); + } + + file_path path(const uint32_t doc) const + { + return _folders[_doc_folder[doc]].path.combine_file(_files[doc]->name); + } + + // A term answered from the value tables rather than by asking each document. Date terms are + // answered so only while `dates_current`: on the day the snapshot was read, in its time zone. + bool is_indexed_term(const search_term& term, bool formats_current, bool dates_current) const; + + // One term's documents, as search_matcher::match_term answers each. Documents outside `within` + // are only answered when the term is indexed; otherwise they are left out. + doc_set evaluate_term(const search_matcher& matcher, const search_term& term, const doc_set& within, + bool formats_current, bool dates_current, const cancel_token& token) const; + + // The day the snapshot was read on. + uint32_t date_day() const + { + return _date_day; + } + + private: + doc_set evaluate_text(const search_matcher& matcher, const search_term& term, const cancel_token& token) const; + doc_set evaluate_tag(const search_term& term) const; + doc_set evaluate_each(const search_matcher& matcher, const search_term& term, const doc_set& within, + const cancel_token& token) const; + + std::vector _folders; + hash_map _folder_by_node; + hash_map _folder_by_path; + std::vector _files; + std::vector _doc_folder; + // Read once, as the folder's revision was read; while the revision holds, so does every pointer. + std::vector _metadata; + doc_set _listed_docs; + + // The distinct names and text fields a free-text term searches inside, with a trigram index over + // them so a term is checked against the few values that could contain it. + std::vector _text_values; + value_docs _text_docs; + trigram_index _text_trigrams; + + // The sizes and numbers a free-text term matches whole, as displayed, keyed by search_fold_key. + hash_map _formatted_ids; + value_docs _formatted_docs; + std::string _format_signature; + + // The entries of tags and artists a free-text term matches one of, keyed by search_fold_key. + hash_map _split_ids; + value_docs _split_docs; + + // Tags as a tag: term compares them, trimmed, keyed by search_fold_key. + hash_map _tag_ids; + value_docs _tag_docs; + + // Documents grouped by the value of a property its value terms read nothing else of an item for, + // so a term is asked of one document holding each value and that answer stands for the rest. + struct property_groups + { + value_docs docs; + std::vector representatives; + }; + + property_groups _ratings; + property_groups _labels; + + // Each document's original, created and modified date keys, three to a document, as the day each + // falls on: an index into _date_values, which holds one date of that day. A date term compares + // only the day, so each day is asked once rather than each document converting its stamps. + std::vector _date_keys; + std::vector _date_values; + // Which keys date_term_keys_of chooses, and what is a valid date, depend on the day it is. + uint32_t _date_day = 0; + + const property_groups* groups_of(const search_term& term) const; + doc_set evaluate_groups(const property_groups& groups, const search_matcher& matcher, + const search_term& term) const; + doc_set evaluate_dates(const search_matcher& matcher, const search_term& term) const; + + uint64_t _generation = 0; + size_t _bytes = 0; + }; + + // Which of a snapshot's folders still hold exactly what it read, against one listing of the index. + class search_index_view + { + std::vector _clean; + doc_set _clean_docs; + uint64_t _clean_doc_count = 0; + uint64_t _collection_doc_count = 0; + // Positions, in the folder list the view was taken from, of the collection folders the snapshot + // does not answer for: changed since it was read, or never read by it. + std::vector _unclean_folders; + + public: + search_index_view(const search_index& index, const search_index::folder_list& folders); + + bool is_clean(const search_index& index, const index_folder_item* node) const; + + bool is_clean_entry(const uint32_t folder) const + { + return _clean[folder] != 0; + } + + const std::vector& unclean_folders() const + { + return _unclean_folders; + } + + const doc_set& clean_docs() const + { + return _clean_docs; + } + + uint64_t clean_doc_count() const + { + return _clean_doc_count; + } + + // Every document the collection holds now, answered by the snapshot or not. + uint64_t collection_doc_count() const + { + return _collection_doc_count; + } + + // Read by the snapshot but changed since, plus held now but never read by it. + uint64_t stale_doc_count(const search_index& index) const + { + return index.doc_count() - _clean_doc_count + (_collection_doc_count - _clean_doc_count); + } + + // Most of the collection is as the snapshot read it. Past that, scanning the changed folders on top + // would cost more than the scan the snapshot exists to replace. + bool is_useful() const + { + return _clean_doc_count * 2 >= _collection_doc_count; + } + }; + + // Answers searches against one snapshot and one view of it. A term repeated across the searches one + // evaluator is asked - a sidebar's rows, a keystroke's predictions - is answered once. + class search_index_evaluator + { + std::shared_ptr _index; + search_index_view _view; + bool _formats_current = false; + bool _dates_current = false; + // Documents in clean folders that are not sidecars: every search starts from these. + doc_set _base; + + struct cached_leaf + { + search_term term; + uint32_t required_presence = 0; + bool can_match_folder = false; + doc_set docs; + }; + + std::vector _leaves; + + // The documents through each presence prefilter asked of this evaluator. Most of the searches one + // evaluator is asked share a requirement - every tag row requires tags - and reading each + // document's presence is the one step whose cost grows with the collection. + struct cached_eligible + { + uint32_t required_presence = 0; + bool can_match_folder = false; + doc_set docs; + }; + + std::vector _eligible; + + public: + search_index_evaluator(std::shared_ptr index, const search_index::folder_list& folders); + + const search_index& index() const + { + return *_index; + } + + const search_index_view& view() const + { + return _view; + } + + // Answers only while its view is useful. + bool is_useful() const + { + return _view.is_useful(); + } + + // The snapshot documents a related-free, selector-free search matches: in a clean folder, not a + // sidecar, through the presence prefilter, and matched by its terms combined as the matcher + // combines them. + doc_set evaluate(const search_matcher& matcher, const cancel_token& token); + + private: + doc_set eligible_docs(const search_matcher& matcher); + doc_set leaf_docs(const search_matcher& matcher, const search_term& term, const doc_set& within, + bool within_is_eligible, const cancel_token& token); + }; +} diff --git a/src/test_app.cpp b/src/test_app.cpp index c5f51c17..246f72fd 100644 --- a/src/test_app.cpp +++ b/src/test_app.cpp @@ -5493,6 +5493,83 @@ static void should_discard_a_superseded_completion() assert_equal(1, second_delivered, "the newest pass is the one that lands"); } +// design.md: each prediction says how many items it finds, and a suggestion that finds more ranks above +// one of its kind that finds fewer. The number shown is the number running the search lists, and one +// drawn from the collection's own words, tags and places that would find nothing is not offered. +static void should_count_address_completions(shared_test_context& stc) +{ + stc.lazy_load_index(); + stc.test_index.update_summary(); + // As every summary pass does, so the snapshot holds what earlier tests changed in the shared index. + stc.test_index.update_search_index(); + + null_state_strategy ss; + deferred_async_strategy as; + view_state s(ss, as, stc.test_index, make_test_player()); + s.view_mode(view_type::items); + + const auto completes = make_search_auto_complete(s, [](std::string) {}); + completes->initialise([](const ui::auto_complete_results&) {}); + + ui::auto_complete_results results; + + const auto run = [&](const std::string& query) + { + results.clear(); + completes->search(query, [&results](const ui::auto_complete_results& r) { results = r; }); + run_completion_worker(as); + as.drain_ui(); + }; + + const auto found_by = [&stc](const std::string& text) + { + return stc.test_index.count_matches(df::search_t::parse(text), test_token, + index_state::search_execution::scan).total_items().count; + }; + + run("key"); + assert_equal(true, !results.empty(), "a query produces completions"); + assert_equal(true, results.front()->count.has_value(), "what was typed is counted too"); + + auto counted = 0; + + for (const auto& r : results) + { + const auto text = r->edit_text(); + assert_equal(false, r->drop_when_empty && r->count == 0u, std::format("'{}' leads somewhere", text)); + + if (r->count && !df::is_path(str::trim(text))) + { + ++counted; + assert_equal(found_by(text), *r->count, std::format("'{}' says how many it finds", text)); + } + } + + assert_equal(true, counted > 1, "completions are counted"); + + run("#key"); + + std::vector tag_counts; + + for (const auto& r : results) + { + if (r->drop_when_empty && str::starts(r->edit_text(), "#") && r->count) tag_counts.emplace_back(*r->count); + } + + assert_equal(true, !tag_counts.empty(), "tags are suggested and counted"); + assert_equal(true, std::ranges::is_sorted(tag_counts, std::greater<>()), "a tag that finds more ranks higher"); + + // Nothing in the collection holds zzzqqq, so nothing completed after it finds anything. + run("zzzqqq ke"); + assert_equal(true, !results.empty(), "what was typed is still offered"); + assert_equal("zzzqqq ke", results.front()->edit_text(), "and offered first"); + + for (const auto& r : results) + { + assert_equal(false, r->drop_when_empty, std::format("'{}' would find nothing", r->edit_text())); + } +} + static void should_gate_application_alpha_effects() { const auto restore_animations = ui::animations_enabled; @@ -5790,6 +5867,60 @@ static void should_keep_sidebar_rebuild_packets_current() "the current sidebar structural packet can publish"); } +// Every sidebar row is counted by the search index, so its total is what clicking it finds, down to the +// sizes and the picture chosen to show it. A tag or label row that would find nothing is not shown; +// rating rows always are, because together they are one scale. +static void should_count_sidebar_rows_from_the_search_index(shared_test_context& stc) +{ + stc.lazy_load_index(); + stc.test_index.update_summary(); + // As every summary pass does, so the snapshot holds what earlier tests changed in the shared index. + stc.test_index.update_search_index(); + + null_state_strategy ss; + null_async_strategy as; + view_state s(ss, as, stc.test_index, make_test_player()); + + search_item_factory::sidebar_config_snapshot config; + config.favorite_tags = "key1 zz_tag_on_nothing"; + + constexpr search_item_factory factory; + search_counter counter(stc.test_index); + assert_equal(true, counter.is_indexed(), "the sidebar counts from the search index"); + + const auto tags = search_item_factory::count_rows(s, counter, factory.create_tags(s, {}, config), true); + const auto labels = search_item_factory::count_rows(s, counter, factory.create_labels(s, {}), true); + const auto ratings = search_item_factory::count_rows(s, counter, factory.create_ratings(s, {}), false); + + const auto has_row = [](const std::vector& rows, const std::string_view key) + { + return std::ranges::any_of(rows, [key](const search_item_ptr& row) { return row->key == key; }); + }; + + assert_equal(true, has_row(tags, "t:key1"), "a tag that finds items is shown"); + assert_equal(false, has_row(tags, "t:zz_tag_on_nothing"), "a favorite tag that finds nothing is not shown"); + assert_equal(6_z, ratings.size(), "every rating row is shown, empty or not"); + + for (const auto* const rows : {&tags, &labels, &ratings}) + { + for (const auto& row : *rows) + { + assert_equal(true, row->summary_known, std::format("'{}' arrives counted", row->key)); + + const auto expected = stc.test_index.count_matches(row->search, test_token, + index_state::search_execution::scan); + assert_equal(expected.total_items().count, row->summary.total_items().count, + std::format("'{}' counts what clicking it finds", row->key)); + assert_equal(true, row->summary == expected, std::format("'{}' counts sizes and picture alike", row->key)); + } + } + + for (const auto& row : labels) + { + assert_equal(true, row->summary.total_items().count > 0, std::format("'{}' finds items", row->key)); + } +} + static void should_consume_cancelled_globe_release() { assert_equal(true, should_invoke_globe_release(false, false, true), "a normal click invokes"); @@ -6219,6 +6350,7 @@ void register_app_tests(view_state& state, test_registry& tests) tests.add("Should offer recent searches for an empty address"s, should_offer_recent_searches_for_an_empty_address); tests.add("Should discard a superseded completion"s, should_discard_a_superseded_completion); + tests.add("Should count address completions"s, should_count_address_completions); // APP-007 - application-local fades must honor the shared alpha-animation gate. tests.add("Should gate application alpha effects"s, should_gate_application_alpha_effects); tests.add("Should snap app logo hover when animations are disabled"s, @@ -6238,6 +6370,7 @@ void register_app_tests(view_state& state, test_registry& tests) should_hand_the_last_ui_owned_reference_back_to_the_ui_thread); // APP-008 - sidebar structural rebuild packets carry settings snapshots and request identity. tests.add("Should keep sidebar rebuild packets current"s, should_keep_sidebar_rebuild_packets_current); + tests.add("Should count sidebar rows from the search index"s, should_count_sidebar_rows_from_the_search_index); // APP-011 - cancelling a globe drag consumes its release. tests.add("Should consume cancelled globe release"s, should_consume_cancelled_globe_release); diff --git a/src/test_fixtures.cpp b/src/test_fixtures.cpp index 78130357..0f9f9d8b 100644 --- a/src/test_fixtures.cpp +++ b/src/test_fixtures.cpp @@ -290,6 +290,11 @@ void shared_test_context::lazy_load_index() assert_equal(expected_cached_item_count, test_index.stats.media_item_count, "cached item count"); assert_equal(0, empty_index.stats.media_item_count, "cached item count"); + + // Collection-wide searches are then answered by the search index while it is current, which holds + // it to the counts each test was verified against when only the scan answered. + test_index.update_search_index(); + assert_equal(true, test_index.search_index() != nullptr, "the shared index reads a search index"); loaded = true; } } diff --git a/src/test_index.cpp b/src/test_index.cpp index 22c8afe0..3bd139e5 100644 --- a/src/test_index.cpp +++ b/src/test_index.cpp @@ -3947,6 +3947,469 @@ static void should_query_trigram_index() assert_equal(false, index.candidates("do").has_value(), "short query requires a scan"); } +static void should_hold_document_sets() +{ + df::doc_set docs(130); + docs.set_range(60, 10); + assert_equal(uint64_t{10}, docs.count(), "a range across a word boundary"); + assert_equal(true, docs.test(60) && docs.test(69) && !docs.test(59) && !docs.test(70), "the range's ends"); + + docs.reset_range(62, 3); + assert_equal(uint64_t{7}, docs.count(), "a range cleared inside one"); + + auto inverted = docs; + inverted.invert(); + assert_equal(uint64_t{123}, inverted.count(), "inverting stays within the set's size"); + assert_equal(uint64_t{0}, docs.count_and(inverted), "a set and its inverse share nothing"); + assert_equal(uint64_t{130}, df::doc_set(130, true).count(), "a full set holds its size exactly"); + + std::vector visited; + docs.for_each([&visited](const uint32_t doc) { visited.emplace_back(doc); }); + assert_equal(true, visited == std::vector{60, 61, 65, 66, 67, 68, 69}, "documents are visited ascending"); +} + +// The search index folds whole sets through the steps the matcher folds one item's answers through. +// Each assignment of answers to the terms is one item here, so a set that disagreed with the matcher +// about any item - any parse of AND, OR and parentheses - would fail. +static void should_combine_sets_as_the_matcher_combines_answers() +{ + for (const auto query : { + "a b c", "a or b c", "(a or b) c", "a or (b c)", "((a b) or c", "a (b or (c d))", "-a or b", + "a or b or c or d", "(a or b) (c or d)", "a b)", "(((a b", "a or b) or c" + }) + { + const auto search = df::search_t::parse(query); + const auto& terms = search.terms(); + const auto items = 1u << terms.size(); + + const auto sets = df::combine_search_terms( + terms, + [&terms, items](const df::search_term& term) + { + const auto bit = static_cast(&term - terms.data()); + df::doc_set answers(items); + for (uint32_t item = 0; item < items; ++item) if ((item >> bit & 1) != 0) answers.set(item); + return answers; + }, + [](df::doc_set& state, const df::doc_set& value) { state.intersect(value); }, + [](df::doc_set& state, const df::doc_set& value) { state.unite(value); }, + [items] { return df::doc_set(items, true); }); + + for (uint32_t item = 0; item < items; ++item) + { + const auto expected = df::combine_search_terms( + terms, + [&terms, item](const df::search_term& term) + { + return (item >> static_cast(&term - terms.data()) & 1) != 0; + }, + [](bool& state, const bool value) { state = state && value; }, + [](bool& state, const bool value) { state = state || value; }, + [] { return true; }); + + assert_equal(expected, sets.test(item), std::format("'{}' answers {:b}", query, item)); + } + } +} + +static std::string describe_query_items(index_state& index, const df::search_t& search, + const index_state::search_execution execution) +{ + std::vector lines; + + index.query_items(search, [&lines](const index_state::query_item_results& items, bool) + { + for (const auto& item : items) + { + if (item.kind == index_state::query_item_kind::folder) + { + lines.emplace_back(std::format("folder {}", item.folder_path.text().sv())); + } + else + { + lines.emplace_back(std::format("{} {} {} {}", item.path.str(), static_cast(item.match.type), + item.match.key ? item.match.key->id : 0, item.match.text.sv())); + } + } + }, test_token, execution); + + std::ranges::sort(lines); + + std::string result; + for (const auto& line : lines) result += line + "\n"; + return result; +} + +// Every collection-wide search the search index answers must find what a scan of every item finds - +// the same items, matched the same way - and count what the scan counts, down to the sizes and the +// picture chosen to represent them. The queries cover each kind of term, negation, OR, grouping, +// wildcards, the folder rules of free text, formatted numbers and text that needs NFC. +static void should_answer_searches_from_the_search_index_as_a_scan_does(shared_test_context& stc) +{ + stc.lazy_load_index(); + auto& index = stc.test_index; + assert_equal(true, index.search_index() != nullptr, "the shared index has a search index"); + + const auto answers_before = df::query_perf.index_answers.load(); + auto found_something = 0; + + for (const auto query : { + "2012-09-14", "Created:2012-09-14", "2009-11-15", "2014 or 2015", "(2014 or 2015) (May or June)", + "dec-25", "modified:2020-aug", "@video", "@audio", "@commodore", "@archive", "@photo", "key1", + "Tag:key1", "#key1", "Tag:dog Tag:london", "not_exist", "Tag:not_exist", "#ke*", "ke*", "KEY1", + "Megapixels:1.6 dog", "Megapixels:1.6", "(< Megapixels:1.0 > Megapixels:0.5)", "Megapixels:2", "dog", + "-dog", "dog cat", "dog or cat", "dog -cat", "(dog or cat) @photo", "-(dog or cat)", "@photo -@video", + "@photo or @video", "@duplicates", "@photo @duplicates", "@photo -@duplicates", "with:tag", + "without:tag", "with:location", "without:location", "ext:jpg", "-ext:jpg", "jpg", "test", "-test", + "Test", "es", "e", "1/250s", "f/2.8", "ISO100", "stereo", "mono", "44.1kHz", "16bit", "2019", + "rating:5", "> rating:3", "rating:-1", "label:select", "Jana or Alfie", "Jana -Alfie", + "-rating:5", "< rating:2", "rating:0", "label:sel*", "-label:select", "label:approved", + "label:\"to do\"", "rating:5 label:select", "rating:4 or label:review", "dog -rating:1", + "original:2012-09-14", "original:2012", "created:2012", "-2012", "> 2013", "< 2013", + "< created:2014", "> modified:2019", "-dec-25", "@photo 2012", "dec-25 -@video", "-(2014 or 2015)", + "age:30", "age:5000", "-age:5000", "modified:2020", "original:dec-25", + "Bertie or ((Jana or Alfie) or Amalka)", "genre: Rock", "-artist: \"Counting Crows\"", + "artist:\"Counting Crows\"", "@remote", "@panorama", "@movie", "@episode", "@inprogress", + "size:1mb", "> size:1mb", "loc:London", "place:London", "country:\"United Kingdom\"", + "#\"tag with space\"", "\"tag with space\"", "camera:canon", "Canon", "london", "**test**", + "caf\xc3\xa9", "\xed\x95\x9c\xea\xb5\xad", "-\xed\x95\x9c\xea\xb5\xad", "with:tag or @video" + }) + { + const auto search = df::search_t::parse(query); + const auto scanned = describe_query_items(index, search, index_state::search_execution::scan); + const auto indexed = describe_query_items(index, search, index_state::search_execution::automatic); + assert_equal(scanned, indexed, std::format("what '{}' finds", query)); + + const auto scan_counts = index.count_matches(search, test_token, index_state::search_execution::scan); + const auto index_counts = index.count_matches(search, test_token); + assert_equal(scan_counts.total_items().count, index_counts.total_items().count, + std::format("how many '{}' finds", query)); + assert_equal(true, scan_counts == index_counts, std::format("what '{}' counts", query)); + + if (!scanned.empty()) ++found_something; + } + + // The comparison only means something if the index answered and the searches found things. + assert_equal(true, df::query_perf.index_answers.load() > answers_before, "the search index answered"); + assert_equal(true, found_something > 40, "most of the searches find items"); + + // A folder whose name a search matches is listed beside the items, through the search index too. + const auto retro = describe_query_items(index, df::search_t::parse("retro"), index_state::search_execution::automatic); + assert_equal(true, retro.find("folder ") != std::string::npos && retro.find("retro") != std::string::npos, + "a folder the search names is listed"); +} + +static void make_search_index_collection(const df::folder_path folder, index_state& index) +{ + df::index_roots roots; + roots.folders.emplace(folder); + index.index_roots(roots); + // No database: nothing more is coming from a cache, as on a first run. + index.cache_load_complete(); + index.index_folders(test_token); + index.scan_uncached(test_token); +} + +static uint64_t count_found(index_state& index, const std::string_view query) +{ + return index.count_matches(df::search_t::parse(query), test_token).total_items().count; +} + +// The trigram index only suggests which values may hold a term. Every three letters of "ananan" are in +// "banana", which does not hold it, so a suggestion the matcher's own predicate does not confirm would +// be found by the search index and not by a scan. +static void should_confirm_what_the_trigram_index_suggests() +{ + const auto folder = _temps.next_folder("search-index-trigrams"); + platform::copy_file(test_files_folder.combine_file("Test.jpg"), folder.combine_file("banana.jpg"), false, true); + + null_async_strategy as; + const location_cache locations; + index_state index(as, locations); + make_search_index_collection(folder, index); + index.update_search_index(); + assert_equal(true, index.search_index() != nullptr, "the collection was read into a search index"); + + const auto count = [&index](const std::string_view query, const index_state::search_execution execution) + { + return index.count_matches(df::search_t::parse(query), test_token, execution).total_items().count; + }; + + const auto answers_before = df::query_perf.index_answers.load(); + assert_equal(uint64_t{1}, count("anana", index_state::search_execution::automatic), "what the name holds is found"); + assert_equal(uint64_t{0}, count("ananan", index_state::search_execution::scan), "a scan finds no item holding the term"); + assert_equal(uint64_t{0}, count("ananan", index_state::search_execution::automatic), + "nor does the search index, though the name holds each of the term's trigrams"); + assert_equal(true, df::query_perf.index_answers.load() > answers_before, "the search index answered"); +} + +// A folder that changed after the search index read it is asked item by item, so a search answers with +// what the folder holds now, without the collection being read again for a change this small. +static void should_scan_folders_changed_since_the_search_index_was_read() +{ + const auto folder = _temps.next_folder("search-index-changed"); + const auto settled = folder.combine("settled"); + const auto changed = folder.combine("changed"); + platform::create_folder(settled); + platform::create_folder(changed); + + for (const auto name : {"alpha.jpg", "charlie.jpg", "delta.jpg"}) + { + platform::copy_file(test_files_folder.combine_file("Test.jpg"), settled.combine_file(name), false, true); + } + + platform::copy_file(test_files_folder.combine_file("Test.jpg"), changed.combine_file("echo.jpg"), false, true); + + null_async_strategy as; + const location_cache locations; + index_state index(as, locations); + make_search_index_collection(folder, index); + index.update_search_index(); + + const auto read = index.search_index(); + assert_equal(true, read != nullptr, "the collection was read into a search index"); + assert_equal(uint64_t{1}, count_found(index, "alpha"), "a file the index read is found"); + assert_equal(uint64_t{0}, count_found(index, "bravo"), "a file not copied yet is not"); + + platform::copy_file(test_files_folder.combine_file("Test.jpg"), changed.combine_file("bravo.jpg"), false, true); + index.validate_folder(changed, true, platform::now()); + index.update_search_index(); + + const auto answers_before = df::query_perf.index_answers.load(); + assert_equal(true, index.search_index() == read, "so small a change is not worth reading the collection again"); + assert_equal(uint64_t{1}, count_found(index, "bravo"), "a file added since is found in its changed folder"); + assert_equal(uint64_t{1}, count_found(index, "echo"), "and the file already there still is"); + assert_equal(uint64_t{1}, count_found(index, "alpha"), "as is one in a folder that did not change"); + assert_equal(uint64_t{5}, count_found(index, "@photo"), "a changed folder is counted once"); + assert_equal(true, df::query_perf.index_answers.load() > answers_before, "the search index answered"); + + const auto search = df::search_t::parse("@photo"); + assert_equal(describe_query_items(index, search, index_state::search_execution::scan), + describe_query_items(index, search, index_state::search_execution::automatic), + "what a changed folder holds is found as a scan finds it"); +} + +// The tolerance for change grows with the collection, and in a small one can pass the point where most +// of it has changed. A snapshot no search would use is read again however few items changed, or the +// counts the sidebar and the address box take from it would be gone until the next restart. +static void should_read_the_search_index_again_once_no_search_would_use_it() +{ + const auto folder = _temps.next_folder("search-index-reread"); + platform::copy_file(test_files_folder.combine_file("Test.jpg"), folder.combine_file("alpha.jpg"), false, true); + + null_async_strategy as; + const location_cache locations; + index_state index(as, locations); + make_search_index_collection(folder, index); + index.update_search_index(); + + const auto read = index.search_index(); + assert_equal(true, read != nullptr, "the collection was read into a search index"); + + platform::copy_file(test_files_folder.combine_file("Test.jpg"), folder.combine_file("bravo.jpg"), false, true); + index.validate_folder(folder, true, platform::now()); + assert_equal(false, search_counter(index).is_indexed(), "no search uses a snapshot of a collection that mostly changed"); + + index.update_search_index(); + assert_equal(true, index.search_index() != read, "so the collection is read again, though only one item changed"); + assert_equal(true, search_counter(index).is_indexed(), "and searches are answered from the index again"); + assert_equal(uint64_t{2}, count_found(index, "@photo"), "with what the collection holds now"); +} + +// Recording a checksum, a picture hash or a duplicate set changes what is derived from an item's bytes, +// which the search index reads live, so the startup pass that finds every set leaves the collection's +// read current; the folders it touched were being read again after every pass. A change to what an item +// is still makes its folder one the index cannot answer for. +static void should_keep_the_search_index_current_through_derived_changes() +{ + const auto folder = _temps.next_folder("search-index-derived"); + const auto first = folder.combine_file("first.jpg"); + const auto second = folder.combine_file("second.jpg"); + platform::copy_file(test_files_folder.combine_file("Test.jpg"), first, false, true); + platform::copy_file(test_files_folder.combine_file("Test.jpg"), second, false, true); + + // Deferred, so the passes indexing queues run after the collection is read, as they do at startup. + deferred_async_strategy as; + const location_cache locations; + index_state index(as, locations); + make_search_index_collection(folder, index); + index.update_search_index(); + assert_equal(true, search_counter(index).is_indexed(), "the collection was read into a search index"); + assert_equal(0u, index.find_item(first).duplicates.load().group, "and its copies are not a set yet"); + + index.update_predictions(); + assert_equal(true, index.find_item(first).duplicates.load().group != 0, "the copies form a set"); + assert_equal(true, search_counter(index).is_indexed(), "finding the set leaves the read current"); + assert_equal(uint64_t{2}, count_found(index, "@duplicates"), "and the set is found through it"); + + index.save_crc(first, revision_of(index.find_item(first)), 0x1234u); + index.save_phash(first, revision_of(index.find_item(first)), crypto::phash_rotations{crypto::phash_declined, 0, 0, 0}); + while (as.run_next(async_queue::work)) + { + } + assert_equal(0x1234u, index.find_item(first).crc32c.load(), "the checksum is recorded"); + assert_equal(true, search_counter(index).is_indexed(), "so does recording a checksum or a picture hash"); + + index.save_media_position(first, 12.5); + assert_equal(false, search_counter(index).is_indexed(), "a change to an item's metadata does not"); +} + +// Indexing changes folders faster than a snapshot could follow and needs the CPU a build would take, so +// nothing is read while it runs; searches scan meanwhile, as they always have. +static void should_not_read_the_search_index_while_indexing() +{ + const auto folder = _temps.next_folder("search-index-indexing"); + platform::copy_file(test_files_folder.combine_file("Test.jpg"), folder.combine_file("alpha.jpg"), false, true); + + null_async_strategy as; + const location_cache locations; + index_state index(as, locations); + make_search_index_collection(folder, index); + + index.indexing = 1; + index.update_search_index(); + assert_equal(true, index.search_index() == nullptr, "nothing is read while indexing runs"); + assert_equal(uint64_t{1}, count_found(index, "alpha"), "searches scan meanwhile"); + + index.indexing = 0; + index.update_search_index(); + const auto read = index.search_index(); + assert_equal(true, read != nullptr, "the collection is read once indexing stops"); + + index.update_search_index(); + assert_equal(true, index.search_index() == read, "an unchanged collection is not read again"); +} + +// Until the collection is read into a search index the sidebar counts from summaries, and while a read is +// too stale to use it counts by scanning, so every snapshot published asks it to count again and make +// each total what clicking its row finds. Only the counts: a snapshot changes no row, and rebuilding the +// rows would discard every text layout to publish numbers. +static void should_recount_the_sidebar_when_the_search_index_is_read() +{ + class recording_async_strategy final : public null_async_strategy + { + public: + view_invalid invalids = view_invalid::none; + + void invalidate_view(const view_invalid invalid) override + { + invalids |= invalid; + } + }; + + const auto folder = _temps.next_folder("search-index-recount"); + platform::copy_file(test_files_folder.combine_file("Test.jpg"), folder.combine_file("alpha.jpg"), false, true); + + recording_async_strategy as; + const location_cache locations; + index_state index(as, locations); + make_search_index_collection(folder, index); + assert_equal(true, index.search_index() == nullptr, "the collection has not been read yet"); + + as.invalids = view_invalid::none; + index.update_search_index(); + assert_equal(true, index.search_index() != nullptr, "the collection was read into a search index"); + assert_equal(true, (as.invalids & view_invalid::sidebar_counts) != view_invalid::none, + "the sidebar counts again from the search index"); + assert_equal(true, (as.invalids & view_invalid::sidebar) == view_invalid::none, "its rows are not rebuilt"); + + as.invalids = view_invalid::none; + index.update_search_index(); + assert_equal(true, as.invalids == view_invalid::none, "an unchanged collection asks for nothing"); + + // One item of two changed, so no search uses the read until the collection is read again. + platform::copy_file(test_files_folder.combine_file("Test.jpg"), folder.combine_file("bravo.jpg"), false, true); + index.validate_folder(folder, true, platform::now()); + as.invalids = view_invalid::none; + index.update_search_index(); + assert_equal(true, (as.invalids & view_invalid::sidebar_counts) != view_invalid::none, + "a collection read again asks the sidebar to count again"); +} + +// The sidebar and the address box count many searches at once through one counter, which must count +// what count_matches counts, never scan a tree to predict, and say how many of a current result set +// each search keeps. +static void should_count_searches_with_a_search_counter(shared_test_context& stc) +{ + stc.lazy_load_index(); + auto& index = stc.test_index; + + search_counter counter(index); + assert_equal(true, counter.is_indexed(), "the counter answers from the search index"); + + for (const auto query : {"@photo", "key1", "#key1", "dog or cat", "-@video", "2012-09-14", "rating:-1"}) + { + const auto search = df::search_t::parse(query); + const auto expected = index.count_matches(search, test_token, index_state::search_execution::scan); + assert_equal(true, counter.count(search, test_token) == expected, std::format("'{}' counted", query)); + + const auto items = counter.try_count_items(search, test_token); + assert_equal(true, items.has_value(), std::format("'{}' predicted", query)); + assert_equal(expected.total_items().count, items.value_or(0), std::format("'{}' predicted count", query)); + } + + const auto tree = df::search_t().add_selector(df::item_selector(test_files_folder, true)); + assert_equal(false, counter.try_count_items(tree, test_token).has_value(), "a tree is never scanned to predict"); + + // One counter answers its searches through one evaluator, which keeps what they share - a term's + // documents, the documents through a presence requirement - so a search must count what a scan counts + // whatever was asked before it. The second pass answers from what the first kept. + for (auto pass = 0; pass < 2; ++pass) + { + for (const auto query : { + "@duplicates", "@photo", "@video", "test", "**test**", "-test", "dog", "dog cat", "-dog", + "dog or cat", "Tag:dog", "#dog", "rating:5", "-rating:5", "label:select", "rating:5 label:select", + "2012", "-2012", "dec-25", "Created:2012-09-14", "with:tag", "without:tag", "@photo dog" + }) + { + const auto search = df::search_t::parse(query); + const auto expected = index.count_matches(search, test_token, index_state::search_execution::scan); + assert_equal(true, counter.count(search, test_token) == expected, + std::format("'{}' counted after the searches before it, pass {}", query, pass)); + } + } + + std::vector photos; + index.query_items(df::search_t::parse("@photo"), [&photos](const index_state::query_item_results& items, bool) + { + for (const auto& item : items) if (item.kind == index_state::query_item_kind::file) photos.emplace_back(item.path); + }, test_token); + + const auto kept = counter.make_kept_items(photos); + + const auto count_in_photos = [&index, &photos](const df::search_t& search) + { + df::unique_paths found; + index.query_items(search, [&found](const index_state::query_item_results& items, bool) + { + for (const auto& item : items) if (item.kind == index_state::query_item_kind::file) found.emplace(item.path); + }, test_token, index_state::search_execution::scan); + + return static_cast(std::ranges::count_if(photos, [&found](const df::file_path& path) + { + return found.contains(path); + })); + }; + + for (const auto query : {"key1", "dog", "@video", "-dog", "@photo"}) + { + const auto search = df::search_t::parse(query); + assert_equal(count_in_photos(search), counter.count_kept(search, kept, test_token).value_or(UINT64_MAX), + std::format("photos '{}' keeps", query)); + } + + // Only walking a folder, reading what of it has not been read, could say what a folder search keeps + // or how many of a folder's items a term finds, so neither is answered while counting. + const auto folder = df::search_t().add_selector(df::item_selector(test_files_folder, false)); + assert_equal(false, counter.count_kept(folder, kept, test_token).has_value(), "a folder search has no kept count"); + assert_equal(true, counter.try_count_items(folder, test_token).has_value(), "a folder is predicted from the index"); + + auto folder_and_term = folder; + folder_and_term.with(prop::tag, "dog"); + assert_equal(false, counter.try_count_items(folder_and_term, test_token).has_value(), + "a folder and a term is not, for it would read the folder's unread files"); +} + static void should_materialize_detached_query_item() { null_async_strategy async; @@ -5206,6 +5669,21 @@ void register_index_tests(view_state& state, test_registry& tests) tests.add("Should encode postings"s, should_encode_postings); tests.add("Should query inverted index"s, should_query_inverted_index); tests.add("Should query trigram index"s, should_query_trigram_index); + tests.add("Should hold document sets"s, should_hold_document_sets); + tests.add("Should combine sets as the matcher combines answers"s, should_combine_sets_as_the_matcher_combines_answers); + tests.add("Should answer searches from the search index as a scan does"s, + should_answer_searches_from_the_search_index_as_a_scan_does); + tests.add("Should confirm what the trigram index suggests"s, should_confirm_what_the_trigram_index_suggests); + tests.add("Should scan folders changed since the search index was read"s, + should_scan_folders_changed_since_the_search_index_was_read); + tests.add("Should read the search index again once no search would use it"s, + should_read_the_search_index_again_once_no_search_would_use_it); + tests.add("Should not read the search index while indexing"s, should_not_read_the_search_index_while_indexing); + tests.add("Should keep the search index current through derived changes"s, + should_keep_the_search_index_current_through_derived_changes); + tests.add("Should recount the sidebar when the search index is read"s, + should_recount_the_sidebar_when_the_search_index_is_read); + tests.add("Should count searches with a search counter"s, should_count_searches_with_a_search_counter); tests.add("Should materialize detached query item"s, should_materialize_detached_query_item); tests.add("Should batch thumbnail publication"s, should_batch_thumbnail_publication); // SRC-039 - folder-thumbnail traversal must charge every admitted child against the budget. diff --git a/src/ui_dialog.h b/src/ui_dialog.h index 161d47f7..6b3f648c 100644 --- a/src/ui_dialog.h +++ b/src/ui_dialog.h @@ -89,6 +89,11 @@ namespace ui { public: int weight = 0; + // How many items choosing this finds, when that could be said cheaply; drawn beside it. + std::optional count; + // Drawn from the collection's own words, tags and places, so a count of none means it leads + // nowhere and it is not offered. + bool drop_when_empty = false; auto_complete_match() noexcept = default; diff --git a/src/util_interfaces.h b/src/util_interfaces.h index c1bf71b8..005459ae 100644 --- a/src/util_interfaces.h +++ b/src/util_interfaces.h @@ -164,6 +164,9 @@ enum class async_queue index_predictions_single, index_summary_single, index_presence_single, + // Reads the collection into the search index. Its own thread, so a build never waits behind, or + // holds up, the indexing, search, sidebar and summary queues. + index_search_single, web, map_tile, };