Preserve private cache properties during metadata updates

docs/implementation.md

@@ -152,6 +152,13 @@ Startup loads cached records, validates collection roots, scans stale files, que
 and rebuilds summaries, duplicate predictions, presence, and search candidates. Cached results may
 publish before extraction completes.
 
+The existing `item_properties.properties` blob stores versioned binary property records. Private
+cache IDs are separate from the public media-property registry, so opaque values such as the face
+cache never become EXIF/XMP fields or UI metadata. Metadata-only writes preserve unknown records
+from the same item's previous blob; binary-property edits preserve all other records byte for byte.
+No extra table, column, or schema version is needed. The format and private IDs live in
+[model_db_pack.h](../src/model_db_pack.h) and [model_property.h](../src/model_property.h).
+
 Search parses into exact matchers and conservative candidate prefilters. Negation and OR never create
 a prefilter that could omit an exact result. Query generations prevent stale result publication.
 Folder-scoped searches validate their filesystem scope; index-wide searches answer from cached

src/model_db.cpp

@@ -208,7 +208,7 @@ class db_statement
 		}
 	}
 
-	bool read() const
+	bool read(const bool checked = false) const
 	{
 		if (_handle != nullptr)
 		{
@@ -222,12 +222,24 @@ class db_statement
 				return false;
 			default:
 				db_trace_error(_db, "sqlite3_step");
+				if (checked)
+					throw app_exception(std::format("Failed to read cached properties: {}",
+					                                str::utf8_cast(sqlite3_errmsg(_db))));
 			}
 		}
+		else if (checked)
+		{
+			throw app_exception("Cached-property query is not prepared"s);
+		}
 
 		return false;
 	}
 
+	bool read_checked() const
+	{
+		return read(true);
+	}
+
 	void exec() const
 	{
 		while (read())
@@ -288,6 +300,18 @@ class db_statement
 		return {data_ptr, static_cast<size_t>(sqlite3_column_bytes(_handle, i))};
 	}
 
+	df::cspan data_checked(const int i) const
+	{
+		if (!_handle) throw app_exception("Cached-property query is not prepared"s);
+		const auto value = data(i);
+		if (!value.data && sqlite3_errcode(_db) == SQLITE_NOMEM)
+		{
+			db_trace_error(_db, "sqlite3_column_blob");
+			throw app_exception("Failed to read cached-property bytes"s);
+		}
+		return value;
+	}
+
 	// Distinguishes a column an upgrade added but never filled from one holding zero, which the
 	// typed readers cannot: both answer 0.
 	bool is_null(const int i) const
@@ -1703,6 +1727,8 @@ void database::perform_writes(std::deque<item_db_write> writes) const
 	const db_statement update_metadata_scanned(
 		_db,
 		"update item_properties set last_scanned = ?, last_indexed = ? where folder = ? and name = ?"s);
+	const db_statement previous_properties(
+		_db, "select properties from item_properties where folder=? and name=?"s);
 	const db_statement update_hash(_db, "update item_properties set hash = ? where folder=? and name=?"s);
 	const db_statement update_crc(_db, "update item_properties set crc = ? where folder=? and name=?"s);
 	const db_statement update_phash(
@@ -1734,15 +1760,21 @@ void database::perform_writes(std::deque<item_db_write> writes) const
 		{
 			const auto md = write.md.value();
 
-			// A null metadata pointer means "this item has no properties". The packed payload
-			// must then be an empty record, not whatever the previous item left in the packer.
+			// Reset public metadata even for a null pointer; opaque cache values belong to this item,
+			// not to the metadata snapshot or to the item the packer previously wrote.
 			packer.reset_to_header();
 
 			if (md)
 			{
 				packer.pack(md);
 			}
 
+			previous_properties.bind(1, path.folder().text());
+			previous_properties.bind(2, path.name());
+			if (previous_properties.read_checked())
+				metadata_preserve_unknown_properties(packer, previous_properties.data_checked(0));
+			previous_properties.reset();
+
 			insert_properties.bind(1, path.folder().text());
 			insert_properties.bind(2, path.name());
 			insert_properties.bind(3, packer.cdata());

src/model_db_pack.h

@@ -6,12 +6,17 @@
 // License details are available at https://www.gnu.org/licenses/lgpl-2.1.html
 // This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY
 
-// Purpose: Metadata serialization for database storage. Packs and unpacks item
-// metadata into compact binary format for efficient SQLite storage.
+// Purpose: Database property serialization: public metadata and opaque, database-only cache values.
 
 #pragma once
 
 
+class metadata_property_error final : public std::runtime_error
+{
+public:
+	using std::runtime_error::runtime_error;
+};
+
 class metadata_packer
 {
 public:
@@ -70,6 +75,29 @@ class metadata_packer
 		}
 	}
 
+	void write_blob(const uint16_t id, df::cspan value)
+	{
+		if (value.size > std::numeric_limits<uint32_t>::max() || (value.size && !value.data))
+			throw metadata_property_error("Invalid binary cache-property value");
+
+		df::blob owned;
+		if (value.size && !_data.empty())
+		{
+			const std::less<const uint8_t*> before;
+			if (!before(value.data, _data.data()) && before(value.data, _data.data() + _data.size()))
+			{
+				const auto offset = static_cast<size_t>(value.data - _data.data());
+				if (value.size > _data.size() - offset)
+					throw metadata_property_error("Binary cache-property value exceeds its buffer");
+				owned.assign(value.data, value.data + value.size);
+				value = owned;
+			}
+		}
+		write_prop_id(id);
+		write_len(value.size);
+		if (value.size) _data.insert(_data.end(), value.data, value.data + value.size);
+	}
+
 	template <typename T>
 	void write(const uint16_t id, const T& v)
 	{
@@ -168,7 +196,7 @@ class metadata_unpacker
 public:
 	metadata_unpacker(const df::cspan data) : _data(data)
 	{
-		if (_data.size >= 2 && _data.data[0] == 0xFF)
+		if (_data.data && _data.size >= 2 && _data.data[0] == 0xFF)
 		{
 			_version = _data.data[1];
 		}
@@ -187,17 +215,22 @@ class metadata_unpacker
 		return (_version == 0 || _version > metadata_packer::version) || remaining() < 2;
 	}
 
-	// The property for the next record, or prop::null when this build does not know the id.
-	prop::key_ref read_type()
+	uint16_t read_id()
 	{
 		if (at_end())
 		{
-			return prop::null;
+			return 0;
 		}
 
 		auto id = static_cast<uint16_t>(_data.data[_pos++]);
 		id |= static_cast<uint16_t>(_data.data[_pos++] << 8);
-		return prop::from_id(id);
+		return id;
+	}
+
+	// The property for the next record, or prop::null when this build does not know the id.
+	prop::key_ref read_type()
+	{
+		return at_end() ? prop::null : prop::from_id(read_id());
 	}
 
 	size_t read_len()
@@ -223,6 +256,34 @@ class metadata_unpacker
 		return result;
 	}
 
+	struct property_record
+	{
+		uint16_t id;
+		df::cspan value;
+		df::cspan encoded;
+	};
+
+	// Borrowed views of complete records, for lossless cache-property edits without unpacking metadata.
+	std::optional<property_record> read_record()
+	{
+		if (_data.size == 0) return std::nullopt;
+		if (_version == 0 || _version > metadata_packer::version)
+			throw metadata_property_error("Unsupported cached-property header");
+		if (remaining() == 0) return std::nullopt;
+		if (remaining() < 3) throw metadata_property_error("Truncated cached-property record");
+
+		const auto start = _pos;
+		const auto id = read_id();
+		const auto marker = _data.data[_pos];
+		const auto length_bytes = marker == 0xff ? 3u : marker == 0xfe ? 5u : 1u;
+		if (remaining() < length_bytes) throw metadata_property_error("Truncated cached-property length");
+		const auto length = read_len();
+		if (length > remaining()) throw metadata_property_error("Truncated cached-property value");
+		const df::cspan value{_data.data + _pos, length};
+		_pos += length;
+		return property_record{id, value, {_data.data + start, _pos - start}};
+	}
+
 	template <typename T>
 	void read_val(T& v)
 	{
@@ -380,3 +441,49 @@ class metadata_unpacker
 
 	void unpack(const prop::item_metadata_ptr& md);
 };
+
+inline std::optional<df::cspan> metadata_find_blob_property(const df::cspan data, const uint16_t id)
+{
+	metadata_unpacker reader(data);
+	std::optional<df::cspan> result;
+	while (const auto record = reader.read_record())
+	{
+		if (record->id == id) result = record->value;
+	}
+	return result;
+}
+
+inline df::blob metadata_replace_blob_property(const df::cspan data, const uint16_t id,
+                                               const std::optional<df::cspan> value)
+{
+	metadata_packer result;
+	metadata_unpacker reader(data);
+	while (const auto record = reader.read_record())
+	{
+		if (record->id != id)
+			result._data.insert(result._data.end(), record->encoded.data,
+			                    record->encoded.data + record->encoded.size);
+	}
+	if (value) result.write_blob(id, *value);
+	return std::move(result._data);
+}
+
+inline void metadata_preserve_unknown_properties(metadata_packer& destination, const df::cspan previous)
+{
+	const auto original_size = destination.size();
+	try
+	{
+		metadata_unpacker reader(previous);
+		while (const auto record = reader.read_record())
+		{
+			if (prop::from_id(record->id) == prop::null)
+				destination._data.insert(destination._data.end(), record->encoded.data,
+				                         record->encoded.data + record->encoded.size);
+		}
+	}
+	catch (const metadata_property_error& error)
+	{
+		destination._data.resize(original_size);
+		df::log(__FUNCTION__, std::format("Invalid cached properties replaced by fresh metadata: {}", error.what()));
+	}
+}

src/model_property.h

@@ -95,6 +95,12 @@ namespace prop
 	struct item_metadata;
 	class key;
 
+	// Database-only properties, deliberately absent from the public media-property registry.
+	namespace cache
+	{
+		constexpr uint16_t faces = 0xfffe;
+	}
+
 	// The projection a file's own GPano metadata declares. Persisted in the index, so a value's
 	// meaning is fixed once assigned: retire one by leaving its number unused, never by reusing it.
 	// `none` is the answer for a file carrying no panorama projection metadata at all, so anything