From 6fbb37f6a2aaf4cf79011a74a17dfbbb3d832017 Mon Sep 17 00:00:00 2001 From: eldadfux Date: Tue, 3 Nov 2026 09:16:41 +0000 Subject: [PATCH 01/10] fix(security): keep attribute names off the storage columns The adapters filter() an attribute a caller names into its column, and filter() keeps underscores, so a name such as `_tenant` reached a column the storage keeps for the document itself: - upsertDocuments(increase:) never checked its attribute, on any adapter. Increasing `_tenant` added the tenant to itself, which under shared tables moved the stored document into another tenant's rows; increasing `_createdAt` rewrote when it was created. - increaseDocumentAttribute() and decreaseDocumentAttribute() checked the attribute only when attributes are defined, so in a schemaless MongoDB collection they moved a document to another tenant the same way. - A schemaless MongoDB document took any key: `_id` chose its $sequence, `_tenant` was stored as its tenant without shared tables, and updateDocuments() wrote `_permissions`, `_uid` and `_createdAt` past the permission and structure checks. Storage::isReserved() names those columns. The attribute an upsert increases is now checked as an increase's is: a declared numeric attribute when attributes are defined, and never a reserved column without them (Exception\NotFound). Structure refuses a reserved key in a schemaless document ("Reserved attribute"). The SQL and MongoDB adapters refuse to increase a reserved column, and MongoDB drops reserved keys from every record it writes, so the rule holds with validation skipped. Co-Authored-By: Claude Opus 5.5 --- src/Database/Adapter.php | 18 +++++++ src/Database/Adapter/Mongo.php | 34 +++++++------ src/Database/Adapter/SQL.php | 6 +-- src/Database/Adapter/SQLite.php | 2 +- src/Database/Storage.php | 9 ++++ src/Database/Trait/Documents.php | 11 +++- src/Database/Validator/Structure.php | 11 ++++ tests/e2e/Adapter/Scopes/DocumentTests.php | 50 ++++++++++++++++++ tests/e2e/Adapter/Scopes/SchemalessTests.php | 53 ++++++++++++++++++++ tests/unit/StorageTest.php | 11 ++++ tests/unit/Validator/StructureTest.php | 22 ++++++++ 11 files changed, 208 insertions(+), 19 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index f44429644..ab968f0f0 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -994,6 +994,24 @@ public function filter(string $value): string return self::$filteredKeyCache[$value] = $filtered; } + /** + * The column an attribute a caller names is written to. filter() keeps underscores, so a name such as `_tenant` + * or `_ten.ant` would reach a column the storage keeps for the document itself and move the document to another + * tenant, or rewrite its id or timestamps. + * + * @throws NotFoundException When the attribute names a reserved column + */ + protected function writableColumn(string $attribute): string + { + $column = $this->filter($attribute); + + if (Storage::isReserved($column)) { + throw new NotFoundException('Attribute not found'); + } + + return $column; + } + /** * The row as every write hook decorates a row written for the document. * diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index fdc80a2e1..4fc8626f2 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -1616,9 +1616,7 @@ public function createDocument(Document $collection, Document $document): Docume $document->removeAttribute(Document::SEQUENCE); - /** @var array $documentArray */ - $documentArray = (array) $document; - $record = $this->replaceCharacters('$', '_', $documentArray); + $record = $this->replaceCharacters('$', '_', $this->withoutStorageKeys((array) $document)); $record = $this->decorateRow($record, $document); if (! empty($sequence)) { @@ -1663,9 +1661,7 @@ public function createDocuments(Document $collection, array $documents): array throw new DatabaseException('All documents must have an sequence if one is set'); } - /** @var array $documentArr */ - $documentArr = (array) $document; - $record = $this->replaceCharacters('$', '_', $documentArr); + $record = $this->replaceCharacters('$', '_', $this->withoutStorageKeys((array) $document)); $record = $this->decorateRow($record, $document); if (! empty($sequence)) { @@ -1757,8 +1753,7 @@ public function updateDocument(Document $collection, string $id, Document $docum { $name = $this->getNamespace().'_'.$this->filter($collection->getId()); - $record = $document->getArrayCopy(); - $record = $this->replaceCharacters('$', '_', $record); + $record = $this->replaceCharacters('$', '_', $this->withoutStorageKeys($document->getArrayCopy())); $filters = [Storage::UID => $id]; $filters = $this->applyTenantFilter($filters, $collection->getId()); @@ -1804,8 +1799,7 @@ public function updateDocuments(Document $collection, Document $updates, array $ $filters = $this->buildFilters($queries); $filters = $this->applyTenantFilter($filters, $collection->getId()); - $record = $updates->getArrayCopy(); - $record = $this->replaceCharacters('$', '_', $record); + $record = $this->replaceCharacters('$', '_', $this->withoutStorageKeys($updates->getArrayCopy())); try { $pipeline = $this->buildOperatorPipeline($record); @@ -2167,15 +2161,14 @@ public function upsertDocuments(Document $collection, array $changes, ?string $i try { $name = $this->getNamespace().'_'.$this->filter($collection->getId()); - $attribute = $this->filter($increase ?? ''); + $attribute = $this->writableColumn($increase ?? ''); $operations = []; $hasPipeline = false; foreach ($changes as $change) { $document = $change->new; $oldDocument = $change->old; - /** @var array $attributes */ - $attributes = $document->getAttributes(); + $attributes = $this->withoutStorageKeys($document->getAttributes()); $attributes[Storage::UID] = $document->getId(); $attributes[Storage::CREATED_AT] = $document[Document::CREATED_AT]; $attributes[Storage::UPDATED_AT] = $document[Document::UPDATED_AT]; @@ -2343,7 +2336,7 @@ public function increaseDocumentAttribute(Document $collection, string $id, stri $min = $min === null ? null : $this->normalizeAtomicNumber($min, 'minimum'); $max = $max === null ? null : $this->normalizeAtomicNumber($max, 'maximum'); - $attribute = $this->filter($attribute); + $attribute = $this->writableColumn($attribute); $current = ['$ifNull' => ['$'.$attribute, 0]]; $filters = [Storage::UID => $id]; $filters = $this->applyTenantFilter($filters, $collectionId); @@ -3277,6 +3270,19 @@ public function castDatetime(string $value): mixed return new UTCDateTime(new NativeDateTime($value)); } + /** + * A document's fields without those named like a storage field. A schemaless document holds any key, and one + * such as `_id`, `_tenant` or `_permissions` would otherwise be written over the field the storage keeps for the + * document's own $sequence, $tenant or $permissions. + * + * @param array $fields + * @return array + */ + private function withoutStorageKeys(array $fields): array + { + return \array_diff_key($fields, Storage::columnMap()); + } + /** * MongoDB field names cannot start with $ or contain dots. */ diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index ec795b30f..a80614295 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -1492,7 +1492,7 @@ public function increaseDocumentAttribute( int|float|string|null $max = null ): bool { $name = $this->filter($collection->getId()); - $attribute = $this->filter($attribute); + $attribute = $this->writableColumn($attribute); $builder = $this->newBuilder($name); $builder->setRaw($attribute, 'COALESCE('.$this->quote($attribute).', 0) + ?', [$value]); @@ -4007,7 +4007,7 @@ protected function executeUpsertBatch( $skipColumns = [Storage::UID, Storage::SEQUENCE, Storage::CREATED_AT, Storage::TENANT]; if (! empty($attribute)) { - $updateColumns = [$this->filter($attribute), Storage::UPDATED_AT]; + $updateColumns = [$this->writableColumn($attribute), Storage::UPDATED_AT]; } else { $updateColumns = \array_values(\array_filter( $allColumnNames, @@ -4019,7 +4019,7 @@ protected function executeUpsertBatch( // conflictSetRaw() takes the column names given to onConflict(); the expression methods quote their own. if (! empty($attribute)) { - $incrementColumn = $this->filter($attribute); + $incrementColumn = $this->writableColumn($attribute); if ($this->sharedTables) { $builder->conflictSetRaw($incrementColumn, $this->getConflictTenantIncrementExpression($incrementColumn)); $builder->conflictSetRaw(Storage::UPDATED_AT, $this->getConflictTenantExpression(Storage::UPDATED_AT)); diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index a887e5b10..3b5cd49b2 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -2376,7 +2376,7 @@ protected function executeUpsertBatch( if (! empty($attribute)) { $updateColumns = [ - $getUpdateClause($attribute, increment: true), + $getUpdateClause($this->writableColumn($attribute), increment: true), $getUpdateClause(Storage::UPDATED_AT), ]; } else { diff --git a/src/Database/Storage.php b/src/Database/Storage.php index d9622270b..29d87df27 100644 --- a/src/Database/Storage.php +++ b/src/Database/Storage.php @@ -71,6 +71,15 @@ public static function attribute(string $column): string return self::columnMap()[$column] ?? $column; } + /** + * Whether $column is a column the storage keeps for a document's own $id, $sequence, $tenant, $permissions or + * timestamps, which no attribute a caller names may be written to. + */ + public static function isReserved(string $column): bool + { + return isset(self::columnMap()[$column]); + } + /** * @return array */ diff --git a/src/Database/Trait/Documents.php b/src/Database/Trait/Documents.php index ceac70b12..dce955f9e 100644 --- a/src/Database/Trait/Documents.php +++ b/src/Database/Trait/Documents.php @@ -50,6 +50,7 @@ use Utopia\Database\RelationshipSide; use Utopia\Database\RelationshipType; use Utopia\Database\Rounding; +use Utopia\Database\Storage; use Utopia\Database\Validator\Authorization\Input; use Utopia\Database\Validator\BigInt; use Utopia\Database\Validator\PartialStructure; @@ -252,7 +253,8 @@ private static function relationshipAttributes(Collection $collection): array /** * The single-valued numeric attribute an increase or decrease changes; null when the adapter does not define - * attributes. + * attributes. Without defined attributes any key names a field, except one stored in a column the storage keeps + * for the document itself, such as `_tenant`. * * @throws NotFoundException * @throws TypeException @@ -260,6 +262,10 @@ private static function relationshipAttributes(Collection $collection): array private function numericAttribute(Collection $collection, string $key): ?Attribute { if (! $this->adapter->supports(Capability::DefinedAttributes)) { + if (Storage::isReserved($this->adapter->filter($key))) { + throw new NotFoundException('Attribute not found'); + } + return null; } @@ -2389,6 +2395,9 @@ private function upsert( } $collection = $this->silent(fn () => $this->getCollection($collection)); + if ($increase !== null && $increase !== '') { + $this->numericAttribute($collection, $increase); + } $documentSecurity = $collection->getAttribute('documentSecurity', false); $collectionAttributes = $collection->attributes(); $time = DateTime::now(); diff --git a/src/Database/Validator/Structure.php b/src/Database/Validator/Structure.php index b68dd73e9..51da4d8cf 100644 --- a/src/Database/Validator/Structure.php +++ b/src/Database/Validator/Structure.php @@ -12,6 +12,7 @@ use Utopia\Database\Document; use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Operator; +use Utopia\Database\Storage; use Utopia\Database\Validator\Datetime as DatetimeValidator; use Utopia\Database\Validator\Operator as OperatorValidator; use Utopia\Query\Schema\ColumnType; @@ -237,6 +238,16 @@ private function isStoredAsNull(string $key): bool protected function checkForUnknownAttributes(array $structure, array $definitions): bool { if (! $this->supportForAttributes) { + // Any key is a field without defined attributes, but one named like a column the storage keeps for the + // document itself would be written over the document's own $sequence, $tenant or $permissions. + foreach (\array_keys($structure) as $key) { + if (Storage::isReserved((string) $key)) { + $this->message = 'Reserved attribute: "'.$key.'"'; + + return false; + } + } + return true; } foreach ($structure as $key => $value) { diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index 825915476..6f619144b 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -31,6 +31,7 @@ use Utopia\Database\Exception\Conflict as ConflictException; use Utopia\Database\Exception\Duplicate as DuplicateException; use Utopia\Database\Exception\Limit as LimitException; +use Utopia\Database\Exception\NotFound as NotFoundException; use Utopia\Database\Exception\Query as QueryException; use Utopia\Database\Exception\Structure as StructureException; use Utopia\Database\Exception\Timeout as TimeoutException; @@ -49,6 +50,7 @@ use Utopia\Database\Relationship; use Utopia\Database\Role; use Utopia\Database\SetType; +use Utopia\Database\Storage; use Utopia\Database\Validator\Structure; use Utopia\Query\CursorDirection; use Utopia\Query\Method; @@ -6412,6 +6414,54 @@ public function testIncreaseArrayAttribute(): void $this->assertInstanceOf(TypeException::class, $e); } } + /** + * filter() keeps underscores, so increasing `_tenant` would move the document to another tenant and increasing + * `_createdAt` would rewrite when it was created: a column the storage keeps for the document itself is no + * attribute to change, with or without defined attributes, and an upsert cannot increase one either. + */ + public function testIncreaseOfAStorageColumnIsRefused(): void + { + /** @var Database $database */ + $database = $this->getDatabase(); + $collection = __FUNCTION__; + $database->createCollection(Collection::create( + id: $collection, + attributes: $database->getAdapter()->supports(Capability::DefinedAttributes) ? [Attribute::integer(key: 'count')] : [], + permissions: [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any())], + documentSecurity: false, + )); + $stored = $database->createDocument($collection, new Document(['$id' => 'counter', 'count' => 1])); + + foreach ([Storage::TENANT, Storage::CREATED_AT, Storage::UID, '_ten.ant'] as $column) { + try { + $database->increaseDocumentAttribute($collection, 'counter', $column); + $this->fail("Increasing {$column} must be refused"); + } catch (NotFoundException) { + } + + try { + $database->decreaseDocumentAttribute($collection, 'counter', $column); + $this->fail("Decreasing {$column} must be refused"); + } catch (NotFoundException) { + } + + if ($database->getAdapter()->hasFeature(Feature\Upserts::class)) { + try { + $database->upsertDocuments($collection, [new Document(['$id' => 'counter', 'count' => 1])], increase: $column); + $this->fail("An upsert increasing {$column} must be refused"); + } catch (NotFoundException) { + } + } + } + + $document = $database->getDocument($collection, 'counter'); + $this->assertSame('counter', $document->getId(), 'The document must stay in the current tenant'); + $this->assertSame($stored->getCreatedAt(), $document->getCreatedAt()); + $this->assertEquals(1, $document->getAttribute('count')); + + $database->deleteCollection($collection); + } + public function testIncreaseDecreasePreserveDates(): void { $document = $this->initIncreaseDecreaseFixture(); diff --git a/tests/e2e/Adapter/Scopes/SchemalessTests.php b/tests/e2e/Adapter/Scopes/SchemalessTests.php index c4dfdff4e..1f94b984d 100644 --- a/tests/e2e/Adapter/Scopes/SchemalessTests.php +++ b/tests/e2e/Adapter/Scopes/SchemalessTests.php @@ -20,6 +20,7 @@ use Utopia\Database\Permission; use Utopia\Database\Query; use Utopia\Database\Role; +use Utopia\Database\Storage; use Utopia\Query\OrderDirection; use Utopia\Query\Schema\IndexType; @@ -255,6 +256,58 @@ public function testSchemalessIncrement(): void $database->deleteCollection($colName); } + /** + * Any key is a field of a schemaless document, except one named like a field the storage keeps for the document + * itself: `_id` would choose its $sequence, `_permissions` would set its permissions past their validation and + * `_tenant` its tenant. The structure refuses such a key, and the adapter never writes one, even with validation + * skipped. + */ + public function testSchemalessDocumentCannotWriteAStorageField(): void + { + /** @var Database $database */ + $database = $this->getDatabase(); + + if ($database->getAdapter()->supports(Capability::DefinedAttributes)) { + $this->expectNotToPerformAssertions(); + + return; + } + + $colName = uniqid('schemaless_storage'); + $database->createCollection(Collection::create( + id: $colName, + permissions: [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any())], + )); + + foreach ([Storage::SEQUENCE, Storage::UID, Storage::TENANT, Storage::PERMISSIONS] as $key) { + try { + $database->createDocument($colName, new Document(['$id' => 'refused', $key => 'stored'])); + $this->fail("A document with a {$key} field must be refused"); + } catch (StructureException $e) { + $this->assertSame('Invalid document structure: Reserved attribute: "'.$key.'"', $e->getMessage()); + } + } + + $kept = $database->createDocument($colName, new Document(['$id' => 'kept', 'title' => 'kept'])); + + $database->skipValidation(function () use ($database, $colName): void { + $database->createDocument($colName, new Document(['$id' => 'skipped', Storage::SEQUENCE => 'chosen', Storage::PERMISSIONS => ['delete("any")']])); + $database->updateDocuments($colName, new Document([Storage::PERMISSIONS => ['delete("any")'], Storage::UID => 'renamed'])); + }); + + $skipped = $database->getDocument($colName, 'skipped'); + $this->assertSame('skipped', $skipped->getId()); + $this->assertNotSame('chosen', $skipped->getSequence(), 'An _id field must not choose the $sequence'); + $this->assertSame([], $skipped->getPermissions()); + + $updated = $database->getDocument($colName, 'kept'); + $this->assertSame('kept', $updated->getId(), 'A bulk update must not rename the document'); + $this->assertSame($kept->getSequence(), $updated->getSequence()); + $this->assertSame([], $updated->getPermissions(), 'A bulk update must not write the permissions field'); + + $database->deleteCollection($colName); + } + public function testSchemalessDecrement(): void { /** @var Database $database */ diff --git a/tests/unit/StorageTest.php b/tests/unit/StorageTest.php index 139bdc1f9..a16a789da 100644 --- a/tests/unit/StorageTest.php +++ b/tests/unit/StorageTest.php @@ -24,6 +24,17 @@ public function test_maps_invert(): void } } + public function test_reserved_columns(): void + { + foreach (Storage::attributeMap() as $column) { + $this->assertTrue(Storage::isReserved($column), $column); + } + + $this->assertFalse(Storage::isReserved(Document::ID)); + $this->assertFalse(Storage::isReserved('title')); + $this->assertFalse(Storage::isReserved('_title')); + } + public function test_permissions_table(): void { $this->assertSame('movies_perms', Storage::permissionsTable('movies')); diff --git a/tests/unit/Validator/StructureTest.php b/tests/unit/Validator/StructureTest.php index 1ff7fb4c3..3a96cf81a 100644 --- a/tests/unit/Validator/StructureTest.php +++ b/tests/unit/Validator/StructureTest.php @@ -12,6 +12,7 @@ use Utopia\Database\Exception; use Utopia\Database\Id; use Utopia\Database\Operator; +use Utopia\Database\Storage; use Utopia\Database\Validator\Structure; use Utopia\Query\Schema\ColumnType; @@ -343,6 +344,27 @@ public function test_unknown_keys(): void $this->assertEquals('Invalid document structure: Unknown attribute: "titlex"', $validator->getDescription()); } + public function test_reserved_keys_without_defined_attributes(): void + { + $validator = new Structure( + new Document($this->collection), + Profiles::of(capabilities: [Capability::UnsignedBigInt]), + ); + $document = [ + '$collection' => Id::custom('posts'), + 'titlex' => 'Any key is a field without defined attributes', + '$createdAt' => '2000-04-01T12:00:00.000+00:00', + '$updatedAt' => '2000-04-01T12:00:00.000+00:00', + ]; + + $this->assertTrue($validator->isValid(new Document($document)), $validator->getDescription()); + + foreach ([Storage::SEQUENCE, Storage::UID, Storage::TENANT, Storage::PERMISSIONS, Storage::CREATED_AT, Storage::COLLECTION] as $key) { + $this->assertFalse($validator->isValid(new Document([...$document, $key => 'stored over the document\'s own field'])), $key); + $this->assertSame('Invalid document structure: Reserved attribute: "'.$key.'"', $validator->getDescription()); + } + } + public function test_integer_as_string(): void { $validator = new Structure( From bf012d28eda7bd931371473bfe6bd2ada5f7eeb3 Mon Sep 17 00:00:00 2001 From: eldadfux Date: Tue, 3 Nov 2026 09:16:57 +0000 Subject: [PATCH 02/10] fix(mongo): match cursor values and $-named exists and elemMatch fields as data - A cursor matched the values of its earlier order attributes by bare equality, so a stored value shaped like {"$gt": ""} ran as that operator and the page returned rows from before the cursor. The values are now matched with $eq. - exists() and notExists() named their fields, and elemMatch() its attribute, as given, so a name such as `$comment` or `$where` was read as a top-level query operator: count() of exists(['$comment']) matched every row, and exists(['$id']) matched none. They now map $id, $sequence and the timestamps to their storage fields and escape any other $-name, as every other filter already did. Co-Authored-By: Claude Opus 5.5 --- src/Database/Adapter/Mongo.php | 40 +++++++++++++++-------------- tests/unit/MongoQueryFilterTest.php | 39 ++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 19 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 4fc8626f2..ee835ee52 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -2459,8 +2459,9 @@ public function find(Document $collection, array $queries = [], ?int $limit = 25 $originalPrev = $orderAttributes[$j]; $prevAttr = $this->filter($this->getInternalKeyForAttribute($originalPrev)); $tmp = $cursor[$originalPrev]; + // $eq, so a stored value shaped like {"$gt": ""} is compared, not run as an operator $andConditions[] = [ - $prevAttr => $tmp, + $prevAttr => ['$eq' => $tmp], ]; } @@ -3506,7 +3507,7 @@ protected function buildFilters(array $queries, string $separator = '$and'): arr /** @var array $elemMatchValues */ $elemMatchValues = $query->getValues(); $filters[$separator][] = [ - $query->getAttribute() => [ + $this->filterField($query->getAttribute()) => [ '$elemMatch' => $this->buildFilters($elemMatchValues, $separator), ], ]; @@ -3550,22 +3551,7 @@ protected function buildFilter(Query $query): array $query->setValues($values); } - if ($query->getAttribute() === Document::ID) { - $query->setAttribute(Storage::UID); - } elseif ($query->getAttribute() === Document::SEQUENCE) { - $query->setAttribute(Storage::SEQUENCE); - $values = $query->getValues(); - foreach ($values as $k => $v) { - $values[$k] = $v; - } - $query->setValues($values); - } elseif ($query->getAttribute() === Document::CREATED_AT) { - $query->setAttribute(Storage::CREATED_AT); - } elseif ($query->getAttribute() === Document::UPDATED_AT) { - $query->setAttribute(Storage::UPDATED_AT); - } elseif (\str_starts_with($query->getAttribute(), '$')) { - $query->setAttribute($this->escapeMongoFieldName($query->getAttribute())); - } + $query->setAttribute($this->filterField($query->getAttribute())); $attribute = $query->getAttribute(); $operator = $this->getQueryOperator($query->getMethod()); @@ -3684,7 +3670,7 @@ protected function buildFilter(Query $query): array $existsOr = \is_array($filter['$or'] ?? null) ? $filter['$or'] : []; foreach ($query->getValues() as $existsAttribute) { $existsAttrStr = \is_string($existsAttribute) ? $existsAttribute : (\is_scalar($existsAttribute) ? (string) $existsAttribute : ''); - $existsOr[] = [$existsAttrStr => [$operator => $value]]; + $existsOr[] = [$this->filterField($existsAttrStr) => [$operator => $value]]; } $filter['$or'] = $existsOr; } else { @@ -3697,6 +3683,22 @@ protected function buildFilter(Query $query): array return $filter; } + /** + * The stored field a filter names: $id, $sequence and the timestamps by their storage fields, and any other name + * starting with $ escaped, so it is matched as a field and never read as a query operator such as $where or + * $comment. + */ + private function filterField(string $attribute): string + { + return match ($attribute) { + Document::ID => Storage::UID, + Document::SEQUENCE => Storage::SEQUENCE, + Document::CREATED_AT => Storage::CREATED_AT, + Document::UPDATED_AT => Storage::UPDATED_AT, + default => \str_starts_with($attribute, '$') ? $this->escapeMongoFieldName($attribute) : $attribute, + }; + } + /** * @throws Exception */ diff --git a/tests/unit/MongoQueryFilterTest.php b/tests/unit/MongoQueryFilterTest.php index 045a12816..004f21c19 100644 --- a/tests/unit/MongoQueryFilterTest.php +++ b/tests/unit/MongoQueryFilterTest.php @@ -294,6 +294,45 @@ public function testDottedAttributesInsideGroupsAndExistsAreEscaped(): void $this->assertSame(['a__dot__b'], $this->fieldNames($this->calls['aggregate'][0] ?? []), 'count() must address the stored field name of a dotted attribute in every group and exists'); } + public function testDollarNamesInExistsAndElemMatchAreMatchedAsFields(): void + { + $queries = [ + Query::exists(['$comment', '$id']), + Query::elemMatch('$where', [Query::equal('n', [1])]), + ]; + + $adapter = $this->createAdapter(); + $adapter->find(new Document(['$id' => self::COLLECTION]), $queries); + $adapter->count(new Document(['$id' => self::COLLECTION]), $queries); + + foreach (['find' => $this->calls['find'][0] ?? [], 'count' => $this->calls['aggregate'][0] ?? []] as $method => $filter) { + $this->assertSame( + ['_comment', '_uid', '_where', 'n'], + $this->fieldNames($filter), + $method.'() must match a field named with a $, never run it as the query operator of that name', + ); + } + } + + public function testACursorComparesTheValuesOfItsEarlierOrdersAsValues(): void + { + $this->createAdapter()->find( + new Document(['$id' => self::COLLECTION]), + orderAttributes: ['name', '$sequence'], + orderTypes: [OrderDirection::Asc, OrderDirection::Asc], + cursor: ['name' => ['$gt' => ''], '$sequence' => 'sequence'], + ); + + $this->assertSame( + [ + ['$and' => [['name' => ['$gt' => ['$gt' => '']]]]], + ['$and' => [['name' => ['$eq' => ['$gt' => '']]], ['_id' => ['$gt' => 'sequence']]]], + ], + $this->calls['find'][0]['$or'] ?? null, + 'A stored value shaped like an operator must be compared, not run as the operator', + ); + } + public function testRandomOrderIsRejectedAsAQueryError(): void { $this->expectException(QueryException::class); From 37ad52a163e8596275a07afa1337e7848a9c9944 Mon Sep 17 00:00:00 2001 From: eldadfux Date: Tue, 3 Nov 2026 09:16:57 +0000 Subject: [PATCH 03/10] fix(sqlite): find a collection's fulltext tables by their exact name The FTS5 tables of a collection were found as `%_fts`, and the prefix of collection `notes` also starts the tables of `notes_private`. A search on `notes` could read `notes_private`'s index past its permissions, since that collection's row with the same sequence answered it, and deleting `notes` dropped `notes_private`'s tables and left their triggers failing every later write to it. The lookup now matches the prefix, exactly the 16 characters of the hashed attribute set, and the suffix. Table and index names read back from sqlite_master and PRAGMA are quoted with quote() instead of being wrapped in backticks as they are. Co-Authored-By: Claude Opus 5.5 --- src/Database/Adapter/SQLite.php | 32 ++++--- .../Adapter/SQLiteFulltextCollectionsTest.php | 85 +++++++++++++++++++ 2 files changed, 105 insertions(+), 12 deletions(-) create mode 100644 tests/unit/Adapter/SQLiteFulltextCollectionsTest.php diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index 3b5cd49b2..8f6596b53 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -73,6 +73,9 @@ class SQLite extends SQL /** Suffix appended to every FTS5 virtual table name created by this adapter. */ private const string FTS_TABLE_SUFFIX = '_fts'; + /** Length of the hashed attribute set between the FTS5 table prefix and suffix. */ + private const int FTS_TABLE_KEY_LENGTH = 16; + /** AFTER INSERT trigger suffix on the parent collection. */ private const string FTS_TRIGGER_INSERT = 'ai'; @@ -669,7 +672,7 @@ public function deleteCollection(string $collection): bool // FTS5 shadow tables don't drop with the parent. foreach ($this->findFulltextTables($id) as $ftsTable) { - $sql = "DROP TABLE IF EXISTS `{$ftsTable}`"; + $sql = 'DROP TABLE IF EXISTS '.$this->quote($ftsTable); $this->execute($this->prepare($sql, event: Event::CollectionDelete)); } @@ -935,7 +938,7 @@ protected function getFulltextTableName(string $collection, array|string $attrib { $names = \array_map($this->filter(...), \is_array($attributes) ? $attributes : [$attributes]); \sort($names); - $key = \substr(\hash('sha1', \implode("\0", $names)), 0, 16); + $key = \substr(\hash('sha1', \implode("\0", $names)), 0, self::FTS_TABLE_KEY_LENGTH); return $this->getFulltextTablePrefix($collection) . $key . self::FTS_TABLE_SUFFIX; } @@ -1105,9 +1108,9 @@ protected function dropFulltextIndexById(string $collection, string $id, Event $ $this->startTransaction(); try { foreach ($triggerSuffixes as $suffix) { - $this->execute($this->prepare("DROP TRIGGER IF EXISTS `{$ftsTable}_{$suffix}`", event: $event)); + $this->execute($this->prepare('DROP TRIGGER IF EXISTS '.$this->quote($ftsTable.'_'.$suffix), event: $event)); } - $sql = "DROP TABLE IF EXISTS `{$ftsTable}`"; + $sql = 'DROP TABLE IF EXISTS '.$this->quote($ftsTable); $this->execute($this->prepare($sql, event: $event)); $this->commitTransaction(); } catch (\Throwable $e) { @@ -1168,14 +1171,18 @@ protected function findFulltextTables(string $collection): array { // ESCAPE '\\' so the literal `_` separators in the prefix don't // act as LIKE wildcards (e.g. `db_users_` matching `db_usersA_`). + // One unescaped `_` per character of the hashed key, never `%`: the + // prefix of collection `a` also starts the tables of collection + // `a_b`, which a search on `a` would otherwise read and a delete of + // `a` would otherwise drop. $statement = $this->prepare(" SELECT name FROM sqlite_master WHERE type='table' - AND name LIKE :_prefix ESCAPE '\\' - AND name LIKE :_suffix ESCAPE '\\' + AND name LIKE :_name ESCAPE '\\' "); - $statement->bindValue(':_prefix', $this->escapeLikePattern($this->getFulltextTablePrefix($collection)) . '%'); - $statement->bindValue(':_suffix', '%' . $this->escapeLikePattern(self::FTS_TABLE_SUFFIX)); + $statement->bindValue(':_name', $this->escapeLikePattern($this->getFulltextTablePrefix($collection)) + . \str_repeat('_', self::FTS_TABLE_KEY_LENGTH) + . $this->escapeLikePattern(self::FTS_TABLE_SUFFIX)); $statement->execute(); $tables = $statement->fetchAll(PDO::FETCH_COLUMN); $statement->closeCursor(); @@ -1697,7 +1704,8 @@ protected function compileAdapterFilter(Query $query, string $collection, string ); } - $subquery = "{$quotedAlias}.{$this->quote(Storage::SEQUENCE)} IN (SELECT rowid FROM `{$ftsTable}` WHERE `{$ftsTable}` MATCH ?)"; + $quotedTable = $this->quote($ftsTable); + $subquery = "{$quotedAlias}.{$this->quote(Storage::SEQUENCE)} IN (SELECT rowid FROM {$quotedTable} WHERE {$quotedTable} MATCH ?)"; return new Expression( $method === Method::Search ? $subquery : "NOT ({$subquery})", @@ -2700,7 +2708,7 @@ public function getSchemaIndexes(string $collection): array */ private function getIndexColumns(string $index): array { - $statement = $this->prepare("PRAGMA index_info(`{$index}`)", event: Event::CollectionRead); + $statement = $this->prepare('PRAGMA index_info('.$this->quote($index).')', event: Event::CollectionRead); $this->execute($statement); $rows = $statement->fetchAll(); $statement->closeCursor(); @@ -2735,7 +2743,7 @@ protected function getFulltextSchemaIndexes(string $collection): array $indexes = []; foreach ($tables as $fulltextTable) { - $statement = $this->prepare("PRAGMA table_info(`{$fulltextTable}`)"); + $statement = $this->prepare('PRAGMA table_info('.$this->quote($fulltextTable).')'); $statement->execute(); $rows = $statement->fetchAll(PDO::FETCH_ASSOC); $statement->closeCursor(); @@ -2830,7 +2838,7 @@ private function buildFulltextAttributeMap(string $collection): array { $map = []; foreach ($this->findFulltextTables($collection) as $table) { - $info = $this->prepare("PRAGMA table_info(`{$table}`)"); + $info = $this->prepare('PRAGMA table_info('.$this->quote($table).')'); $info->execute(); $cols = $info->fetchAll(PDO::FETCH_ASSOC); $info->closeCursor(); diff --git a/tests/unit/Adapter/SQLiteFulltextCollectionsTest.php b/tests/unit/Adapter/SQLiteFulltextCollectionsTest.php new file mode 100644 index 000000000..36b88279d --- /dev/null +++ b/tests/unit/Adapter/SQLiteFulltextCollectionsTest.php @@ -0,0 +1,85 @@ +addRole(Role::any()->toString()); + + $this->database = new Database(new SQLite(new PDO('sqlite::memory:')), new Cache(new NoCache())); + $this->database + ->setDatabase(self::NAMESPACE) + ->setNamespace(self::NAMESPACE) + ->setAuthorization($authorization); + $this->database->create(); + + foreach (['notes' => Role::any(), 'notes_private' => Role::user('owner')] as $collection => $reader) { + $this->database->createCollection(Collection::create( + id: $collection, + attributes: [Attribute::string('body', size: 128)], + indexes: [Index::fulltext(key: 'body_search', attributes: ['body'])], + permissions: [Permission::create(Role::any()), Permission::read($reader)], + documentSecurity: false, + )); + } + + $this->database->createDocument('notes', new Document(['$id' => 'first', 'body' => 'a public note'])); + $this->database->createDocument('notes', new Document(['$id' => 'second', 'body' => 'another public note'])); + $this->database->createDocument('notes_private', new Document(['$id' => 'first', 'body' => 'nothing to see'])); + $this->database->createDocument('notes_private', new Document(['$id' => 'second', 'body' => 'the secret salary'])); + } + + public function testASearchReadsOnlyItsOwnCollectionsIndex(): void + { + $this->assertSame(['first', 'second'], $this->search('notes', 'public')); + $this->assertSame([], $this->search('notes', 'secret'), 'The row of notes_private with the same sequence must not answer for notes'); + } + + public function testDeletingACollectionKeepsTheIndexOfACollectionWhoseIdItStarts(): void + { + $this->database->deleteCollection('notes'); + + $this->database->createDocument('notes_private', new Document(['$id' => 'third', 'body' => 'another secret'])); + + $this->assertSame(['second', 'third'], $this->database->getAuthorization()->skip(fn () => $this->search('notes_private', 'secret'))); + } + + /** + * @return list + */ + private function search(string $collection, string $term): array + { + return \array_values(\array_map( + static fn (Document $document): string => $document->getId(), + $this->database->find($collection, [Query::search('body', $term), Query::orderAsc('$id')]), + )); + } +} From f75b6b997fa1586bba16e22c78b6b50921ee2cc7 Mon Sep 17 00:00:00 2001 From: eldadfux Date: Tue, 3 Nov 2026 09:16:57 +0000 Subject: [PATCH 04/10] fix(security): enclose raw conditions so an OR stays within the tenant A builder from Database::from() keeps every statement within the selected tenant, but a raw condition was ANDed with the tenant condition as written: whereRaw('a = ? OR b = ?') and filter([Query::raw(...)]) compiled to `_tenant IN (?) AND a = ? OR b = ?`, which read, updated and deleted another tenant's rows. The scoped builders now enclose every raw condition in parentheses. The raw conditions the adapters write themselves are enclosed too, which changes their text but not their meaning: the PostgreSQL vector statements PostgresQueryBehaviorTest pins gain the parentheses. Co-Authored-By: Claude Opus 5.5 --- src/Database/Builder/Postgres.php | 2 +- src/Database/Builder/ScopesCollections.php | 27 +++++++++ tests/unit/PostgresQueryBehaviorTest.php | 6 +- tests/unit/QueryBuilderTenancyTest.php | 64 ++++++++++++++++++++++ 4 files changed, 95 insertions(+), 4 deletions(-) diff --git a/src/Database/Builder/Postgres.php b/src/Database/Builder/Postgres.php index fac26ba8e..9fc9eb4ee 100644 --- a/src/Database/Builder/Postgres.php +++ b/src/Database/Builder/Postgres.php @@ -64,7 +64,7 @@ public function compileFilter(Query $query): string } } - return parent::compileFilter($query); + return $this->enclose($query, parent::compileFilter($query)); } /** diff --git a/src/Database/Builder/ScopesCollections.php b/src/Database/Builder/ScopesCollections.php index dc02edc37..99c8dddba 100644 --- a/src/Database/Builder/ScopesCollections.php +++ b/src/Database/Builder/ScopesCollections.php @@ -4,6 +4,7 @@ use Utopia\Database\Exception\Query as QueryException; use Utopia\Query\Builder\Statement; +use Utopia\Query\Method; use Utopia\Query\Query; /** @@ -84,6 +85,32 @@ public function into(string $table): static return parent::into($table); } + /** + * The scope's tenant condition is ANDed with every raw condition, so a raw condition is enclosed in parentheses: + * an OR in it would otherwise bind looser than that AND and read or change another tenant's rows. + * + * @param list $bindings + */ + #[\Override] + public function whereRaw(string $expression, array $bindings = []): static + { + return parent::whereRaw('('.$expression.')', $bindings); + } + + #[\Override] + public function compileFilter(Query $query): string + { + return $this->enclose($query, parent::compileFilter($query)); + } + + /** + * A raw filter enclosed in parentheses, for the same reason as whereRaw(). + */ + private function enclose(Query $query, string $sql): string + { + return $query->getMethod() === Method::Raw && $query->getAttribute() !== '' ? '('.$sql.')' : $sql; + } + #[\Override] public function build(): Statement { diff --git a/tests/unit/PostgresQueryBehaviorTest.php b/tests/unit/PostgresQueryBehaviorTest.php index 382426b78..479f77176 100644 --- a/tests/unit/PostgresQueryBehaviorTest.php +++ b/tests/unit/PostgresQueryBehaviorTest.php @@ -119,7 +119,7 @@ public function testVectorDistanceIsProjectedHydratedAndOrderedBeforeTieBreaker( $this->assertCount(1, $documents); $this->assertSame(0.25, $documents[0]->getAttribute(Document::DISTANCE)); $this->assertMatchesRegularExpression('/SELECT \*, .*::text AS "'.Storage::DISTANCE.'"/', $sql); - $this->assertStringContainsString('WHERE "table_main"."embedding" IS NOT NULL', $sql); + $this->assertStringContainsString('WHERE ("table_main"."embedding" IS NOT NULL)', $sql); $this->assertMatchesRegularExpression('/ORDER BY .*<=>.*\), "'.Storage::SEQUENCE.'" ASC/', $sql); $this->assertSame([ [1, '[1,0,0]', \PDO::PARAM_STR], @@ -172,7 +172,7 @@ public function testVectorCursorComparesDistanceBeforeSequenceTieBreaker(): void ); $this->assertStringContainsString( - '"table_main"."embedding" IS NOT NULL AND ((("table_main"."embedding" <=> ?::vector)) > ? OR ((("table_main"."embedding" <=> ?::vector)) = ? AND "table_main"."'.Storage::SEQUENCE.'" > ?))', + '("table_main"."embedding" IS NOT NULL) AND (((("table_main"."embedding" <=> ?::vector)) > ? OR ((("table_main"."embedding" <=> ?::vector)) = ? AND "table_main"."'.Storage::SEQUENCE.'" > ?)))', $sql, ); $this->assertMatchesRegularExpression('/ORDER BY .*<=>.*\), "'.Storage::SEQUENCE.'" ASC/', $sql); @@ -234,7 +234,7 @@ public function testVectorCursorBeforeBindsRoundTripDistance(): void ); $this->assertStringContainsString( - '"table_main"."embedding" IS NOT NULL AND ((("table_main"."embedding" <=> ?::vector)) < ? OR ((("table_main"."embedding" <=> ?::vector)) = ? AND "table_main"."'.Storage::SEQUENCE.'" < ?))', + '("table_main"."embedding" IS NOT NULL) AND (((("table_main"."embedding" <=> ?::vector)) < ? OR ((("table_main"."embedding" <=> ?::vector)) = ? AND "table_main"."'.Storage::SEQUENCE.'" < ?)))', $sql, ); $this->assertMatchesRegularExpression('/ORDER BY .*<=>.*\) DESC, "'.Storage::SEQUENCE.'" DESC/', $sql); diff --git a/tests/unit/QueryBuilderTenancyTest.php b/tests/unit/QueryBuilderTenancyTest.php index 8e1775c2f..3e185fe05 100644 --- a/tests/unit/QueryBuilderTenancyTest.php +++ b/tests/unit/QueryBuilderTenancyTest.php @@ -242,6 +242,50 @@ public function testARawDeleteRemovesOnlyTheSelectedTenantsRows(): void ); } + /** + * The tenant condition is ANDed with a raw condition, and AND binds tighter than an OR inside it: unless the raw + * condition is enclosed, `name = 'one-a1' OR name = 'two-a1'` also reads tenant two's author. + */ + public function testAnOrInARawConditionReadsOnlyTheSelectedTenantsRows(): void + { + $database = $this->shared(); + $database->setTenant(self::FIRST); + $condition = 'name = ? OR name = ?'; + $names = ['one-a1', 'two-a1']; + + $whereRaw = $database->getAuthorization()->skip( + fn () => $this->builder($database)->select(['name'])->whereRaw($condition, $names)->execute(), + ); + $filter = $database->getAuthorization()->skip( + fn () => $this->builder($database)->select(['name'])->filter([Query::raw($condition, $names)])->execute(), + ); + + $this->assertSame(['one-a1'], $this->names($whereRaw), 'whereRaw()'); + $this->assertSame(['one-a1'], $this->names($filter), 'A raw filter'); + } + + public function testARawDeleteWithAnOrRemovesOnlyTheSelectedTenantsRows(): void + { + $database = $this->shared(); + $database->setTenant(self::FIRST); + + $deleted = $database->getAuthorization()->skip( + fn () => $this->builder($database, self::REVIEWS)->whereRaw('stars = ? OR stars = ?', [5, 1])->delete()->execute(), + ); + + $this->assertSame(1, $deleted, 'Only tenant one\'s "r1" has 5 stars; tenant two\'s "r1" has 1'); + $this->assertSame( + [ + 'r1' => [self::SECOND => 1], + 'r2' => [self::FIRST => 4], + 'r3' => [self::FIRST => 3], + 'r4' => [self::FIRST => 2], + 'stale' => [self::NO_TENANT => 9], + ], + $this->stored(self::REVIEWS, 'stars'), + ); + } + public function testWithoutATenantTheBuilderReadsAndWritesNothing(): void { $database = $this->shared(); @@ -582,6 +626,26 @@ private function read(Database $database, array $joins, string $alias = ''): arr return $rows; } + /** + * The authors' names a read returned, sorted. + * + * @return list + */ + private function names(mixed $documents): array + { + $this->assertIsArray($documents); + $names = []; + foreach ($documents as $document) { + $this->assertInstanceOf(Document::class, $document); + $name = $document->getAttribute('name'); + $this->assertIsString($name); + $names[] = $name; + } + \sort($names); + + return $names; + } + /** * @param array $values * @param list $filters From 0a6c05dc40d8f8fc628e0046f3903e4d29e9c74b Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 4 Nov 2026 17:47:12 +1300 Subject: [PATCH 05/10] fix(mongo): keep $-named aliases of storage fields off every write The $ to _ swap ran after the storage keys were stripped, so a schemaless `$uid` passed the strip and landed on `_uid`: a bulk update renamed every matched document and an upsert overwrote the id it had just set. A field is now kept only when its stored name, after the swap, is not a storage field, or it is the document's own $id, $tenant, $permissions or timestamp. createDocument and createDocuments also drop those fields from the documents they return, so they never report a value that was not stored. Orders, cursors and selections now name a $-prefixed field by its stored `_` name, as filters already did, so ordering by `$rank` no longer sorts by a missing `rank` field and cursor pagination no longer ends early. Co-Authored-By: Claude Opus 5.5 --- src/Database/Adapter/Mongo.php | 47 ++++++++--- tests/e2e/Adapter/Scopes/SchemalessTests.php | 86 ++++++++++++++++++++ tests/unit/MongoQueryFilterTest.php | 46 +++++++++++ 3 files changed, 169 insertions(+), 10 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index ee835ee52..42444f5c7 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -1616,7 +1616,7 @@ public function createDocument(Document $collection, Document $document): Docume $document->removeAttribute(Document::SEQUENCE); - $record = $this->replaceCharacters('$', '_', $this->withoutStorageKeys((array) $document)); + $record = $this->replaceCharacters('$', '_', (array) $this->withoutStorageAttributes($document)); $record = $this->decorateRow($record, $document); if (! empty($sequence)) { @@ -1661,7 +1661,7 @@ public function createDocuments(Document $collection, array $documents): array throw new DatabaseException('All documents must have an sequence if one is set'); } - $record = $this->replaceCharacters('$', '_', $this->withoutStorageKeys((array) $document)); + $record = $this->replaceCharacters('$', '_', (array) $this->withoutStorageAttributes($document)); $record = $this->decorateRow($record, $document); if (! empty($sequence)) { @@ -2431,8 +2431,7 @@ public function find(Document $collection, array $queries = [], ?int $limit = 25 $sortOptions = []; foreach ($orderAttributes as $i => $originalAttribute) { - $attribute = $this->getInternalKeyForAttribute($originalAttribute); - $attribute = $this->filter($attribute); + $attribute = $this->orderField($originalAttribute); $orderType = $orderTypes[$i] ?? OrderDirection::Asc; $direction = $orderType; @@ -2457,7 +2456,7 @@ public function find(Document $collection, array $queries = [], ?int $limit = 25 $andConditions = []; for ($j = 0; $j < $i; $j++) { $originalPrev = $orderAttributes[$j]; - $prevAttr = $this->filter($this->getInternalKeyForAttribute($originalPrev)); + $prevAttr = $this->orderField($originalPrev); $tmp = $cursor[$originalPrev]; // $eq, so a stored value shaped like {"$gt": ""} is compared, not run as an operator $andConditions[] = [ @@ -3272,16 +3271,39 @@ public function castDatetime(string $value): mixed } /** - * A document's fields without those named like a storage field. A schemaless document holds any key, and one - * such as `_id`, `_tenant` or `_permissions` would otherwise be written over the field the storage keeps for the - * document's own $sequence, $tenant or $permissions. + * A document's fields without those stored over a storage field. A schemaless document holds any key, and one + * such as `_id`, `_tenant` or `_permissions`, or `$uid`, which the `$` to `_` swap stores as `_uid`, would + * otherwise be written over the field the storage keeps for the document's own $id, $sequence, $tenant or + * $permissions. * * @param array $fields * @return array */ private function withoutStorageKeys(array $fields): array { - return \array_diff_key($fields, Storage::columnMap()); + return \array_filter($fields, $this->isWritableField(...), \ARRAY_FILTER_USE_KEY); + } + + private function isWritableField(int|string $key): bool + { + if (isset(Storage::attributeMap()[$key])) { + return true; + } + + $stored = \array_key_first($this->replaceCharacters('$', '_', [$key => null])); + + return ! Storage::isReserved((string) $stored); + } + + private function withoutStorageAttributes(Document $document): Document + { + foreach (\array_keys($document->getArrayCopy()) as $key) { + if (! $this->isWritableField($key)) { + $document->removeAttribute((string) $key); + } + } + + return $document; } /** @@ -3699,6 +3721,11 @@ private function filterField(string $attribute): string }; } + private function orderField(string $attribute): string + { + return $this->filter($this->filterField($attribute)); + } + /** * @throws Exception */ @@ -3808,7 +3835,7 @@ private function getAttributeProjection(array $selections): array continue; } - $projection[$selection] = 1; + $projection[$this->filterField($selection)] = 1; } $projection[Storage::UID] = 1; diff --git a/tests/e2e/Adapter/Scopes/SchemalessTests.php b/tests/e2e/Adapter/Scopes/SchemalessTests.php index 1f94b984d..ae30c219e 100644 --- a/tests/e2e/Adapter/Scopes/SchemalessTests.php +++ b/tests/e2e/Adapter/Scopes/SchemalessTests.php @@ -308,6 +308,92 @@ public function testSchemalessDocumentCannotWriteAStorageField(): void $database->deleteCollection($colName); } + /** + * A `$`-prefixed key is stored with its `$` swapped for `_`, so `$uid` would land on the `_uid` field that holds the + * document's own $id, with validation on, since the structure only refuses the stored names themselves. + */ + public function testSchemalessDocumentCannotWriteAStorageFieldThroughItsDollarName(): void + { + /** @var Database $database */ + $database = $this->getDatabase(); + + if ($database->getAdapter()->supports(Capability::DefinedAttributes)) { + $this->expectNotToPerformAssertions(); + + return; + } + + $colName = uniqid('schemaless_storage_alias'); + $database->createCollection(Collection::create( + id: $colName, + permissions: [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any())], + )); + + $created = $database->createDocument($colName, new Document(['$id' => 'kept', '$uid' => 'created', 'title' => 'kept'])); + $this->assertSame('kept', $created->getId()); + $this->assertFalse($created->offsetExists('$uid'), 'A created document must not report a field that was never stored'); + $database->createDocument($colName, new Document(['$id' => 'other', 'title' => 'other'])); + + $database->updateDocuments($colName, new Document(['$uid' => 'renamed', 'title' => 'updated'])); + + $this->assertTrue($database->getDocument($colName, 'renamed')->isEmpty(), 'A bulk update must not rename the documents'); + foreach (['kept', 'other'] as $id) { + $updated = $database->getDocument($colName, $id); + $this->assertSame($id, $updated->getId()); + $this->assertSame('updated', $updated->getAttribute('title')); + } + + $database->upsertDocuments($colName, [ + new Document(['$id' => 'kept', '$uid' => 'moved', 'title' => 'upserted']), + new Document(['$id' => 'fresh', '$uid' => 'elsewhere', 'title' => 'fresh']), + ]); + + $this->assertTrue($database->getDocument($colName, 'moved')->isEmpty(), 'An upsert must not rename an existing document'); + $this->assertTrue($database->getDocument($colName, 'elsewhere')->isEmpty(), 'An upsert must not insert a document under another id'); + $this->assertSame('upserted', $database->getDocument($colName, 'kept')->getAttribute('title')); + $this->assertSame('fresh', $database->getDocument($colName, 'fresh')->getAttribute('title')); + $this->assertSame(3, $database->count($colName)); + + $database->deleteCollection($colName); + } + + /** + * A `$`-prefixed field is stored as `_`-prefixed, and orders, cursors and selections must read it by that name. + */ + public function testSchemalessDollarFieldIsOrderedPagedAndSelectedByItsStoredName(): void + { + /** @var Database $database */ + $database = $this->getDatabase(); + + if ($database->getAdapter()->supports(Capability::DefinedAttributes)) { + $this->expectNotToPerformAssertions(); + + return; + } + + $colName = uniqid('schemaless_dollar_field'); + $database->createCollection(Collection::create( + id: $colName, + permissions: [Permission::create(Role::any()), Permission::read(Role::any())], + )); + + foreach (['b' => 2, 'c' => 3, 'a' => 1] as $id => $rank) { + $database->createDocument($colName, new Document(['$id' => $id, '$rank' => $rank])); + } + + $ordered = $database->find($colName, [Query::orderDesc('$rank')]); + $this->assertSame(['c', 'b', 'a'], \array_map(fn (Document $document) => $document->getId(), $ordered)); + + $first = $database->getDocument($colName, 'a'); + $paged = $database->find($colName, [Query::orderAsc('$rank'), Query::cursorAfter($first)]); + $this->assertSame(['b', 'c'], \array_map(fn (Document $document) => $document->getId(), $paged)); + + $selected = $database->getDocument($colName, 'b', [Query::select(['$rank'])]); + $this->assertSame(2, $selected->getAttribute('$rank')); + + $database->deleteCollection($colName); + } + public function testSchemalessDecrement(): void { /** @var Database $database */ diff --git a/tests/unit/MongoQueryFilterTest.php b/tests/unit/MongoQueryFilterTest.php index 004f21c19..e6b0f34bb 100644 --- a/tests/unit/MongoQueryFilterTest.php +++ b/tests/unit/MongoQueryFilterTest.php @@ -333,6 +333,51 @@ public function testACursorComparesTheValuesOfItsEarlierOrdersAsValues(): void ); } + public function testDollarNamesInOrdersCursorsAndSelectionsAreTheirStoredFields(): void + { + $this->createAdapter()->find( + new Document(['$id' => self::COLLECTION]), + [Query::select(['$rank', 'title'])], + orderAttributes: ['$rank', '$sequence'], + orderTypes: [OrderDirection::Desc, OrderDirection::Asc], + cursor: ['$rank' => 2, '$sequence' => 'sequence'], + ); + + $options = $this->calls['findOptions'][0] ?? []; + + $this->assertSame(['_rank' => -1, '_id' => 1], $options['sort'] ?? null, 'A $-prefixed order must sort by the field it is stored as'); + $this->assertSame( + [ + ['$and' => [['_rank' => ['$lt' => 2]]]], + ['$and' => [['_rank' => ['$eq' => 2]], ['_id' => ['$gt' => 'sequence']]]], + ], + $this->calls['find'][0]['$or'] ?? null, + 'A cursor must compare a $-prefixed order by the field it is stored as', + ); + $projection = $options['projection'] ?? null; + $this->assertIsArray($projection); + $this->assertArrayHasKey('_rank', $projection, 'A $-prefixed selection must project the field it is stored as'); + $this->assertArrayNotHasKey('$rank', $projection); + $this->assertArrayHasKey('title', $projection); + } + + public function testABulkUpdateNeverSetsAStorageFieldByItsDollarName(): void + { + $this->createAdapter()->updateDocuments( + new Document(['$id' => self::COLLECTION]), + new Document(['$uid' => 'renamed', '_tenant' => 2, '$where' => 'kept', 'title' => 'updated']), + [new Document(['$sequence' => 'sequence'])], + ); + + $set = $this->calls['update'][0]['$set'] ?? null; + $this->assertIsArray($set); + + $this->assertArrayNotHasKey('_uid', $set, 'A $uid update must not rename the documents'); + $this->assertArrayNotHasKey('_tenant', $set); + $this->assertSame('kept', $set['_where'] ?? null, 'Any other $-prefixed field is still written'); + $this->assertSame('updated', $set['title'] ?? null); + } + public function testRandomOrderIsRejectedAsAQueryError(): void { $this->expectException(QueryException::class); @@ -474,6 +519,7 @@ public function createCollection(string $name, array $options = []): bool public function find(string $collection, array $filters = [], array $options = []): stdClass { ($this->record)('find', $filters); + ($this->record)('findOptions', $options); return (object) ['cursor' => (object) ['firstBatch' => $this->rows, 'id' => 0]]; } From 649950a6ebe9ae82ddf0b41e5a07f23aaaca42cf Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 4 Nov 2026 17:47:12 +1300 Subject: [PATCH 06/10] fix(sqlite): find a collection's fulltext tables among virtual tables only An ordinary collection named like `notes_0123456789abcdef_fts` has a table whose name matches the fulltext pattern of `notes`, so a search on notes queried it and deleting notes dropped it. Co-Authored-By: Claude Opus 5.5 --- src/Database/Adapter/SQLite.php | 1 + .../Adapter/SQLiteFulltextCollectionsTest.php | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index 8f6596b53..fb0e12f48 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -1179,6 +1179,7 @@ protected function findFulltextTables(string $collection): array SELECT name FROM sqlite_master WHERE type='table' AND name LIKE :_name ESCAPE '\\' + AND sql LIKE 'CREATE VIRTUAL TABLE %' "); $statement->bindValue(':_name', $this->escapeLikePattern($this->getFulltextTablePrefix($collection)) . \str_repeat('_', self::FTS_TABLE_KEY_LENGTH) diff --git a/tests/unit/Adapter/SQLiteFulltextCollectionsTest.php b/tests/unit/Adapter/SQLiteFulltextCollectionsTest.php index 36b88279d..3c7c85368 100644 --- a/tests/unit/Adapter/SQLiteFulltextCollectionsTest.php +++ b/tests/unit/Adapter/SQLiteFulltextCollectionsTest.php @@ -72,6 +72,28 @@ public function testDeletingACollectionKeepsTheIndexOfACollectionWhoseIdItStarts $this->assertSame(['second', 'third'], $this->database->getAuthorization()->skip(fn () => $this->search('notes_private', 'secret'))); } + public function testDeletingACollectionKeepsACollectionNamedLikeItsIndex(): void + { + $lookalike = 'notes_0123456789abcdef_fts'; + $this->database->createCollection(Collection::create( + id: $lookalike, + attributes: [Attribute::string('body', size: 128)], + permissions: [Permission::create(Role::any()), Permission::read(Role::any())], + documentSecurity: false, + )); + $this->database->createDocument($lookalike, new Document(['$id' => 'kept', 'body' => 'an ordinary row'])); + + $this->assertSame(['first', 'second'], $this->search('notes', 'public')); + + $this->database->deleteCollection('notes'); + + $this->assertSame( + ['kept'], + \array_map(static fn (Document $document): string => $document->getId(), $this->database->find($lookalike)), + 'Only an FTS5 table is an index of notes, never an ordinary table that shares the shape of its name', + ); + } + /** * @return list */ From 83c70f9c32bde15b3f28169364329f9c000efe9b Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 4 Nov 2026 17:47:12 +1300 Subject: [PATCH 07/10] fix(validator): refuse storage-column keys with defined attributes too An attribute declared as `_tenant` passed the schemaful branch, and the adapter then dropped the value on write while the caller saw it accepted. Co-Authored-By: Claude Opus 5.5 --- src/Database/Validator/Structure.php | 17 ++++++------ tests/unit/Validator/StructureTest.php | 37 ++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/src/Database/Validator/Structure.php b/src/Database/Validator/Structure.php index 51da4d8cf..261c22f53 100644 --- a/src/Database/Validator/Structure.php +++ b/src/Database/Validator/Structure.php @@ -237,19 +237,20 @@ private function isStoredAsNull(string $key): bool */ protected function checkForUnknownAttributes(array $structure, array $definitions): bool { - if (! $this->supportForAttributes) { - // Any key is a field without defined attributes, but one named like a column the storage keeps for the - // document itself would be written over the document's own $sequence, $tenant or $permissions. - foreach (\array_keys($structure) as $key) { - if (Storage::isReserved((string) $key)) { - $this->message = 'Reserved attribute: "'.$key.'"'; + // A key named like a column the storage keeps for the document itself, even one declared as an attribute, + // would be written over the document's own $sequence, $tenant or $permissions. + foreach (\array_keys($structure) as $key) { + if (Storage::isReserved((string) $key)) { + $this->message = 'Reserved attribute: "'.$key.'"'; - return false; - } + return false; } + } + if (! $this->supportForAttributes) { return true; } + foreach ($structure as $key => $value) { if (! isset($definitions[$key])) { $this->message = 'Unknown attribute: "'.$key.'"'; diff --git a/tests/unit/Validator/StructureTest.php b/tests/unit/Validator/StructureTest.php index 3a96cf81a..cc87963d4 100644 --- a/tests/unit/Validator/StructureTest.php +++ b/tests/unit/Validator/StructureTest.php @@ -365,6 +365,43 @@ public function test_reserved_keys_without_defined_attributes(): void } } + public function test_reserved_keys_with_defined_attributes(): void + { + $collection = $this->collection; + /** @var array> $attributes */ + $attributes = $collection['attributes']; + $attributes[] = [ + '$id' => Storage::TENANT, + 'type' => ColumnType::String->value, + 'size' => 64, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => [], + ]; + $collection['attributes'] = $attributes; + + $validator = new Structure( + new Document($collection), + Profiles::of(capabilities: [Capability::DefinedAttributes, Capability::UnsignedBigInt]), + ); + + $this->assertFalse($validator->isValid(new Document([ + '$collection' => Id::custom('posts'), + 'title' => 'Demo Title', + 'description' => 'Demo description', + 'rating' => 5, + 'price' => 1.99, + 'published' => true, + 'tags' => ['dog', 'cat', 'mouse'], + 'feedback' => 'team@appwrite.io', + Storage::TENANT => 'declared, but stored over the document\'s own tenant', + '$createdAt' => '2000-04-01T12:00:00.000+00:00', + '$updatedAt' => '2000-04-01T12:00:00.000+00:00', + ]))); + $this->assertSame('Invalid document structure: Reserved attribute: "'.Storage::TENANT.'"', $validator->getDescription()); + } + public function test_integer_as_string(): void { $validator = new Structure( From 0bb758d18b2e46487fc9a364725ab765e1dcc319 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 4 Nov 2026 17:47:12 +1300 Subject: [PATCH 08/10] test(postgres): cover an OR in a raw filter staying enclosed The Postgres builder overrides compileFilter(), so the SQLite raw-filter case never exercised its enclosure. Co-Authored-By: Claude Opus 5.5 --- tests/unit/QueryBuilderTenancyTest.php | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/unit/QueryBuilderTenancyTest.php b/tests/unit/QueryBuilderTenancyTest.php index 3e185fe05..168c4c8f1 100644 --- a/tests/unit/QueryBuilderTenancyTest.php +++ b/tests/unit/QueryBuilderTenancyTest.php @@ -12,6 +12,7 @@ use Utopia\Database\Adapter\SQL; use Utopia\Database\Adapter\SQLite; use Utopia\Database\Attribute; +use Utopia\Database\Builder\Postgres as PostgresBuilder; use Utopia\Database\Collection; use Utopia\Database\Database; use Utopia\Database\Document; @@ -264,6 +265,17 @@ public function testAnOrInARawConditionReadsOnlyTheSelectedTenantsRows(): void $this->assertSame(['one-a1'], $this->names($filter), 'A raw filter'); } + public function testAnOrInAPostgresRawFilterIsEnclosed(): void + { + $statement = (new PostgresBuilder()) + ->from(self::AUTHORS) + ->select(['name']) + ->filter([Query::raw('name = ? OR name = ?', ['one-a1', 'two-a1']), Query::equal(Storage::TENANT, [self::FIRST])]) + ->build(); + + $this->assertSame('SELECT "name" FROM "authors" WHERE (name = ? OR name = ?) AND "_tenant" IN (?)', $statement->query); + } + public function testARawDeleteWithAnOrRemovesOnlyTheSelectedTenantsRows(): void { $database = $this->shared(); From bc509ee6d9b5f93c0eec064983d64d8da3d7fd4d Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 4 Nov 2026 18:20:02 +1300 Subject: [PATCH 09/10] fix(mongo): increase a $-named field where it is stored writableColumn() stripped the $, so an increase of a schemaless `$rank` added to a `rank` field that does not exist while `_rank` kept its value, and an upsert with increase overwrote `_rank` with the incoming value. The name is now mapped as filters map it before the reserved-column check, which also refuses `$uid` and `$id` as increase targets. Co-Authored-By: Claude Opus 5.5 --- src/Database/Adapter/Mongo.php | 6 ++++ tests/e2e/Adapter/Scopes/SchemalessTests.php | 16 +++++++-- tests/unit/MongoQueryFilterTest.php | 37 ++++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 42444f5c7..aad6142d6 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -3726,6 +3726,12 @@ private function orderField(string $attribute): string return $this->filter($this->filterField($attribute)); } + #[\Override] + protected function writableColumn(string $attribute): string + { + return parent::writableColumn($this->filterField($attribute)); + } + /** * @throws Exception */ diff --git a/tests/e2e/Adapter/Scopes/SchemalessTests.php b/tests/e2e/Adapter/Scopes/SchemalessTests.php index ae30c219e..eaf311f09 100644 --- a/tests/e2e/Adapter/Scopes/SchemalessTests.php +++ b/tests/e2e/Adapter/Scopes/SchemalessTests.php @@ -354,11 +354,17 @@ public function testSchemalessDocumentCannotWriteAStorageFieldThroughItsDollarNa $this->assertSame('fresh', $database->getDocument($colName, 'fresh')->getAttribute('title')); $this->assertSame(3, $database->count($colName)); + $database->updateDocument($colName, 'other', new Document(['$uid' => 'renamed', 'title' => 'single'])); + + $this->assertTrue($database->getDocument($colName, 'renamed')->isEmpty(), 'A single update must not rename the document'); + $this->assertSame('single', $database->getDocument($colName, 'other')->getAttribute('title')); + $database->deleteCollection($colName); } /** - * A `$`-prefixed field is stored as `_`-prefixed, and orders, cursors and selections must read it by that name. + * A `$`-prefixed field is stored as `_`-prefixed, and orders, cursors, selections and increases must address it by + * that name. */ public function testSchemalessDollarFieldIsOrderedPagedAndSelectedByItsStoredName(): void { @@ -374,7 +380,7 @@ public function testSchemalessDollarFieldIsOrderedPagedAndSelectedByItsStoredNam $colName = uniqid('schemaless_dollar_field'); $database->createCollection(Collection::create( id: $colName, - permissions: [Permission::create(Role::any()), Permission::read(Role::any())], + permissions: [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any())], )); foreach (['b' => 2, 'c' => 3, 'a' => 1] as $id => $rank) { @@ -391,6 +397,12 @@ public function testSchemalessDollarFieldIsOrderedPagedAndSelectedByItsStoredNam $selected = $database->getDocument($colName, 'b', [Query::select(['$rank'])]); $this->assertSame(2, $selected->getAttribute('$rank')); + $database->increaseDocumentAttribute($colName, 'a', '$rank', 10); + $this->assertSame(11, $database->getDocument($colName, 'a')->getAttribute('$rank'), 'An increase must change the stored field'); + + $database->upsertDocuments($colName, [new Document(['$id' => 'c', '$rank' => 5])], increase: '$rank'); + $this->assertSame(8, $database->getDocument($colName, 'c')->getAttribute('$rank'), 'An upsert increase must add to the stored field'); + $database->deleteCollection($colName); } diff --git a/tests/unit/MongoQueryFilterTest.php b/tests/unit/MongoQueryFilterTest.php index e6b0f34bb..ebb7c1045 100644 --- a/tests/unit/MongoQueryFilterTest.php +++ b/tests/unit/MongoQueryFilterTest.php @@ -10,6 +10,7 @@ use Utopia\Database\Attribute; use Utopia\Database\Capability; use Utopia\Database\Document; +use Utopia\Database\Exception\NotFound as NotFoundException; use Utopia\Database\Exception\Query as QueryException; use Utopia\Database\Index; use Utopia\Database\Query; @@ -378,6 +379,40 @@ public function testABulkUpdateNeverSetsAStorageFieldByItsDollarName(): void $this->assertSame('updated', $set['title'] ?? null); } + public function testAnIncreaseOfADollarNameChangesTheFieldItIsStoredAs(): void + { + $this->createAdapter()->increaseDocumentAttribute( + new Document(['$id' => self::COLLECTION]), + 'record', + '$rank', + 1, + '2000-01-01 00:00:00.000', + ); + + $updates = $this->calls['query'][0]['updates'] ?? null; + $this->assertIsArray($updates); + $this->assertIsArray($updates[0]); + $this->assertIsArray($updates[0]['u']); + $this->assertIsArray($updates[0]['u'][0]); + $set = $updates[0]['u'][0]['$set'] ?? null; + $this->assertIsArray($set); + $this->assertArrayHasKey('_rank', $set, 'A $-prefixed field must be increased where it is stored'); + $this->assertArrayNotHasKey('rank', $set); + } + + public function testAnIncreaseOfADollarAliasOfAStorageFieldIsRefused(): void + { + $this->expectException(NotFoundException::class); + + $this->createAdapter()->increaseDocumentAttribute( + new Document(['$id' => self::COLLECTION]), + 'record', + '$uid', + 1, + '2000-01-01 00:00:00.000', + ); + } + public function testRandomOrderIsRejectedAsAQueryError(): void { $this->expectException(QueryException::class); @@ -589,6 +624,8 @@ public function createIndexes(string $collection, array $indexes, array $options #[\Override] public function query(array $command, ?string $db = null): stdClass { + ($this->record)('query', $command); + return (object) ['cursor' => (object) ['firstBatch' => $this->created]]; } }; From 48ac8b681f75d414d7a6b64800d952b785dfc19f Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 4 Nov 2026 18:20:02 +1300 Subject: [PATCH 10/10] fix(sqlite): take only fts5 tables as a collection's fulltext index Any virtual table named like the index, such as an rtree created through a raw query, was still dropped with the collection. Co-Authored-By: Claude Opus 5.5 --- src/Database/Adapter/SQLite.php | 2 +- .../Adapter/SQLiteFulltextCollectionsTest.php | 36 ++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index fb0e12f48..13c453b1f 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -1179,7 +1179,7 @@ protected function findFulltextTables(string $collection): array SELECT name FROM sqlite_master WHERE type='table' AND name LIKE :_name ESCAPE '\\' - AND sql LIKE 'CREATE VIRTUAL TABLE %' + AND sql LIKE 'CREATE VIRTUAL TABLE % USING fts5(%' "); $statement->bindValue(':_name', $this->escapeLikePattern($this->getFulltextTablePrefix($collection)) . \str_repeat('_', self::FTS_TABLE_KEY_LENGTH) diff --git a/tests/unit/Adapter/SQLiteFulltextCollectionsTest.php b/tests/unit/Adapter/SQLiteFulltextCollectionsTest.php index 3c7c85368..150c3e05f 100644 --- a/tests/unit/Adapter/SQLiteFulltextCollectionsTest.php +++ b/tests/unit/Adapter/SQLiteFulltextCollectionsTest.php @@ -28,13 +28,16 @@ final class SQLiteFulltextCollectionsTest extends TestCase private Database $database; + private PDO $pdo; + #[\Override] protected function setUp(): void { $authorization = new Authorization(); $authorization->addRole(Role::any()->toString()); - $this->database = new Database(new SQLite(new PDO('sqlite::memory:')), new Cache(new NoCache())); + $this->pdo = new PDO('sqlite::memory:'); + $this->database = new Database(new SQLite($this->pdo), new Cache(new NoCache())); $this->database ->setDatabase(self::NAMESPACE) ->setNamespace(self::NAMESPACE) @@ -84,9 +87,11 @@ public function testDeletingACollectionKeepsACollectionNamedLikeItsIndex(): void $this->database->createDocument($lookalike, new Document(['$id' => 'kept', 'body' => 'an ordinary row'])); $this->assertSame(['first', 'second'], $this->search('notes', 'public')); + $this->assertCount(1, $this->fulltextTables('notes')); $this->database->deleteCollection('notes'); + $this->assertSame([], $this->fulltextTables('notes'), 'Deleting notes must drop its own index'); $this->assertSame( ['kept'], \array_map(static fn (Document $document): string => $document->getId(), $this->database->find($lookalike)), @@ -94,6 +99,35 @@ public function testDeletingACollectionKeepsACollectionNamedLikeItsIndex(): void ); } + public function testDeletingACollectionKeepsAnotherVirtualTableNamedLikeItsIndex(): void + { + $lookalike = self::NAMESPACE.'_notes_0123456789abcdef_fts'; + $this->pdo->exec('CREATE VIRTUAL TABLE "'.$lookalike.'" USING rtree(id, low, high)'); + + $this->database->deleteCollection('notes'); + + $this->assertSame([], $this->fulltextTables('notes')); + $statement = $this->pdo->query("SELECT name FROM sqlite_master WHERE name = '{$lookalike}'"); + $this->assertNotFalse($statement); + $this->assertSame([$lookalike], $statement->fetchAll(PDO::FETCH_COLUMN), 'Only an FTS5 table is an index of notes'); + } + + /** + * The FTS5 tables left for a collection, by the shape of their name. + * + * @return list + */ + private function fulltextTables(string $collection): array + { + $statement = $this->pdo->query( + "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE % USING fts5(%'" + ." AND name LIKE '".self::NAMESPACE.'\\_'.$collection.'\\_'.\str_repeat('_', 16)."\\_fts' ESCAPE '\\'", + ); + $this->assertNotFalse($statement); + + return \array_values($statement->fetchAll(PDO::FETCH_COLUMN)); + } + /** * @return list */