From 38964bfbe3ee7e1e7f8bec7ed4881ee4afee061a Mon Sep 17 00:00:00 2001 From: jiatolentino Date: Thu, 22 Oct 2026 12:01:55 +0800 Subject: [PATCH] feat: DM-4388: add a client-wide default_project for creates `DataMasqueInstanceConfig.default_project` names the project a create goes into when its model has no `project`. Every create of a connection, ruleset, discovery config, RG config, table reference or library sends it, and `create_or_update_*` looks for an existing resource in it, before falling back to the server's only active project. Updating a connection leaves it in its project. --- HISTORY.rst | 5 + datamasque/client/base.py | 19 ++- datamasque/client/connections.py | 3 +- .../client/discovery_config_libraries.py | 6 +- datamasque/client/discovery_configs.py | 6 +- datamasque/client/models/connection.py | 2 +- datamasque/client/models/discovery_config.py | 2 +- .../client/models/discovery_config_library.py | 2 +- datamasque/client/models/dm_instance.py | 6 + datamasque/client/models/rg_config.py | 2 +- datamasque/client/models/ruleset.py | 2 +- datamasque/client/models/ruleset_library.py | 2 +- datamasque/client/models/table_reference.py | 2 +- datamasque/client/rg_configs.py | 6 +- datamasque/client/ruleset_libraries.py | 6 +- datamasque/client/rulesets.py | 6 +- datamasque/client/table_references.py | 3 +- tests/test_default_project.py | 137 ++++++++++++++++++ 18 files changed, 196 insertions(+), 21 deletions(-) create mode 100644 tests/test_default_project.py diff --git a/HISTORY.rst b/HISTORY.rst index 7a7a4ae..9194c17 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -23,6 +23,11 @@ History * **Breaking:** ``get_unfinished_runs`` returns its runs keyed by ``ConnectionId`` instead of connection name, because two connections in different projects can share a name. * ``list_connections``, ``list_table_references`` and ``list_files_of_type`` take an optional ``project`` filter. +* Added ``default_project`` to ``DataMasqueInstanceConfig``. + A create whose model has no ``project`` goes into ``default_project``, + and ``create_or_update_*`` looks for an existing resource there, + before falling back to the server's only active project. + Updating a connection leaves it in its project. * ``DataMasqueFile`` gains a read-only ``project`` field, which is ``None`` for built-in seeds. * ``get_generated_rulesets`` sets each ruleset's ``project`` to the connection's project. * Added project models and management APIs. diff --git a/datamasque/client/base.py b/datamasque/client/base.py index 68db356..d45ebb7 100644 --- a/datamasque/client/base.py +++ b/datamasque/client/base.py @@ -133,6 +133,7 @@ class BaseClient: password: Optional[str] verify_ssl: bool token_source: Optional[Callable[[], str]] + default_project: Optional[str] def __init__(self, connection_config: DataMasqueInstanceConfig) -> None: self.base_url = connection_config.base_url @@ -140,6 +141,7 @@ def __init__(self, connection_config: DataMasqueInstanceConfig) -> None: self.password = connection_config.password self.verify_ssl = connection_config.verify_ssl self.token_source = connection_config.token_source + self.default_project = connection_config.default_project self._session = _build_session(self.verify_ssl) self._timeout = (connection_config.connect_timeout_seconds, connection_config.read_timeout_seconds) if connection_config.spcs_pat: @@ -318,15 +320,17 @@ def _delete_if_exists(self, path: str, *, params: Optional[dict] = None) -> None def _resolve_project(self, project: Optional[Union[ProjectId, str]]) -> Union[ProjectId, str]: """ - Returns `project`, or when it is `None`, the ID of the only active project. + Returns `project`, else the client's `default_project`, else the ID of the only active project. - That is the project the server puts a create into when it is sent no `project`. + The last is the project the server puts a create into when it is sent no `project`. With no active project, or more than one, the server rejects such a create, so this raises `DataMasqueException` instead. """ if project is not None: return project + if self.default_project is not None: + return self.default_project response = self.make_request("GET", "/api/projects/", params={"limit": 1}) page = Page[Project].model_validate(response.json()) @@ -337,6 +341,17 @@ def _resolve_project(self, project: Optional[Union[ProjectId, str]]) -> Union[Pr return page.results[0].id + def _default_project_field(self, project: Optional[Union[ProjectId, str]]) -> dict[str, str]: + """ + Returns the `project` field a create sends when the resource names no `project`: the client's `default_project`. + + Empty when the resource names one, which its own fields carry, or when there is no `default_project`. + """ + + if project is None and self.default_project is not None: + return {"project": self.default_project} + return {} + def _iter_paginated( self, path: str, diff --git a/datamasque/client/connections.py b/datamasque/client/connections.py index ddf4570..20e383d 100644 --- a/datamasque/client/connections.py +++ b/datamasque/client/connections.py @@ -35,7 +35,7 @@ def create_or_update_connection(self, connection_config: ConnectionConfig) -> Co Creates or updates the connection in DM, and sets the `id` field on the given `connection_config`. An existing connection with the same name in the same project is updated. - With no `project` set, the server's only active project is used, + With no `project` set, the client's `default_project` is used, else the server's only active project, and `DataMasqueException` is raised when it has no active project or more than one. """ @@ -53,6 +53,7 @@ def create_or_update_connection(self, connection_config: ConnectionConfig) -> Co "version": "1.0", } | connection_config.model_dump(exclude_none=True, by_alias=True, mode="json") if connection_id is None: + data |= self._default_project_field(connection_config.project) response = self.make_request("POST", "/api/connections/", data=data) else: response = self.make_request("PUT", f"/api/connections/{connection_id}/", data=data) diff --git a/datamasque/client/discovery_config_libraries.py b/datamasque/client/discovery_config_libraries.py index 781d277..d43dc9c 100644 --- a/datamasque/client/discovery_config_libraries.py +++ b/datamasque/client/discovery_config_libraries.py @@ -81,7 +81,9 @@ def create_discovery_config_library(self, library: DiscoveryConfigLibrary) -> Di if not library.yaml: raise ValueError("Cannot create a discovery config library without YAML content (yaml is empty)") - data = library.model_dump(exclude_none=True, by_alias=True, mode="json") + data = library.model_dump(exclude_none=True, by_alias=True, mode="json") | self._default_project_field( + library.project + ) response = self.make_request("POST", "/api/discovery/config-libraries/", data=data) created = DiscoveryConfigLibrary.model_validate(response.json()) library.id = created.id @@ -125,7 +127,7 @@ def create_or_update_discovery_config_library(self, library: DiscoveryConfigLibr """ Creates the library, or updates the existing one with the same name in the same project. - With no `project` set, the server's only active project is used, + With no `project` set, the client's `default_project` is used, else the server's only active project, and `DataMasqueException` is raised when it has no active project or more than one. Sets the library's `id` property. """ diff --git a/datamasque/client/discovery_configs.py b/datamasque/client/discovery_configs.py index cea748f..f0eb76e 100644 --- a/datamasque/client/discovery_configs.py +++ b/datamasque/client/discovery_configs.py @@ -86,7 +86,9 @@ def create_discovery_config(self, config: DiscoveryConfig) -> DiscoveryConfig: if not config.yaml: raise ValueError("Cannot create a discovery config without YAML content (yaml is empty)") - data = config.model_dump(exclude_none=True, by_alias=True, mode="json") + data = config.model_dump(exclude_none=True, by_alias=True, mode="json") | self._default_project_field( + config.project + ) response = self.make_request("POST", "/api/discovery/configs/", data=data) created = DiscoveryConfig.model_validate(response.json()) config.id = created.id @@ -130,7 +132,7 @@ def create_or_update_discovery_config(self, config: DiscoveryConfig) -> Discover """ Creates the config, or updates the existing one with the same name and type in the same project. - With no `project` set, the server's only active project is used, + With no `project` set, the client's `default_project` is used, else the server's only active project, and `DataMasqueException` is raised when it has no active project or more than one. Sets the config's `id` property. """ diff --git a/datamasque/client/models/connection.py b/datamasque/client/models/connection.py index 086b166..cde6bd0 100644 --- a/datamasque/client/models/connection.py +++ b/datamasque/client/models/connection.py @@ -125,7 +125,7 @@ class ConnectionConfig(BaseModel): """ The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, or the project's ID as a string. - Left as `None`, a create goes into the server's default project. + Left as `None`, a create goes into the client's `default_project`, else the server's default project. """ id: Optional[ConnectionId] = None # Server-populated and read-only, so it is excluded when serializing a config back to the API. diff --git a/datamasque/client/models/discovery_config.py b/datamasque/client/models/discovery_config.py index 6a5a909..3d767bb 100644 --- a/datamasque/client/models/discovery_config.py +++ b/datamasque/client/models/discovery_config.py @@ -44,7 +44,7 @@ class DiscoveryConfig(BaseModel): """ The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, or the project's ID as a string. - Left as `None`, a create goes into the server's default project. + Left as `None`, a create goes into the client's `default_project`, else the server's default project. """ yaml: Optional[str] = Field(default=None, alias="config_yaml") config_type: DiscoveryConfigType diff --git a/datamasque/client/models/discovery_config_library.py b/datamasque/client/models/discovery_config_library.py index c4ef39d..06058bb 100644 --- a/datamasque/client/models/discovery_config_library.py +++ b/datamasque/client/models/discovery_config_library.py @@ -24,7 +24,7 @@ class DiscoveryConfigLibrary(BaseModel): """ The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, or the project's ID as a string. - Left as `None`, a create goes into the server's default project. + Left as `None`, a create goes into the client's `default_project`, else the server's default project. """ yaml: Optional[str] = Field(default=None, alias="config_yaml") # Server-populated read-only fields, excluded from request bodies. diff --git a/datamasque/client/models/dm_instance.py b/datamasque/client/models/dm_instance.py index 7b30a98..0f82152 100644 --- a/datamasque/client/models/dm_instance.py +++ b/datamasque/client/models/dm_instance.py @@ -49,6 +49,12 @@ class DataMasqueInstanceConfig(BaseModel): """How long a request may take to connect, including the TLS handshake.""" read_timeout_seconds: float = DEFAULT_READ_TIMEOUT_SECONDS """How long a request may wait for the server between bytes of its response.""" + default_project: Optional[str] = None + """The project a create goes into when the resource names none, as a `-` reference (e.g. `1-default`) + or a bare project id. + + Applies to connections, rulesets, discovery configs, RG configs, table references and libraries. + Leave unset to use the server's only active project.""" @model_validator(mode="after") def _validate_auth_source(self) -> "DataMasqueInstanceConfig": diff --git a/datamasque/client/models/rg_config.py b/datamasque/client/models/rg_config.py index f5de6ac..441306b 100644 --- a/datamasque/client/models/rg_config.py +++ b/datamasque/client/models/rg_config.py @@ -43,7 +43,7 @@ class RGConfig(BaseModel): """ The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, or the project's ID as a string. - Left as `None`, a create goes into the server's default project. + Left as `None`, a create goes into the client's `default_project`, else the server's default project. """ yaml: Optional[str] = Field(default=None, alias="config_yaml") diff --git a/datamasque/client/models/ruleset.py b/datamasque/client/models/ruleset.py index 2c6275d..a42be76 100644 --- a/datamasque/client/models/ruleset.py +++ b/datamasque/client/models/ruleset.py @@ -42,7 +42,7 @@ class Ruleset(GitTrackedEntity): """ The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, or the project's ID as a string. - Left as `None`, a create goes into the server's default project. + Left as `None`, a create goes into the client's `default_project`, else the server's default project. """ yaml: str = Field(default="", alias="config_yaml") ruleset_type: RulesetType = Field(default=RulesetType.database, alias="mask_type") diff --git a/datamasque/client/models/ruleset_library.py b/datamasque/client/models/ruleset_library.py index d0b75d9..dd54b20 100644 --- a/datamasque/client/models/ruleset_library.py +++ b/datamasque/client/models/ruleset_library.py @@ -18,7 +18,7 @@ class RulesetLibrary(GitTrackedEntity): """ The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, or the project's ID as a string. - Left as `None`, a create goes into the server's default project. + Left as `None`, a create goes into the client's `default_project`, else the server's default project. """ yaml: Optional[str] = Field(default=None, alias="config_yaml") diff --git a/datamasque/client/models/table_reference.py b/datamasque/client/models/table_reference.py index 3bcac90..5b6c9b5 100644 --- a/datamasque/client/models/table_reference.py +++ b/datamasque/client/models/table_reference.py @@ -77,7 +77,7 @@ class TableReference(BaseModel): """ The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, or the project's ID as a string. - Left as `None`, a create goes into the server's default project. + Left as `None`, a create goes into the client's `default_project`, else the server's default project. """ connection: Union[ConnectionId, ConnectionConfig] """The connection the data lives in. Accepts a `ConnectionId` or a `ConnectionConfig` (its `id` is extracted).""" diff --git a/datamasque/client/rg_configs.py b/datamasque/client/rg_configs.py index 4ba96fe..ea94237 100644 --- a/datamasque/client/rg_configs.py +++ b/datamasque/client/rg_configs.py @@ -78,7 +78,9 @@ def create_rg_config(self, config: RGConfig) -> RGConfig: (`id`, `is_valid`, `validation_errors`, `created`, `modified`) and returns the config. """ - data = config.model_dump(exclude_none=True, by_alias=True, mode="json") + data = config.model_dump(exclude_none=True, by_alias=True, mode="json") | self._default_project_field( + config.project + ) response = self.make_request("POST", "/api/ruleset-generation-configs/", data=data) created = RGConfig.model_validate(response.json()) config.id = created.id @@ -121,7 +123,7 @@ def create_or_update_rg_config(self, config: RGConfig) -> RGConfig: """ Creates the config, or updates the existing one with the same name in the same project. - With no `project` set, the server's only active project is used, + With no `project` set, the client's `default_project` is used, else the server's only active project, and `DataMasqueException` is raised when it has no active project or more than one. Sets the config's `id` property. """ diff --git a/datamasque/client/ruleset_libraries.py b/datamasque/client/ruleset_libraries.py index e03b6c2..b029ead 100644 --- a/datamasque/client/ruleset_libraries.py +++ b/datamasque/client/ruleset_libraries.py @@ -79,7 +79,9 @@ def create_ruleset_library(self, library: RulesetLibrary) -> RulesetLibrary: and returns the library. """ - data = library.model_dump(exclude_none=True, by_alias=True, mode="json") + data = library.model_dump(exclude_none=True, by_alias=True, mode="json") | self._default_project_field( + library.project + ) response = self.make_request("POST", "/api/ruleset-libraries/", data=data) created_library = RulesetLibrary.model_validate(response.json()) library.id = created_library.id @@ -118,7 +120,7 @@ def create_or_update_ruleset_library(self, library: RulesetLibrary) -> RulesetLi """ Creates the library, or updates the existing one with the same name in the same project. - With no `project` set, the server's only active project is used, + With no `project` set, the client's `default_project` is used, else the server's only active project, and `DataMasqueException` is raised when it has no active project or more than one. Sets the library's `id` property. """ diff --git a/datamasque/client/rulesets.py b/datamasque/client/rulesets.py index 9a5159c..ed82ef8 100644 --- a/datamasque/client/rulesets.py +++ b/datamasque/client/rulesets.py @@ -22,7 +22,7 @@ def create_or_update_ruleset(self, ruleset: Ruleset) -> Ruleset: """ Creates the ruleset, or updates the existing one with the same name and type in the same project. - With no `project` set, the server's only active project is used, + With no `project` set, the client's `default_project` is used, else the server's only active project, and `DataMasqueException` is raised when it has no active project or more than one. Populates the given ruleset's `id`, `is_valid`, `validation_errors`, `unlicensed_feature_warnings`, and `git` fields from the server response, and returns the same ruleset instance for convenience. @@ -30,7 +30,9 @@ def create_or_update_ruleset(self, ruleset: Ruleset) -> Ruleset: # The server finds the existing ruleset itself, so this only checks that an unset project resolves. self._resolve_project(ruleset.project) - data = ruleset.model_dump(exclude_none=True, by_alias=True, mode="json") + data = ruleset.model_dump(exclude_none=True, by_alias=True, mode="json") | self._default_project_field( + ruleset.project + ) response = self.make_request("POST", "/api/rulesets/", data=data, params={"upsert": "true"}) created = Ruleset.model_validate(response.json()) ruleset.id = created.id diff --git a/datamasque/client/table_references.py b/datamasque/client/table_references.py index a3c6ff2..6d5f5b8 100644 --- a/datamasque/client/table_references.py +++ b/datamasque/client/table_references.py @@ -81,6 +81,7 @@ def create_table_reference(self, table_reference: TableReference) -> TableRefere data.pop("options", None) if data.get("project") is None: data.pop("project", None) + data |= self._default_project_field(table_reference.project) response = self.make_request("POST", "/api/table-references/", data=data) created = TableReference.model_validate(response.json()) table_reference.id = created.id @@ -122,7 +123,7 @@ def create_or_update_table_reference(self, table_reference: TableReference) -> T """ Creates the table reference, or updates the existing one with the same name in the same project. - With no `project` set, the server's only active project is used, + With no `project` set, the client's `default_project` is used, else the server's only active project, and `DataMasqueException` is raised when it has no active project or more than one. Sets the table reference's `id` property. """ diff --git a/tests/test_default_project.py b/tests/test_default_project.py new file mode 100644 index 0000000..899ad5b --- /dev/null +++ b/tests/test_default_project.py @@ -0,0 +1,137 @@ +"""Tests for the client's `default_project`, which a create uses when the resource names no project.""" + +from collections.abc import Callable +from typing import Any, Optional + +import pytest +import requests_mock + +from datamasque.client import DataMasqueClient +from datamasque.client.models.connection import ConnectionId +from datamasque.client.models.discovery_config import DiscoveryConfig +from datamasque.client.models.discovery_config_library import DiscoveryConfigLibrary +from datamasque.client.models.dm_instance import DataMasqueInstanceConfig +from datamasque.client.models.rg_config import RGConfig +from datamasque.client.models.ruleset import Ruleset, RulesetType +from datamasque.client.models.ruleset_library import RulesetLibrary +from datamasque.client.models.table_reference import TableReference +from tests.helpers import active_projects_page, database_connection_config + +BASE_URL = "http://test-server" + + +def make_client(default_project: Optional[str] = None) -> DataMasqueClient: + return DataMasqueClient( + DataMasqueInstanceConfig(base_url=BASE_URL, username="u", password="p", default_project=default_project) + ) + + +def create_ruleset(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.post(f"{BASE_URL}/api/rulesets/?upsert=true", json={"id": "r-1", "name": "r", "project": "2-acme"}) + ruleset = Ruleset(name="r", yaml="version: '1.0'", ruleset_type=RulesetType.database, project=project) + return client.create_or_update_ruleset(ruleset) + + +def create_connection(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.get(f"{BASE_URL}/api/connections/", json=[]) + m.post(f"{BASE_URL}/api/connections/", json={"id": "c-1", "project": "2-acme"}) + connection = database_connection_config() + connection.project = project + return client.create_or_update_connection(connection) + + +def create_discovery_config(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.post( + f"{BASE_URL}/api/discovery/configs/", + json={"id": "dc-1", "name": "dc", "config_type": "database", "project": "2-acme"}, + ) + config = DiscoveryConfig(name="dc", yaml="labels: []\n", config_type="database", project=project) + return client.create_discovery_config(config) + + +def create_rg_config(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.post(f"{BASE_URL}/api/ruleset-generation-configs/", json={"id": "rg-1", "name": "rg", "project": "2-acme"}) + return client.create_rg_config(RGConfig(name="rg", yaml="x: 1\n", project=project)) + + +def create_table_reference(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.post( + f"{BASE_URL}/api/table-references/", + json={"id": "tr-1", "name": "tr", "connection": "c-1", "source": "people.csv", "project": "2-acme"}, + ) + table_reference = TableReference(name="tr", connection=ConnectionId("c-1"), source="people.csv", project=project) + return client.create_table_reference(table_reference) + + +def create_ruleset_library(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.post(f"{BASE_URL}/api/ruleset-libraries/", json={"id": "rl-1", "name": "rl", "project": "2-acme"}) + return client.create_ruleset_library(RulesetLibrary(name="rl", yaml="version: '1.0'\n", project=project)) + + +def create_discovery_config_library(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.post(f"{BASE_URL}/api/discovery/config-libraries/", json={"id": "dl-1", "name": "dl", "project": "2-acme"}) + return client.create_discovery_config_library( + DiscoveryConfigLibrary(name="dl", yaml="labels: []\n", project=project) + ) + + +Create = Callable[[DataMasqueClient, Optional[str], requests_mock.Mocker], Any] + +CREATES = [ + pytest.param(create_ruleset, id="ruleset"), + pytest.param(create_connection, id="connection"), + pytest.param(create_discovery_config, id="discovery_config"), + pytest.param(create_rg_config, id="rg_config"), + pytest.param(create_table_reference, id="table_reference"), + pytest.param(create_ruleset_library, id="ruleset_library"), + pytest.param(create_discovery_config_library, id="discovery_config_library"), +] + + +@pytest.mark.parametrize("create", CREATES) +def test_a_create_sends_the_resources_project_over_the_default(create: Create) -> None: + with requests_mock.Mocker() as m: + create(make_client(default_project="1-default"), "2-acme", m) + + assert m.last_request.json()["project"] == "2-acme" + + +@pytest.mark.parametrize("create", CREATES) +def test_a_create_sends_the_default_project_when_the_resource_names_none(create: Create) -> None: + with requests_mock.Mocker() as m: + create(make_client(default_project="1-default"), None, m) + + assert m.last_request.json()["project"] == "1-default" + assert all(request.path != "/api/projects/" for request in m.request_history) + + +@pytest.mark.parametrize("create", CREATES) +def test_a_create_with_no_default_project_sends_no_project(create: Create) -> None: + with requests_mock.Mocker() as m: + m.get(f"{BASE_URL}/api/projects/", json=active_projects_page(1)) + create(make_client(), None, m) + + assert "project" not in m.last_request.json() + + +def test_a_connection_update_sends_no_project() -> None: + """An update leaves the connection in its own project, whatever the default.""" + client = make_client(default_project="1-default") + connection = database_connection_config() + connection.project = None + with requests_mock.Mocker() as m: + m.get(f"{BASE_URL}/api/connections/", json=[{**connection.model_dump(mode="json"), "id": "c-1"}]) + m.put(f"{BASE_URL}/api/connections/c-1/", json={"id": "c-1", "project": "2-acme"}) + + client.create_or_update_connection(connection) + + assert m.last_request.method == "PUT" + assert "project" not in m.last_request.json() + + +def test_an_upsert_looks_for_the_name_in_the_default_project() -> None: + with requests_mock.Mocker() as m: + create_connection(make_client(default_project="1-default"), None, m) + + lookup = m.request_history[0] + assert (lookup.method, lookup.path, lookup.qs) == ("GET", "/api/connections/", {"project": ["1-default"]})