Add delete_project, get_project_delete_conflicts and list_resources

HISTORY.rst

@@ -5,8 +5,8 @@ History
 1.7.0 (unreleased)
 ------------------
 
-* Added ``create_project``, which takes a ``ProjectCreateRequest`` and returns the created ``Project``.
-  Only admin users can create projects, and API tokens cannot.
+* Added project models and management APIs.
+* Added ``list_resources``, which lists resources of every type across projects.
 
 1.6.0 (unreleased)
 ------------------

datamasque/client/__init__.py

@@ -23,6 +23,7 @@
     InvalidDiscoveryConfigError,
     InvalidLibraryError,
     InvalidRulesetError,
+    ProjectDeleteConflictError,
     RGConfigNotFoundError,
     RunNotCancellableError,
 )
@@ -115,7 +116,20 @@
     RulesetPlanUpdateRequest,
 )
 from datamasque.client.models.license import LicenseInfo, SwitchableLicenseMetadata
-from datamasque.client.models.project import Project, ProjectCreateRequest, ProjectId
+from datamasque.client.models.project import (
+    Project,
+    ProjectCreateRequest,
+    ProjectDeleteConflict,
+    ProjectDeleteConflictReason,
+    ProjectId,
+)
+from datamasque.client.models.resource import (
+    ConnectionResource,
+    DiscoveryConfigResource,
+    Resource,
+    ResourceTypeName,
+    RulesetResource,
+)
 from datamasque.client.models.rg_config import RGConfig, RGConfigId
 from datamasque.client.models.ruleset import Ruleset, RulesetId, RulesetType
 from datamasque.client.models.ruleset_library import RulesetLibrary, RulesetLibraryId
@@ -189,6 +203,7 @@
     "CommonStatistics",
     "ConnectionConfig",
     "ConnectionId",
+    "ConnectionResource",
     "ConstraintColumns",
     "DataMasqueApiError",
     "DataMasqueClient",
@@ -209,6 +224,7 @@
     "DiscoveryConfigLibrary",
     "DiscoveryConfigLibraryId",
     "DiscoveryConfigNotFoundError",
+    "DiscoveryConfigResource",
     "DiscoveryConfigType",
     "DiscoveryMatch",
     "CosmosDbConnectionConfig",
@@ -272,11 +288,16 @@
     "PatternsStatistics",
     "Project",
     "ProjectCreateRequest",
+    "ProjectDeleteConflict",
+    "ProjectDeleteConflictError",
+    "ProjectDeleteConflictReason",
     "ProjectId",
     "RGConfig",
     "RGConfigId",
     "RGConfigNotFoundError",
     "ReferencingForeignKey",
+    "Resource",
+    "ResourceTypeName",
     "Ruleset",
     "RulesetGenerationRequest",
     "RulesetGenerationWithRGConfigRequest",
@@ -288,6 +309,7 @@
     "RulesetPlanOptions",
     "RulesetPlanPartialUpdateRequest",
     "RulesetPlanUpdateRequest",
+    "RulesetResource",
     "RulesetType",
     "RunConnectionRef",
     "RunId",

datamasque/client/base.py

@@ -317,8 +317,14 @@ def _iter_paginated(
         params: Optional[dict] = None,
         page_size: int = 100,
     ) -> Iterator[_T]:
+        """Iterate every `T` across all pages of an admin-server list endpoint."""
+
+        for results in self._iter_pages(path, params=params, page_size=page_size):
+            yield from (model.model_validate(item) for item in results)
+
+    def _iter_pages(self, path: str, *, params: Optional[dict] = None, page_size: int = 100) -> Iterator[Any]:
         """
-        Iterate every `T` across all pages of an admin-server list endpoint.
+        Iterate the `results` of each page of an admin-server list endpoint.
 
         Opts into pagination by sending `limit`/`offset` on the first request,
         then follows the absolute `next` URL returned by the server.
@@ -334,7 +340,7 @@ def _iter_paginated(
         while url:
             response = self.make_request("GET", url, params=current_params)
             data = response.json()
-            yield from (model.model_validate(item) for item in data["results"])
+            yield data["results"]
             url = data.get("next")
             # The `next` URL is absolute and already contains the pagination cursor;
             # do not re-send our initial params alongside it.

datamasque/client/dmclient.py

@@ -6,6 +6,7 @@
 from datamasque.client.files import FileClient
 from datamasque.client.license import LicenseClient
 from datamasque.client.projects import ProjectClient
+from datamasque.client.resources import ResourceClient
 from datamasque.client.rg_configs import RGConfigClient
 from datamasque.client.ruleset_libraries import RulesetLibraryClient
 from datamasque.client.rulesets import RulesetClient
@@ -20,6 +21,7 @@
 class DataMasqueClient(
     LicenseClient,
     ProjectClient,
+    ResourceClient,
     ConnectionClient,
     RulesetClient,
     RulesetLibraryClient,

datamasque/client/exceptions.py

@@ -1,5 +1,7 @@
 from requests import Response
 
+from datamasque.client.models.project import ProjectDeleteConflict
+
 
 class DataMasqueException(Exception):
     """Generic exception base class."""
@@ -66,6 +68,21 @@ class RGConfigNotFoundError(DataMasqueApiError):
     """
 
 
+class ProjectDeleteConflictError(DataMasqueApiError):
+    """
+    Raised when `delete_project` cannot move the project's resources to the destination project.
+
+    The server deletes and moves nothing in this case.
+
+    Attributes:
+        conflicts: Every reason the resources cannot move.
+    """
+
+    def __init__(self, message: str, *, response: Response, conflicts: list[ProjectDeleteConflict]) -> None:
+        super().__init__(message, response=response)
+        self.conflicts = conflicts
+
+
 class DataMasqueTransportError(DataMasqueException):
     """
     Raised when a request to the DataMasque server fails before any response is received.

datamasque/client/models/project.py

@@ -1,7 +1,8 @@
 """Project models for the DataMasque API."""
 
 from datetime import datetime
-from typing import NewType
+from enum import Enum
+from typing import NewType, Optional, Union
 
 from pydantic import BaseModel, ConfigDict
 
@@ -22,18 +23,52 @@ class Project(BaseModel):
     updated: datetime
 
 
+def unwrap_project_id(project: Union[ProjectId, Project]) -> ProjectId:
+    """Return the id of `project`, which a caller may give as a `Project` or as its `ProjectId`."""
+    return project.id if isinstance(project, Project) else project
+
+
 class ProjectCreateRequest(BaseModel):
     """
     Request body for `POST /api/projects/`.
 
-    Names are unique across active and archived projects, ignoring case.
-    The server trims leading and trailing whitespace from `name` and `description`,
-    and rejects a name that contains "/", ".." or a control character.
-    It may reject other names too,
-    and its 400 response gives the reason.
+    A name must not contain "/", ".." or a control character,
+    and must differ, ignoring case, from the name of every other project, including deleted ones.
+    The server trims leading and trailing whitespace from `name` and `description`.
     """
 
     model_config = ConfigDict(extra="forbid")
 
     name: str
     description: str = ""
+
+
+class ProjectDeleteConflictReason(str, Enum):
+    """Why a project's resources cannot move to the destination project when the project is deleted."""
+
+    name_in_use = "name_in_use"
+    """The destination project already contains a resource of the same type and name."""
+    file_exists = "file_exists"
+    """The destination project already has a file or directory where the resource's files would move to."""
+    unexpected_file = "unexpected_file"
+    """The project's directory contains a file or directory that belongs to none of its resources."""
+
+
+class ProjectDeleteConflict(BaseModel):
+    """A conflict that prevents a project's resources moving to the destination project when the project is deleted."""
+
+    model_config = ConfigDict(extra="allow")
+
+    reason: ProjectDeleteConflictReason
+    detail: str
+    """A message that describes the conflict."""
+    type: Optional[str] = None
+    """
+    For `name_in_use` and `file_exists`: the type of the resource that would move, such as `connections` or `rulesets`.
+    """
+    id: Optional[str] = None
+    """For `name_in_use` and `file_exists`: the ID of the resource that would move."""
+    name: Optional[str] = None
+    """For `name_in_use` and `file_exists`: the name of the resource that would move."""
+    path: Optional[str] = None
+    """For `file_exists` and `unexpected_file`: the path of the file or directory."""

datamasque/client/models/resource.py

@@ -0,0 +1,71 @@
+"""Resource listing models for the DataMasque API."""
+
+from datetime import datetime
+from enum import Enum
+from typing import Optional
+
+from pydantic import BaseModel, ConfigDict
+
+from datamasque.client.models.discovery_config import DiscoveryConfigType
+from datamasque.client.models.runs import MaskType
+
+
+class ResourceTypeName(str, Enum):
+    """A type of resource that belongs to a project, named as the resources listing groups it."""
+
+    connections = "connections"
+    connection_filesets = "connection_filesets"
+    oracle_wallets = "oracle_wallets"
+    snowflake_keys = "snowflake_keys"
+    table_references = "table_references"
+    rulesets = "rulesets"
+    ruleset_libraries = "ruleset_libraries"
+    discovery_configs = "discovery_configs"
+    discovery_config_libraries = "discovery_config_libraries"
+    ruleset_generation_configs = "ruleset_generation_configs"
+    seeds = "seeds"
+
+
+class Resource(BaseModel):
+    """
+    One resource in the resources listing: the summary the listing gives, not the resource's full model.
+
+    Connections, rulesets and discovery configs are listed as the subclasses that add the field each carries.
+    """
+
+    model_config = ConfigDict(extra="allow")
+
+    id: str
+    name: str
+    project: Optional[str] = None
+    """Its project's `<id>-<slug>` reference, or `None` for a resource in no project, such as a built-in seed."""
+    created: Optional[datetime] = None
+    """`None` for a type that does not record it, currently connections."""
+    modified: Optional[datetime] = None
+    """`None` for a type that does not record it, currently connections."""
+
+
+class ConnectionResource(Resource):
+    """A connection in the resources listing."""
+
+    mask_type: MaskType
+
+
+class RulesetResource(Resource):
+    """A ruleset in the resources listing."""
+
+    mask_type: MaskType
+
+
+class DiscoveryConfigResource(Resource):
+    """A discovery config in the resources listing."""
+
+    config_type: DiscoveryConfigType
+
+
+# The model each type's rows are listed as; a type not named here is listed as `Resource`.
+RESOURCE_MODEL_BY_TYPE_NAME: dict[str, type[Resource]] = {
+    ResourceTypeName.connections.value: ConnectionResource,
+    ResourceTypeName.rulesets.value: RulesetResource,
+    ResourceTypeName.discovery_configs.value: DiscoveryConfigResource,
+}

datamasque/client/projects.py

@@ -1,25 +1,91 @@
 import logging
+from typing import Union
+
+from requests import Response
 
 from datamasque.client.base import BaseClient
-from datamasque.client.models.project import Project, ProjectCreateRequest
+from datamasque.client.exceptions import ProjectDeleteConflictError
+from datamasque.client.models.project import (
+    Project,
+    ProjectCreateRequest,
+    ProjectDeleteConflict,
+    ProjectId,
+    unwrap_project_id,
+)
 
 logger = logging.getLogger(__name__)
 
 
+def _get_conflicts(response: Response) -> list[ProjectDeleteConflict]:
+    return [ProjectDeleteConflict.model_validate(conflict) for conflict in response.json()["conflicts"]]
+
+
 class ProjectClient(BaseClient):
     """Project API methods. Mixed into `DataMasqueClient`."""
 
     def create_project(self, request: ProjectCreateRequest) -> Project:
         """
         Creates a new project on the server, and returns it as the server stored it.
 
-        Only admin users can create projects, and API tokens cannot.
-        A name the server rejects (see `ProjectCreateRequest`) raises `DataMasqueApiError` with status 400,
-        and a caller who is not an admin user raises it with status 403.
+        Raises:
+            DataMasqueApiError: the server rejects the name (see `ProjectCreateRequest`), or any other non-2xx response.
         """
 
         data = request.model_dump(mode="json")
         response = self.make_request("POST", "/api/projects/", data=data)
         project = Project.model_validate(response.json())
         logger.info('Creation of project "%s" successful', project.name)
         return project
+
+    def get_project_delete_conflicts(
+        self,
+        project: Union[ProjectId, Project],
+        destination: Union[ProjectId, Project],
+    ) -> list[ProjectDeleteConflict]:
+        """
+        Returns the conflicts that would prevent `delete_project` moving the project's resources to the destination.
+
+        An empty list means `delete_project` can delete the project with this destination,
+        unless the resources in either project change first.
+
+        Raises:
+            DataMasqueApiError: the destination does not exist, is deleted, or is the project;
+                the project does not exist or is already deleted; or any other non-2xx response.
+        """
+
+        response = self.make_request(
+            "GET",
+            f"/api/projects/{unwrap_project_id(project)}/delete-check/",
+            params={"destination": unwrap_project_id(destination)},
+        )
+        return _get_conflicts(response)
+
+    def delete_project(self, project: Union[ProjectId, Project], destination: Union[ProjectId, Project]) -> None:
+        """
+        Deletes a project, after moving every resource in it to the destination project.
+
+        When the resources cannot move, the server changes nothing.
+        `get_project_delete_conflicts` lists the same conflicts without trying to delete the project.
+
+        Raises:
+            ProjectDeleteConflictError: the resources cannot move to the destination; its `conflicts` says why.
+            DataMasqueApiError: the destination does not exist, is deleted, or is the project;
+                the project does not exist or is already deleted; or any other non-2xx response.
+        """
+
+        project_id = unwrap_project_id(project)
+        response = self.make_request(
+            "DELETE",
+            f"/api/projects/{project_id}/",
+            params={"destination": unwrap_project_id(destination)},
+            require_status_check=False,
+        )
+        if response.status_code == 409:
+            raise ProjectDeleteConflictError(
+                str(response.json()["detail"]),
+                response=response,
+                conflicts=_get_conflicts(response),
+            )
+
+        self._raise_for_status(response)
+        logger.info("Deletion of project %s successful", project_id)