datamaker-py 0.10.0__tar.gz → 0.11.0__tar.gz

{datamaker_py-0.10.0 → datamaker_py-0.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datamaker-py
-Version: 0.10.0
+Version: 0.11.0
 Summary: The official Python library for the Automators DataMaker API.
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
@@ -62,6 +62,35 @@ if __name__ == "__main__":
 
 ```
 
+## Push a set into Tosca Cloud
+
+`push_tosca_dataset` creates or replaces a native Tosca Cloud data set from a saved
+set, an input set version, a dataset or inline rows. A Tosca test step reads it with
+`{TDM[<data set>][<subset>][<column>]}`.
+
+```python
+from datamaker import DataMaker, ToscaDatasetSchemaChangedError
+
+dm = DataMaker()
+
+try:
+    result = dm.push_tosca_dataset(
+        integration_id,
+        name="Orders",
+        input_set_id="is_123",        # or set_id=..., dataset_id=..., rows=[...]
+        on_existing="overwrite",      # "fail" (default) | "overwrite" | "new"
+    )
+    print(result.dataset_id, result.rows_count, result.outcome, result.example)
+except ToscaDatasetSchemaChangedError as e:
+    # The rows dropped a column the data set has. Pass on_schema_change="recreate"
+    # to delete and recreate it instead.
+    print("removed:", e.removed_columns, "added:", e.added_columns)
+```
+
+An input set gets one Tosca subset per test case by default; `subsets_by` picks another
+column or turns it off with `False`. `get_tosca_datasets(integration_id)` lists what a
+space already holds.
+
 ## Autocomplete Types
 
 This SDK automatically generates TypeScript autocomplete definitions for use in the [datamaker application](https://github.com/automators/datamaker). 

{datamaker_py-0.10.0 → datamaker_py-0.11.0}/README.md

@@ -51,6 +51,35 @@ if __name__ == "__main__":
 
 ```
 
+## Push a set into Tosca Cloud
+
+`push_tosca_dataset` creates or replaces a native Tosca Cloud data set from a saved
+set, an input set version, a dataset or inline rows. A Tosca test step reads it with
+`{TDM[<data set>][<subset>][<column>]}`.
+
+```python
+from datamaker import DataMaker, ToscaDatasetSchemaChangedError
+
+dm = DataMaker()
+
+try:
+    result = dm.push_tosca_dataset(
+        integration_id,
+        name="Orders",
+        input_set_id="is_123",        # or set_id=..., dataset_id=..., rows=[...]
+        on_existing="overwrite",      # "fail" (default) | "overwrite" | "new"
+    )
+    print(result.dataset_id, result.rows_count, result.outcome, result.example)
+except ToscaDatasetSchemaChangedError as e:
+    # The rows dropped a column the data set has. Pass on_schema_change="recreate"
+    # to delete and recreate it instead.
+    print("removed:", e.removed_columns, "added:", e.added_columns)
+```
+
+An input set gets one Tosca subset per test case by default; `subsets_by` picks another
+column or turns it off with `False`. `get_tosca_datasets(integration_id)` lists what a
+space already holds.
+
 ## Autocomplete Types
 
 This SDK automatically generates TypeScript autocomplete definitions for use in the [datamaker application](https://github.com/automators/datamaker). 

{datamaker_py-0.10.0 → datamaker_py-0.11.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datamaker-py"
-version = "0.10.0"
+version = "0.11.0"
 description = "The official Python library for the Automators DataMaker API."
 readme = "README.md"
 requires-python = ">=3.11"

datamaker_py-0.11.0/src/datamaker/__init__.py

@@ -0,0 +1,12 @@
+from .main import DataMaker  # noqa
+from .template import Template  # noqa
+from .error import (  # noqa
+    DataMakerError,
+    DataMakerAPIError,
+    ReadOnlyConnectionError,
+    ToscaDatasetExistsError,
+    ToscaDatasetLockedError,
+    ToscaDatasetSchemaChangedError,
+    ToscaDatasetTooLargeError,
+)
+from .routes.tosca import ToscaDatasetPushResult  # noqa

datamaker_py-0.11.0/src/datamaker/error.py

@@ -0,0 +1,94 @@
+from typing import Any, List, Optional
+
+
+class DataMakerError(Exception):
+    pass
+
+
+class DataMakerAPIError(DataMakerError):
+    """The API answered with a non-2xx status.
+
+    Subclasses ``DataMakerError`` and keeps its message, so existing
+    ``except DataMakerError`` handlers are unaffected. Adds the status code and
+    the decoded JSON body (``None`` when the body was not JSON).
+    """
+
+    def __init__(self, message: str, status_code: int, body: Any = None):
+        super().__init__(message)
+        self.status_code = status_code
+        self.body = body
+
+    def _detail(self, key: str) -> Any:
+        return self.body.get(key) if isinstance(self.body, dict) else None
+
+
+class ReadOnlyConnectionError(DataMakerAPIError):
+    """403: the connection is read-only, so DataMaker refused the write."""
+
+
+class ToscaDatasetExistsError(DataMakerAPIError):
+    """409: a Tosca data set of that name exists and ``on_existing`` is "fail"."""
+
+    @property
+    def dataset_id(self) -> Optional[str]:
+        """The id of the data set that is already there."""
+        return self._detail("datasetId")
+
+
+class ToscaDatasetLockedError(DataMakerAPIError):
+    """409: the data set is locked in Tosca Cloud, so it cannot be overwritten.
+
+    A test run or a "Test data - Expert" step holds it. Retry once it is
+    unlocked.
+    """
+
+    @property
+    def dataset_id(self) -> Optional[str]:
+        return self._detail("datasetId")
+
+
+class ToscaDatasetSchemaChangedError(DataMakerAPIError):
+    """409: an overwrite would drop columns the Tosca data set has.
+
+    Tosca only overwrites in place when the columns match. Added columns are
+    handled by the API; a removed column is refused unless the push asks for
+    ``on_schema_change="recreate"``.
+    """
+
+    @property
+    def dataset_id(self) -> Optional[str]:
+        return self._detail("datasetId")
+
+    @property
+    def added_columns(self) -> List[str]:
+        return self._detail("added") or []
+
+    @property
+    def removed_columns(self) -> List[str]:
+        return self._detail("removed") or []
+
+
+class ToscaDatasetTooLargeError(DataMakerAPIError):
+    """413: more rows or bytes than one push may write.
+
+    The row pair (``max_rows``, ``rows``) or the byte pair (``max_bytes``,
+    ``bytes``) is set, depending on which cap was hit. A set, input set or
+    dataset refused before its rows are loaded carries neither; the message
+    names the cap.
+    """
+
+    @property
+    def max_rows(self) -> Optional[int]:
+        return self._detail("maxRows")
+
+    @property
+    def rows(self) -> Optional[int]:
+        return self._detail("rows")
+
+    @property
+    def max_bytes(self) -> Optional[int]:
+        return self._detail("maxBytes")
+
+    @property
+    def bytes(self) -> Optional[int]:
+        return self._detail("bytes")

{datamaker_py-0.10.0 → datamaker_py-0.11.0}/src/datamaker/main.py

@@ -1,6 +1,6 @@
 import os
 from dotenv import load_dotenv
-from typing import Any, Optional, Dict, List
+from typing import Any, Optional, Dict, List, Union
 from .types import WorkspaceUploadResult
 from .routes.base import BaseClient
 from .routes.generation import GenerationClient
@@ -26,6 +26,7 @@ from .routes.sets import SetsClient
 from .routes.plans import PlansClient
 from .routes.masking_policies import MaskingPoliciesClient
 from .routes.keymaps import KeyMapsClient
+from .routes.tosca import ToscaClient, ToscaDatasetPushResult
 
 load_dotenv()
 
@@ -79,6 +80,7 @@ class DataMaker:
         self._masking_policies = MaskingPoliciesClient(
             api_key, default_headers, base_url, verify
         )
+        self._tosca = ToscaClient(api_key, default_headers, base_url, verify)
 
         # Maintain backward compatibility
         self.api_key = self._generation.api_key
@@ -939,6 +941,103 @@ class DataMaker:
             project_id=project_id,
         )
 
+    # =================== TOSCA METHODS ===================
+    def get_tosca_datasets(self, integration_id: str, space: Optional[str] = None):
+        """List the test data sets in a Tosca Cloud space.
+
+        Args:
+            integration_id: The Tosca Cloud connection to read through.
+            space: The Tosca Cloud space (id or name). Defaults to "default".
+
+        Returns:
+            A dictionary with space and items, each item carrying id, name,
+            subsetCount and isLocked.
+        """
+        return self._tosca.get_tosca_datasets(integration_id, space)
+
+    def push_tosca_dataset(
+        self,
+        integration_id: str,
+        name: str,
+        *,
+        set_id: Optional[str] = None,
+        input_set_id: Optional[str] = None,
+        version: Optional[Union[int, str]] = None,
+        dataset_id: Optional[str] = None,
+        rows: Optional[List[Dict[str, Any]]] = None,
+        allow_unlocked: bool = False,
+        on_existing: str = "fail",
+        on_schema_change: str = "refuse",
+        subsets_by: Optional[Union[str, bool]] = None,
+        space: str = "default",
+    ) -> ToscaDatasetPushResult:
+        """Create or replace a Tosca Cloud data set from DataMaker rows.
+
+        A Tosca test step reads the data set with
+        {TDM[<data set>][<subset>][<column>]}. Give exactly one source: set_id,
+        input_set_id (with an optional version), dataset_id or rows.
+
+        Args:
+            integration_id: The Tosca Cloud connection to push through.
+            name: The Tosca data set name. Unique per space.
+            set_id: Push the rows of this DataMaker set. It must be locked
+                unless allow_unlocked is True.
+            input_set_id: Push a frozen version of this input set.
+            version: The input set version number, or "latest" (the default).
+            dataset_id: Push the rows of this DataMaker dataset.
+            rows: Inline rows, each a dictionary of column name to value.
+            allow_unlocked: Push a set that is not locked.
+            on_existing: "fail" (default), "overwrite" (replace the rows,
+                keeping the id and subsets) or "new" (a copy beside it).
+            on_schema_change: On overwrite, when the rows drop a column:
+                "refuse" (default) or "recreate" (the id and subsets are lost).
+            subsets_by: Create one subset per distinct value of this column.
+                An input set is split by "TestCase" by default; False turns
+                it off.
+            space: The Tosca Cloud space (id or name). Defaults to "default".
+
+        Returns:
+            A result with dataset_id, name, space, rows_count, outcome
+            ("created", "overwritten", "recreated" or "copied"), columns,
+            added_columns, subsets and example.
+
+        Raises:
+            ReadOnlyConnectionError: The connection is read-only (403).
+            ToscaDatasetExistsError: The name is taken and on_existing is
+                "fail" (409). Carries dataset_id.
+            ToscaDatasetLockedError: An overwrite of a data set that is
+                locked in Tosca Cloud (409). Carries dataset_id.
+            ToscaDatasetSchemaChangedError: An overwrite would drop columns
+                (409). Carries added_columns and removed_columns.
+            ToscaDatasetTooLargeError: Too many rows or bytes for one push
+                (413). Carries max_rows or max_bytes when the API sends them.
+
+        Example:
+            >>> dm = DataMaker()
+            >>> result = dm.push_tosca_dataset(
+            ...     integration_id,
+            ...     name="Orders",
+            ...     input_set_id="is_123",
+            ...     on_existing="overwrite",
+            ... )
+            >>> result.example
+            '{TDM[Orders][Customer]}'
+        """
+        return self._tosca.push_tosca_dataset(
+            integration_id,
+            name,
+            set_id=set_id,
+            input_set_id=input_set_id,
+            version=version,
+            dataset_id=dataset_id,
+            rows=rows,
+            allow_unlocked=allow_unlocked,
+            on_existing=on_existing,
+            on_schema_change=on_schema_change,
+            subsets_by=subsets_by,
+            space=space,
+        )
+
     # =================== PROPERTY ACCESS TO CLIENTS ===================
     # For advanced users who want direct access to specific clients
 
@@ -1174,3 +1273,8 @@ class DataMaker:
     def masking_policies(self):
         """Access to masking policies client."""
         return self._masking_policies
+
+    @property
+    def tosca(self):
+        """Access to Tosca client."""
+        return self._tosca

{datamaker_py-0.10.0 → datamaker_py-0.11.0}/src/datamaker/routes/__init__.py

@@ -14,6 +14,7 @@ from .export_and_validation import ExportClient, ValidationClient
 from .scenario_files import ScenarioFilesClient
 from .sets import SetsClient
 from .keymaps import KeyMapsClient
+from .tosca import ToscaClient
 
 __all__ = [
     "BaseClient",
@@ -36,4 +37,5 @@ __all__ = [
     "ScenarioFilesClient",
     "SetsClient",
     "KeyMapsClient",
+    "ToscaClient",
 ]

{datamaker_py-0.10.0 → datamaker_py-0.11.0}/src/datamaker/routes/base.py

@@ -1,7 +1,7 @@
 import os
 import requests
 from typing import Optional, Dict
-from ..error import DataMakerError
+from ..error import DataMakerAPIError
 
 
 class BaseClient:
@@ -35,6 +35,14 @@ class BaseClient:
         response = requests.request(method, url, headers=self.headers, **kwargs)
 
         if response.status_code not in [200, 201]:
-            raise DataMakerError(f"API request failed: {response.text}")
+            try:
+                body = response.json()
+            except ValueError:
+                body = None
+            raise DataMakerAPIError(
+                f"API request failed: {response.text}",
+                status_code=response.status_code,
+                body=body,
+            )
 
         return response

datamaker_py-0.11.0/src/datamaker/routes/tosca.py

@@ -0,0 +1,270 @@
+"""Client for Tosca Cloud test data sets.
+
+A Tosca Cloud data set (Prepare > Test data) is what a test step reads with
+``{TDM[<data set>][<subset>][<column>]}``. The push goes through DataMaker:
+the API reads the rows with the caller's own access, uploads them to Tosca
+with the connection's credentials and writes the audit event, so a scenario
+never holds Tosca credentials or calls Tosca itself.
+"""
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Union
+from urllib.parse import quote
+
+from .base import BaseClient
+from ..error import (
+    DataMakerAPIError,
+    DataMakerError,
+    ReadOnlyConnectionError,
+    ToscaDatasetExistsError,
+    ToscaDatasetLockedError,
+    ToscaDatasetSchemaChangedError,
+    ToscaDatasetTooLargeError,
+)
+
+ON_EXISTING = ("fail", "overwrite", "new")
+ON_SCHEMA_CHANGE = ("refuse", "recreate")
+
+
+@dataclass(frozen=True)
+class ToscaDatasetPushResult:
+    """The data set as Tosca holds it after a push."""
+
+    dataset_id: str
+    name: str
+    space: str
+    rows_count: int
+    # "created", "overwritten", "recreated" or "copied"
+    outcome: str
+    # A ready-to-paste reference, e.g. "{TDM[Orders][Customer]}"
+    example: str
+    # [{"name", "type"}] as Tosca typed them: String, Double, Boolean, DateTime
+    columns: List[Dict[str, str]] = field(default_factory=list)
+    # Columns an overwrite added to the existing data set
+    added_columns: List[str] = field(default_factory=list)
+    # {"created": [names], "existing": [names], "skipped": count}
+    subsets: Dict[str, Any] = field(default_factory=dict)
+    # The API's response as sent, for fields this class does not name yet.
+    raw: Dict[str, Any] = field(default_factory=dict, repr=False, compare=False)
+
+
+class ToscaClient(BaseClient):
+    """Client for Tosca Cloud test data operations."""
+
+    def get_tosca_datasets(
+        self, integration_id: str, space: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """List the test data sets in a Tosca Cloud space.
+
+        Args:
+            integration_id: The Tosca Cloud connection to read through.
+            space: The Tosca Cloud space (id or name). Defaults to "default".
+
+        Returns:
+            A dictionary with ``space`` and ``items``, each item carrying
+            ``id``, ``name``, ``subsetCount`` and ``isLocked``.
+        """
+        if not integration_id:
+            raise DataMakerError("integration_id is required to list Tosca data sets.")
+
+        endpoint = f"/integrations/{integration_id}/tosca/datasets"
+        if space:
+            endpoint += f"?space={quote(space, safe='')}"
+
+        response = self._make_request("GET", endpoint)
+        return response.json()
+
+    def push_tosca_dataset(
+        self,
+        integration_id: str,
+        name: str,
+        *,
+        set_id: Optional[str] = None,
+        input_set_id: Optional[str] = None,
+        version: Optional[Union[int, str]] = None,
+        dataset_id: Optional[str] = None,
+        rows: Optional[List[Dict[str, Any]]] = None,
+        allow_unlocked: bool = False,
+        on_existing: str = "fail",
+        on_schema_change: str = "refuse",
+        subsets_by: Optional[Union[str, bool]] = None,
+        space: str = "default",
+    ) -> ToscaDatasetPushResult:
+        """Create or replace a Tosca Cloud data set from DataMaker rows.
+
+        Give exactly one source: ``set_id``, ``input_set_id`` (with an optional
+        ``version``), ``dataset_id`` or ``rows``.
+
+        Args:
+            integration_id: The Tosca Cloud connection to push through.
+            name: The Tosca data set name. Unique per space; test steps
+                reference the data set by this name.
+            set_id: Push the rows of this DataMaker set. The set must be
+                locked unless ``allow_unlocked`` is True.
+            input_set_id: Push a frozen version of this input set.
+            version: The input set version number, or "latest" (the default).
+                Only valid with ``input_set_id``.
+            dataset_id: Push the rows of this DataMaker dataset.
+            rows: Inline rows, each a dictionary of column name to value.
+                Column types are inferred.
+            allow_unlocked: Push a set that is not locked. Only valid with
+                ``set_id``.
+            on_existing: What to do when a data set called ``name`` exists:
+                "fail" (default), "overwrite" (replace its rows, keeping its
+                id and subsets) or "new" (Tosca creates a copy beside it).
+            on_schema_change: On overwrite, when the rows drop a column the
+                data set has: "refuse" (default) or "recreate" (delete and
+                recreate it; test steps keep working, its id and subsets are
+                lost). Added columns are always handled.
+            subsets_by: Create one subset per distinct value of this column,
+                so each Tosca test case reads only its rows. Left unset, an
+                input set is split by "TestCase" and other sources are not
+                split. False turns it off.
+            space: The Tosca Cloud space (id or name). Defaults to "default".
+
+        Returns:
+            A ``ToscaDatasetPushResult`` with ``dataset_id``, ``name``,
+            ``space``, ``rows_count``, ``outcome``, ``columns``,
+            ``added_columns``, ``subsets`` and ``example``.
+
+        Raises:
+            DataMakerError: If the arguments do not describe exactly one source.
+            ReadOnlyConnectionError: 403, the connection is read-only.
+            ToscaDatasetExistsError: 409, the name is taken and ``on_existing``
+                is "fail". Carries ``dataset_id``.
+            ToscaDatasetLockedError: 409, an overwrite of a data set that is
+                locked in Tosca Cloud. Carries ``dataset_id``.
+            ToscaDatasetSchemaChangedError: 409, an overwrite would drop
+                columns. Carries ``added_columns`` and ``removed_columns``.
+            ToscaDatasetTooLargeError: 413, more rows or bytes than one push
+                allows. Carries ``max_rows`` or ``max_bytes`` when the API
+                sends them.
+            DataMakerAPIError: Any other non-2xx answer, with ``status_code``
+                and ``body``.
+
+        Example:
+            >>> dm = DataMaker()
+            >>> result = dm.push_tosca_dataset(
+            ...     integration_id,
+            ...     name="Orders",
+            ...     input_set_id="is_123",
+            ...     on_existing="overwrite",
+            ... )
+            >>> result.example
+            '{TDM[Orders][Customer]}'
+        """
+        if not integration_id:
+            raise DataMakerError("integration_id is required to push a Tosca data set.")
+        if not name:
+            raise DataMakerError("name is required to push a Tosca data set.")
+        if on_existing not in ON_EXISTING:
+            raise DataMakerError(
+                f"on_existing must be one of {', '.join(ON_EXISTING)}; got {on_existing!r}."
+            )
+        if on_schema_change not in ON_SCHEMA_CHANGE:
+            raise DataMakerError(
+                f"on_schema_change must be one of {', '.join(ON_SCHEMA_CHANGE)}; "
+                f"got {on_schema_change!r}."
+            )
+        if subsets_by is True or subsets_by == "":
+            raise DataMakerError("subsets_by must be a column name or False.")
+
+        given = [
+            label
+            for label, value in (
+                ("set_id", set_id),
+                ("input_set_id", input_set_id),
+                ("dataset_id", dataset_id),
+                ("rows", rows),
+            )
+            if value is not None
+        ]
+        if len(given) != 1:
+            raise DataMakerError(
+                "Give exactly one source: set_id, input_set_id, dataset_id or rows"
+                + (f"; got {', '.join(given)}." if given else "; got none.")
+            )
+        if version is not None and input_set_id is None:
+            raise DataMakerError("version only applies to input_set_id.")
+        if allow_unlocked and set_id is None:
+            raise DataMakerError("allow_unlocked only applies to set_id.")
+
+        source: Dict[str, Any]
+        if set_id is not None:
+            source = {"kind": "set", "setId": set_id}
+            if allow_unlocked:
+                source["allowUnlocked"] = True
+        elif input_set_id is not None:
+            source = {
+                "kind": "inputSet",
+                "inputSetId": input_set_id,
+                "version": "latest" if version is None else version,
+            }
+        elif dataset_id is not None:
+            source = {"kind": "dataset", "datasetId": dataset_id}
+        else:
+            if not rows:
+                raise DataMakerError("rows must not be empty.")
+            source = {"kind": "rows", "rows": rows}
+
+        payload: Dict[str, Any] = {
+            "space": space,
+            "name": name,
+            "source": source,
+            "onExisting": on_existing,
+            "onSchemaChange": on_schema_change,
+        }
+        if subsets_by is not None:
+            payload["subsetsBy"] = subsets_by
+
+        try:
+            response = self._make_request(
+                "POST", f"/integrations/{integration_id}/tosca/datasets", json=payload
+            )
+        except DataMakerAPIError as error:
+            raise self._typed_error(error, on_existing) from error
+
+        data = response.json()
+        return ToscaDatasetPushResult(
+            dataset_id=data["datasetId"],
+            name=data["name"],
+            space=data["space"],
+            rows_count=data["rowsCount"],
+            outcome=data["outcome"],
+            example=data["example"],
+            columns=data.get("columns") or [],
+            added_columns=data.get("addedColumns") or [],
+            subsets=data.get("subsets") or {},
+            raw=data,
+        )
+
+    @staticmethod
+    def _typed_error(error: DataMakerAPIError, on_existing: str) -> DataMakerAPIError:
+        """Narrow a failed push to the error a scenario can act on."""
+        body = error.body if isinstance(error.body, dict) else {}
+        args = (str(error), error.status_code, error.body)
+
+        # 403 is also what a caller without read access to the source gets,
+        # so the status alone does not mean the connection is read-only.
+        if error.status_code == 403 and "read-only" in str(body.get("error", "")):
+            return ReadOnlyConnectionError(*args)
+        if error.status_code == 409:
+            by_code = {
+                "exists": ToscaDatasetExistsError,
+                "locked": ToscaDatasetLockedError,
+                "schemaChanged": ToscaDatasetSchemaChangedError,
+            }
+            if body.get("code") in by_code:
+                return by_code[body["code"]](*args)
+            # No code: an API from before the 409s were named. "Name taken"
+            # and "locked" then share a body shape, and which one it is
+            # follows from what the push asked for.
+            if "added" in body or "removed" in body:
+                return ToscaDatasetSchemaChangedError(*args)
+            if "datasetId" in body and on_existing == "fail":
+                return ToscaDatasetExistsError(*args)
+            if "datasetId" in body and on_existing == "overwrite":
+                return ToscaDatasetLockedError(*args)
+        if error.status_code == 413:
+            return ToscaDatasetTooLargeError(*args)
+        return error

{datamaker_py-0.10.0 → datamaker_py-0.11.0}/src/datamaker_py.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datamaker-py
-Version: 0.10.0
+Version: 0.11.0
 Summary: The official Python library for the Automators DataMaker API.
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
@@ -62,6 +62,35 @@ if __name__ == "__main__":
 
 ```
 
+## Push a set into Tosca Cloud
+
+`push_tosca_dataset` creates or replaces a native Tosca Cloud data set from a saved
+set, an input set version, a dataset or inline rows. A Tosca test step reads it with
+`{TDM[<data set>][<subset>][<column>]}`.
+
+```python
+from datamaker import DataMaker, ToscaDatasetSchemaChangedError
+
+dm = DataMaker()
+
+try:
+    result = dm.push_tosca_dataset(
+        integration_id,
+        name="Orders",
+        input_set_id="is_123",        # or set_id=..., dataset_id=..., rows=[...]
+        on_existing="overwrite",      # "fail" (default) | "overwrite" | "new"
+    )
+    print(result.dataset_id, result.rows_count, result.outcome, result.example)
+except ToscaDatasetSchemaChangedError as e:
+    # The rows dropped a column the data set has. Pass on_schema_change="recreate"
+    # to delete and recreate it instead.
+    print("removed:", e.removed_columns, "added:", e.added_columns)
+```
+
+An input set gets one Tosca subset per test case by default; `subsets_by` picks another
+column or turns it off with `False`. `get_tosca_datasets(integration_id)` lists what a
+space already holds.
+
 ## Autocomplete Types
 
 This SDK automatically generates TypeScript autocomplete definitions for use in the [datamaker application](https://github.com/automators/datamaker). 

{datamaker_py-0.10.0 → datamaker_py-0.11.0}/src/datamaker_py.egg-info/SOURCES.txt

@@ -25,6 +25,7 @@ src/datamaker/routes/scenario_files.py
 src/datamaker/routes/sets.py
 src/datamaker/routes/teams.py
 src/datamaker/routes/templates.py
+src/datamaker/routes/tosca.py
 src/datamaker/routes/users.py
 src/datamaker_py.egg-info/PKG-INFO
 src/datamaker_py.egg-info/SOURCES.txt
@@ -33,4 +34,5 @@ src/datamaker_py.egg-info/requires.txt
 src/datamaker_py.egg-info/top_level.txt
 tests/test_integration.py
 tests/test_main.py
-tests/test_routes.py
+tests/test_routes.py
+tests/test_tosca.py

datamaker_py-0.11.0/tests/test_tosca.py

@@ -0,0 +1,425 @@
+"""Tests for pushing rows into Tosca Cloud data sets."""
+
+import json
+
+import pytest
+from unittest.mock import Mock, patch
+
+from src.datamaker.main import DataMaker
+from src.datamaker.error import (
+    DataMakerAPIError,
+    DataMakerError,
+    ReadOnlyConnectionError,
+    ToscaDatasetExistsError,
+    ToscaDatasetLockedError,
+    ToscaDatasetSchemaChangedError,
+    ToscaDatasetTooLargeError,
+)
+from src.datamaker.routes.tosca import ToscaDatasetPushResult
+
+PUSHED = {
+    "datasetId": "TDS-1",
+    "name": "Orders",
+    "space": "default",
+    "rowsCount": 2,
+    "columns": [{"name": "Customer", "type": "String"}],
+    "outcome": "created",
+    "addedColumns": [],
+    "subsets": {"created": ["TC-1"], "existing": [], "skipped": 0},
+    "example": "{TDM[Orders][Customer]}",
+}
+
+
+def ok(body=None, status_code=201):
+    response = Mock()
+    response.status_code = status_code
+    response.json.return_value = body or PUSHED
+    return response
+
+
+def failed(status_code, body):
+    response = Mock()
+    response.status_code = status_code
+    response.text = json.dumps(body)
+    response.json.return_value = body
+    return response
+
+
+def push(api_key, **kwargs):
+    return DataMaker(api_key=api_key).push_tosca_dataset(
+        "int-1", name="Orders", **kwargs
+    )
+
+
+class TestPushToscaDataset:
+    """The request each source kind sends, and the result it returns."""
+
+    @patch("requests.request")
+    def test_set_source(self, mock_request, api_key):
+        mock_request.return_value = ok()
+
+        result = push(api_key, set_id="set-1")
+
+        method, url = mock_request.call_args.args
+        assert method == "POST"
+        assert url.endswith("/integrations/int-1/tosca/datasets")
+        assert mock_request.call_args.kwargs["json"] == {
+            "space": "default",
+            "name": "Orders",
+            "source": {"kind": "set", "setId": "set-1"},
+            "onExisting": "fail",
+            "onSchemaChange": "refuse",
+        }
+        assert isinstance(result, ToscaDatasetPushResult)
+        assert result.dataset_id == "TDS-1"
+        assert result.name == "Orders"
+        assert result.space == "default"
+        assert result.rows_count == 2
+        assert result.outcome == "created"
+        assert result.columns == [{"name": "Customer", "type": "String"}]
+        assert result.added_columns == []
+        assert result.subsets == {"created": ["TC-1"], "existing": [], "skipped": 0}
+        assert result.example == "{TDM[Orders][Customer]}"
+        assert result.raw == PUSHED
+
+    @patch("requests.request")
+    def test_unlocked_set(self, mock_request, api_key):
+        mock_request.return_value = ok()
+
+        push(api_key, set_id="set-1", allow_unlocked=True)
+
+        assert mock_request.call_args.kwargs["json"]["source"] == {
+            "kind": "set",
+            "setId": "set-1",
+            "allowUnlocked": True,
+        }
+
+    @patch("requests.request")
+    def test_input_set_source_defaults_to_latest(self, mock_request, api_key):
+        mock_request.return_value = ok()
+
+        push(api_key, input_set_id="is-1")
+
+        payload = mock_request.call_args.kwargs["json"]
+        assert payload["source"] == {
+            "kind": "inputSet",
+            "inputSetId": "is-1",
+            "version": "latest",
+        }
+        # Left out so the API applies its own default (TestCase for an input set).
+        assert "subsetsBy" not in payload
+
+    @patch("requests.request")
+    def test_input_set_source_with_options(self, mock_request, api_key):
+        mock_request.return_value = ok(
+            {**PUSHED, "outcome": "overwritten", "addedColumns": ["Region"]},
+            status_code=200,
+        )
+
+        result = push(
+            api_key,
+            input_set_id="is-1",
+            version=3,
+            on_existing="overwrite",
+            on_schema_change="recreate",
+            subsets_by="Region",
+            space="qa",
+        )
+
+        payload = mock_request.call_args.kwargs["json"]
+        assert payload["source"]["version"] == 3
+        assert payload["onExisting"] == "overwrite"
+        assert payload["onSchemaChange"] == "recreate"
+        assert payload["subsetsBy"] == "Region"
+        assert payload["space"] == "qa"
+        assert result.outcome == "overwritten"
+        assert result.added_columns == ["Region"]
+
+    @patch("requests.request")
+    def test_subsets_can_be_turned_off(self, mock_request, api_key):
+        mock_request.return_value = ok()
+
+        push(api_key, input_set_id="is-1", subsets_by=False)
+
+        assert mock_request.call_args.kwargs["json"]["subsetsBy"] is False
+
+    @patch("requests.request")
+    def test_dataset_source(self, mock_request, api_key):
+        mock_request.return_value = ok()
+
+        push(api_key, dataset_id="ds-1")
+
+        assert mock_request.call_args.kwargs["json"]["source"] == {
+            "kind": "dataset",
+            "datasetId": "ds-1",
+        }
+
+    @patch("requests.request")
+    def test_rows_source(self, mock_request, api_key):
+        mock_request.return_value = ok({**PUSHED, "outcome": "copied"})
+        rows = [{"Customer": "0042"}, {"Customer": "0043"}]
+
+        result = push(api_key, rows=rows, on_existing="new")
+
+        payload = mock_request.call_args.kwargs["json"]
+        assert payload["source"] == {"kind": "rows", "rows": rows}
+        assert payload["onExisting"] == "new"
+        assert result.outcome == "copied"
+
+
+class TestPushToscaDatasetValidation:
+    """Bad arguments are refused before any request is made."""
+
+    @pytest.mark.parametrize(
+        "kwargs, match",
+        [
+            ({}, "exactly one source"),
+            ({"set_id": "s", "input_set_id": "i"}, "exactly one source"),
+            ({"set_id": "s", "rows": [{"a": 1}]}, "exactly one source"),
+            ({"input_set_id": "i", "dataset_id": "d"}, "exactly one source"),
+            ({"rows": []}, "rows must not be empty"),
+            ({"set_id": "s", "version": 2}, "version only applies"),
+            ({"dataset_id": "d", "allow_unlocked": True}, "allow_unlocked only applies"),
+            ({"set_id": "s", "on_existing": "replace"}, "on_existing must be one of"),
+            ({"set_id": "s", "on_schema_change": "drop"}, "on_schema_change must be"),
+            ({"set_id": "s", "subsets_by": True}, "subsets_by must be"),
+        ],
+    )
+    @patch("requests.request")
+    def test_rejected_locally(self, mock_request, api_key, kwargs, match):
+        with pytest.raises(DataMakerError, match=match):
+            push(api_key, **kwargs)
+        mock_request.assert_not_called()
+
+    @patch("requests.request")
+    def test_requires_integration_id_and_name(self, mock_request, api_key):
+        dm = DataMaker(api_key=api_key)
+        with pytest.raises(DataMakerError, match="integration_id is required"):
+            dm.push_tosca_dataset("", name="Orders", set_id="s")
+        with pytest.raises(DataMakerError, match="name is required"):
+            dm.push_tosca_dataset("int-1", name="", set_id="s")
+        mock_request.assert_not_called()
+
+
+class TestPushToscaDatasetErrors:
+    """Each failure a scenario can act on arrives as its own type."""
+
+    @patch("requests.request")
+    def test_read_only_connection(self, mock_request, api_key):
+        body = {
+            "error": "This Tosca connection is read-only, so creating a test "
+            "data set is blocked."
+        }
+        mock_request.return_value = failed(403, body)
+
+        with pytest.raises(ReadOnlyConnectionError, match="read-only") as raised:
+            push(api_key, set_id="s")
+
+        assert raised.value.status_code == 403
+        assert raised.value.body == body
+
+    @patch("requests.request")
+    def test_403_for_an_unreadable_source_is_not_read_only(
+        self, mock_request, api_key
+    ):
+        mock_request.return_value = failed(403, {"error": "Missing set:read"})
+
+        with pytest.raises(DataMakerAPIError) as raised:
+            push(api_key, set_id="s")
+
+        assert type(raised.value) is DataMakerAPIError
+        assert raised.value.status_code == 403
+
+    @patch("requests.request")
+    def test_schema_change(self, mock_request, api_key):
+        mock_request.return_value = failed(
+            409,
+            {
+                "error": "The rows no longer have column Country",
+                "datasetId": "TDS-1",
+                "added": ["Region"],
+                "removed": ["Country"],
+            },
+        )
+
+        with pytest.raises(ToscaDatasetSchemaChangedError) as raised:
+            push(api_key, set_id="s", on_existing="overwrite")
+
+        assert raised.value.status_code == 409
+        assert raised.value.dataset_id == "TDS-1"
+        assert raised.value.added_columns == ["Region"]
+        assert raised.value.removed_columns == ["Country"]
+
+    @patch("requests.request")
+    def test_name_exists(self, mock_request, api_key):
+        mock_request.return_value = failed(
+            409, {"error": "A data set named Orders exists", "datasetId": "TDS-1"}
+        )
+
+        with pytest.raises(ToscaDatasetExistsError) as raised:
+            push(api_key, set_id="s")
+
+        assert raised.value.status_code == 409
+        assert raised.value.dataset_id == "TDS-1"
+
+    @pytest.mark.parametrize(
+        "body, on_existing, expected",
+        [
+            ({"code": "exists", "datasetId": "TDS-1"}, "fail", ToscaDatasetExistsError),
+            ({"code": "locked", "datasetId": "TDS-1"}, "overwrite", ToscaDatasetLockedError),
+            (
+                {"code": "schemaChanged", "datasetId": "TDS-1", "added": [], "removed": ["A"]},
+                "overwrite",
+                ToscaDatasetSchemaChangedError,
+            ),
+            # The code wins over what the request's on_existing would suggest.
+            ({"code": "locked", "datasetId": "TDS-1"}, "fail", ToscaDatasetLockedError),
+        ],
+    )
+    @patch("requests.request")
+    def test_409_code_names_the_reason(
+        self, mock_request, api_key, body, on_existing, expected
+    ):
+        mock_request.return_value = failed(409, {"error": "refused", **body})
+
+        with pytest.raises(expected) as raised:
+            push(api_key, set_id="s", on_existing=on_existing)
+
+        assert type(raised.value) is expected
+        assert raised.value.dataset_id == "TDS-1"
+
+    @patch("requests.request")
+    def test_409_with_an_unknown_code_stays_generic(self, mock_request, api_key):
+        mock_request.return_value = failed(409, {"error": "refused", "code": "other"})
+
+        with pytest.raises(DataMakerAPIError) as raised:
+            push(api_key, set_id="s", on_existing="new")
+
+        assert type(raised.value) is DataMakerAPIError
+
+    @patch("requests.request")
+    def test_locked_data_set(self, mock_request, api_key):
+        # No code: same body shape as "name exists", told apart by the request.
+        mock_request.return_value = failed(
+            409, {"error": 'Data set "Orders" is locked', "datasetId": "TDS-1"}
+        )
+
+        with pytest.raises(ToscaDatasetLockedError) as raised:
+            push(api_key, set_id="s", on_existing="overwrite")
+
+        assert raised.value.dataset_id == "TDS-1"
+
+    @patch("requests.request")
+    def test_too_large_before_loading_names_no_numbers(self, mock_request, api_key):
+        mock_request.return_value = failed(
+            413, {"error": "A push may write at most 50000 rows; dataset has 60000."}
+        )
+
+        with pytest.raises(ToscaDatasetTooLargeError, match="50000") as raised:
+            push(api_key, dataset_id="d")
+
+        assert raised.value.max_rows is None
+        assert raised.value.max_bytes is None
+
+    @patch("requests.request")
+    def test_too_many_rows(self, mock_request, api_key):
+        mock_request.return_value = failed(
+            413, {"error": "Too many rows", "maxRows": 50000, "rows": 60000}
+        )
+
+        with pytest.raises(ToscaDatasetTooLargeError) as raised:
+            push(api_key, set_id="s")
+
+        assert raised.value.status_code == 413
+        assert raised.value.max_rows == 50000
+        assert raised.value.rows == 60000
+        assert raised.value.max_bytes is None
+
+    @patch("requests.request")
+    def test_too_many_bytes(self, mock_request, api_key):
+        mock_request.return_value = failed(
+            413, {"error": "Too large", "maxBytes": 1000, "bytes": 2000}
+        )
+
+        with pytest.raises(ToscaDatasetTooLargeError) as raised:
+            push(api_key, set_id="s")
+
+        assert raised.value.max_bytes == 1000
+        assert raised.value.bytes == 2000
+        assert raised.value.max_rows is None
+
+    @pytest.mark.parametrize(
+        "status_code, body",
+        [
+            (400, {"error": "The set is not locked"}),
+            (404, {"error": "Set not found"}),
+            (429, {"message": "Too many Tosca data set pushes. Try again in 60s."}),
+            (502, {"error": "Tosca refused the data set", "toscaError": "boom"}),
+        ],
+    )
+    @patch("requests.request")
+    def test_other_failures_carry_status_and_body(
+        self, mock_request, api_key, status_code, body
+    ):
+        mock_request.return_value = failed(status_code, body)
+
+        with pytest.raises(DataMakerAPIError) as raised:
+            push(api_key, set_id="s")
+
+        assert type(raised.value) is DataMakerAPIError
+        assert raised.value.status_code == status_code
+        assert raised.value.body == body
+
+    @patch("requests.request")
+    def test_typed_errors_are_still_datamaker_errors(self, mock_request, api_key):
+        mock_request.return_value = failed(413, {"error": "too big", "maxRows": 10})
+
+        with pytest.raises(DataMakerError):
+            push(api_key, set_id="s")
+
+
+class TestGetToscaDatasets:
+    @patch("requests.request")
+    def test_lists_a_space(self, mock_request, api_key):
+        listing = {
+            "space": "qa",
+            "items": [
+                {"id": "TDS-1", "name": "Orders", "subsetCount": 2, "isLocked": False}
+            ],
+        }
+        mock_request.return_value = ok(listing, status_code=200)
+
+        result = DataMaker(api_key=api_key).get_tosca_datasets("int-1", space="my qa")
+
+        method, url = mock_request.call_args.args
+        assert method == "GET"
+        assert url.endswith("/integrations/int-1/tosca/datasets?space=my%20qa")
+        assert result == listing
+
+    @patch("requests.request")
+    def test_default_space(self, mock_request, api_key):
+        mock_request.return_value = ok({"space": "default", "items": []}, 200)
+
+        DataMaker(api_key=api_key).get_tosca_datasets("int-1")
+
+        assert mock_request.call_args.args[1].endswith(
+            "/integrations/int-1/tosca/datasets"
+        )
+
+
+class TestApiErrorDetails:
+    """Every failed request now carries its status and body."""
+
+    @patch("requests.request")
+    def test_non_json_body(self, mock_request, api_key):
+        response = Mock()
+        response.status_code = 502
+        response.text = "Bad Gateway"
+        response.json.side_effect = ValueError("not json")
+        mock_request.return_value = response
+
+        with pytest.raises(DataMakerAPIError, match="Bad Gateway") as raised:
+            DataMaker(api_key=api_key).get_projects()
+
+        assert raised.value.status_code == 502
+        assert raised.value.body is None

datamaker_py-0.10.0/src/datamaker/__init__.py

@@ -1,2 +0,0 @@
-from .main import DataMaker  # noqa
-from .template import Template  # noqa

datamaker_py-0.10.0/src/datamaker/error.py

@@ -1,2 +0,0 @@
-class DataMakerError(Exception):
-    pass