masterly 0.4.0__tar.gz → 0.5.0__tar.gz

{masterly-0.4.0 → masterly-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: masterly
-Version: 0.4.0
+Version: 0.5.0
 Summary: Python client for Masterly — extract governed data products and ingest records from your lakehouse
 Project-URL: Homepage, https://masterlydata.com
 Project-URL: Documentation, https://masterlydata.com/docs/reference/python-sdk/
@@ -222,7 +222,9 @@ recomputes without it, and its history stays readable. A delete for a key the so
 hold does nothing.
 
 When a system can only hand over everything it has, send it as a **full snapshot**: every live
-record of the Source that the snapshot does not carry is deleted after the upserts.
+record of the target that the snapshot does not carry is deleted after the upserts (a source
+that [feeds several models](#a-source-that-feeds-several-models) names the target with
+`model=`; its other targets are untouched).
 
 ```python
 client.sources.ingest("crm", every_customer, mode="full", batch_size=5000)
@@ -413,10 +415,84 @@ source = client.sources.get("erp")
 client.sources.update("erp", display_name="SAP ERP (Sweden)", if_match=source["version"])
 ```
 
+A field the field map does not name passes through under its own name, so a field named like a
+model attribute lands on it. When such a field is not meant for the model, name it in
+`ignored_fields=` (on `create`, `SourceTarget` and `targets.add`): it is dropped before the field
+map, lands on no attribute and is not reported as schema drift. It comes back as
+`mapping["ignored_fields"]`; to change it later, edit the mapping you read and pass it to
+`update` (or `targets.update`). An install on an API release before this field existed drops
+`ignored_fields` without an error and returns no such key, so read the mapping back to confirm it
+was kept.
+
 Everything takes an id or an exact name, so a script reads the way the domain is discussed.
 A model's `definition` carries its attributes, keys and constraints — ingest validates every
 record against them, so widening or tightening one changes what the pipeline accepts.
 
+### A source that feeds several models
+
+A source feeds one or more models — its **targets** — and each target has its own source
+key and field map. A CRM export that carries customers and their addresses is one source
+with two targets, registered with `targets=` instead of `target_model=`:
+
+```python
+from masterly import SourceTarget
+
+client.sources.create(
+    "crm",
+    display_name="Salesforce CRM",
+    targets=[
+        SourceTarget("Customer", source_key="customer_number"),
+        SourceTarget("Address", source_key="address_id", field_map={"ADDR_ID": "address_id"}),
+    ],
+)
+```
+
+A batch lands in **one** target, so a push to such a source names it with `model=` — one call
+per model. On a source with one target `model` may be left out, and the call sends exactly
+what it always sent; on a source with several, an unnamed batch is refused with
+`SOURCE_TARGET_REQUIRED` before anything is queued, `error.details["targets"]` listing them:
+
+```python
+client.sources.ingest("crm", customers, model="Customer")
+client.sources.ingest("crm", addresses, model="Address")
+client.sources.upload_csv("crm", open("addresses.csv").read(), model="Address", delimiter=";")
+
+with client.sources.full_load("crm", model="Address") as load:   # one target's snapshot
+    load.send(every_address)
+```
+
+Records, quarantine and counts are per target too — a source key names one record within a
+target, so the same key under two targets is two records:
+
+```python
+for row in client.sources.quarantine("crm", model="Address", status="open"):
+    print(row["model_name"], row["reason_code"], row["payload"])
+
+stats = client.sources.stats("crm")           # the whole source, plus `targets[]` per model
+stats = client.sources.stats("crm", model="Address")
+```
+
+`client.sources.targets` lists, adds, reads, edits and removes a source's targets. A target
+has a revision of its own, stated on its governed writes; adding one needs a model with a
+published version, and only a target that holds no records — deleted ones included — can be
+removed, never the last one:
+
+```python
+client.sources.targets.add("crm", "Address", source_key="address_id")
+target = client.sources.targets.get("crm", "Address")
+target["mapping"]["field_map"]["STREET"] = "street"
+client.sources.targets.update("crm", "Address", mapping=target["mapping"], if_match=target["version"])
+client.sources.targets.remove("crm", "Address", if_match=target["version"])   # only while empty
+```
+
+On what `client.sources.get` returns, `targets[]` carries every target; the top-level
+`target_model`, `mapping`, `profile` and `drift` are **deprecated** and describe the first
+target — on a one-target source, exactly what they always did. The same goes for `target_model=`
+and `mapping=` on `client.sources.update`, which write the first target; edit any target through
+`client.sources.targets.update`. These calls talk to routes that are not in a published release
+of the API yet; an install on an earlier release answers them `404 NOT_FOUND`, and sends every
+batch to its one target.
+
 Editing a model or a source is a **governed write**: more than one person edits them, and the
 edit is derived from a read, so `update` states the revision it replaces (ADR 0070) and a stale
 one is refused rather than silently overwriting:
@@ -433,6 +509,45 @@ A refused write raises `ApiError`; `error.conflict` names the fields that moved.
 without a typed method is reached through `client.request(...)`, which takes the same
 `if_match`.
 
+### Deactivate, reactivate or delete a source
+
+A source has a `status`: `active`, or `inactive` once deactivated. An **inactive** source admits
+nothing — a pushed batch, a CSV upload, a full load's next batch and a quarantine retry are all
+refused with `409 SOURCE_INACTIVE` and nothing is queued. Everything it already landed stays,
+and **its records keep contributing to golden records**: nothing is recomputed, in either
+direction. Reactivating admits records again with nothing lost. Both are governed writes, so
+they take the source's `version` as `if_match`:
+
+```python
+source = client.sources.get("erp")
+client.sources.deactivate("erp", if_match=source["version"])   # status: "inactive"
+
+source = client.sources.get("erp")
+client.sources.reactivate("erp", if_match=source["version"])   # status: "active"
+```
+
+A **delete** removes the source and everything it landed — its records in every target and
+their history, quarantine, profile, drift, run history, schedule and sealed connection — and
+recomputes the golden records it contributed to without it; an entity left with no live
+record clears. It is heavy work, so it runs as a job: the source is marked `deleting` at once,
+the answer is the job receipt, and `client.jobs.wait` follows the removal to its end. Read
+what it would touch first, and ask a person to type the source's `name` before you send it:
+
+```python
+impact = client.sources.deletion_impact("erp")
+print(impact["targets"], impact["golden"]["would_change"], impact["golden"]["would_clear"])
+if impact["deletable"] and typed_name == impact["name"]:
+    source = client.sources.get("erp")
+    receipt = client.sources.delete("erp", if_match=source["version"])
+    client.jobs.wait(receipt["job_id"])
+```
+
+The delete is refused with `409 SOURCE_IN_USE` while a data product reads the source through a
+raw relation — `error.details["dependents"]` names each one; change or delete the product
+first — and the model's built-in manual source can be neither deactivated nor deleted
+(`409 SOURCE_BUILT_IN`). The four calls are session routes. They talk to routes that are not
+in a published release yet; an install on an earlier release answers them `404 NOT_FOUND`.
+
 ## Command line: `masterly config migrate`
 
 The configuration files in a GitOps repository follow a schema that Masterly versions. A

{masterly-0.4.0 → masterly-0.5.0}/README.md

@@ -193,7 +193,9 @@ recomputes without it, and its history stays readable. A delete for a key the so
 hold does nothing.
 
 When a system can only hand over everything it has, send it as a **full snapshot**: every live
-record of the Source that the snapshot does not carry is deleted after the upserts.
+record of the target that the snapshot does not carry is deleted after the upserts (a source
+that [feeds several models](#a-source-that-feeds-several-models) names the target with
+`model=`; its other targets are untouched).
 
 ```python
 client.sources.ingest("crm", every_customer, mode="full", batch_size=5000)
@@ -384,10 +386,84 @@ source = client.sources.get("erp")
 client.sources.update("erp", display_name="SAP ERP (Sweden)", if_match=source["version"])
 ```
 
+A field the field map does not name passes through under its own name, so a field named like a
+model attribute lands on it. When such a field is not meant for the model, name it in
+`ignored_fields=` (on `create`, `SourceTarget` and `targets.add`): it is dropped before the field
+map, lands on no attribute and is not reported as schema drift. It comes back as
+`mapping["ignored_fields"]`; to change it later, edit the mapping you read and pass it to
+`update` (or `targets.update`). An install on an API release before this field existed drops
+`ignored_fields` without an error and returns no such key, so read the mapping back to confirm it
+was kept.
+
 Everything takes an id or an exact name, so a script reads the way the domain is discussed.
 A model's `definition` carries its attributes, keys and constraints — ingest validates every
 record against them, so widening or tightening one changes what the pipeline accepts.
 
+### A source that feeds several models
+
+A source feeds one or more models — its **targets** — and each target has its own source
+key and field map. A CRM export that carries customers and their addresses is one source
+with two targets, registered with `targets=` instead of `target_model=`:
+
+```python
+from masterly import SourceTarget
+
+client.sources.create(
+    "crm",
+    display_name="Salesforce CRM",
+    targets=[
+        SourceTarget("Customer", source_key="customer_number"),
+        SourceTarget("Address", source_key="address_id", field_map={"ADDR_ID": "address_id"}),
+    ],
+)
+```
+
+A batch lands in **one** target, so a push to such a source names it with `model=` — one call
+per model. On a source with one target `model` may be left out, and the call sends exactly
+what it always sent; on a source with several, an unnamed batch is refused with
+`SOURCE_TARGET_REQUIRED` before anything is queued, `error.details["targets"]` listing them:
+
+```python
+client.sources.ingest("crm", customers, model="Customer")
+client.sources.ingest("crm", addresses, model="Address")
+client.sources.upload_csv("crm", open("addresses.csv").read(), model="Address", delimiter=";")
+
+with client.sources.full_load("crm", model="Address") as load:   # one target's snapshot
+    load.send(every_address)
+```
+
+Records, quarantine and counts are per target too — a source key names one record within a
+target, so the same key under two targets is two records:
+
+```python
+for row in client.sources.quarantine("crm", model="Address", status="open"):
+    print(row["model_name"], row["reason_code"], row["payload"])
+
+stats = client.sources.stats("crm")           # the whole source, plus `targets[]` per model
+stats = client.sources.stats("crm", model="Address")
+```
+
+`client.sources.targets` lists, adds, reads, edits and removes a source's targets. A target
+has a revision of its own, stated on its governed writes; adding one needs a model with a
+published version, and only a target that holds no records — deleted ones included — can be
+removed, never the last one:
+
+```python
+client.sources.targets.add("crm", "Address", source_key="address_id")
+target = client.sources.targets.get("crm", "Address")
+target["mapping"]["field_map"]["STREET"] = "street"
+client.sources.targets.update("crm", "Address", mapping=target["mapping"], if_match=target["version"])
+client.sources.targets.remove("crm", "Address", if_match=target["version"])   # only while empty
+```
+
+On what `client.sources.get` returns, `targets[]` carries every target; the top-level
+`target_model`, `mapping`, `profile` and `drift` are **deprecated** and describe the first
+target — on a one-target source, exactly what they always did. The same goes for `target_model=`
+and `mapping=` on `client.sources.update`, which write the first target; edit any target through
+`client.sources.targets.update`. These calls talk to routes that are not in a published release
+of the API yet; an install on an earlier release answers them `404 NOT_FOUND`, and sends every
+batch to its one target.
+
 Editing a model or a source is a **governed write**: more than one person edits them, and the
 edit is derived from a read, so `update` states the revision it replaces (ADR 0070) and a stale
 one is refused rather than silently overwriting:
@@ -404,6 +480,45 @@ A refused write raises `ApiError`; `error.conflict` names the fields that moved.
 without a typed method is reached through `client.request(...)`, which takes the same
 `if_match`.
 
+### Deactivate, reactivate or delete a source
+
+A source has a `status`: `active`, or `inactive` once deactivated. An **inactive** source admits
+nothing — a pushed batch, a CSV upload, a full load's next batch and a quarantine retry are all
+refused with `409 SOURCE_INACTIVE` and nothing is queued. Everything it already landed stays,
+and **its records keep contributing to golden records**: nothing is recomputed, in either
+direction. Reactivating admits records again with nothing lost. Both are governed writes, so
+they take the source's `version` as `if_match`:
+
+```python
+source = client.sources.get("erp")
+client.sources.deactivate("erp", if_match=source["version"])   # status: "inactive"
+
+source = client.sources.get("erp")
+client.sources.reactivate("erp", if_match=source["version"])   # status: "active"
+```
+
+A **delete** removes the source and everything it landed — its records in every target and
+their history, quarantine, profile, drift, run history, schedule and sealed connection — and
+recomputes the golden records it contributed to without it; an entity left with no live
+record clears. It is heavy work, so it runs as a job: the source is marked `deleting` at once,
+the answer is the job receipt, and `client.jobs.wait` follows the removal to its end. Read
+what it would touch first, and ask a person to type the source's `name` before you send it:
+
+```python
+impact = client.sources.deletion_impact("erp")
+print(impact["targets"], impact["golden"]["would_change"], impact["golden"]["would_clear"])
+if impact["deletable"] and typed_name == impact["name"]:
+    source = client.sources.get("erp")
+    receipt = client.sources.delete("erp", if_match=source["version"])
+    client.jobs.wait(receipt["job_id"])
+```
+
+The delete is refused with `409 SOURCE_IN_USE` while a data product reads the source through a
+raw relation — `error.details["dependents"]` names each one; change or delete the product
+first — and the model's built-in manual source can be neither deactivated nor deleted
+(`409 SOURCE_BUILT_IN`). The four calls are session routes. They talk to routes that are not
+in a published release yet; an install on an earlier release answers them `404 NOT_FOUND`.
+
 ## Command line: `masterly config migrate`
 
 The configuration files in a GitOps repository follow a schema that Masterly versions. A

{masterly-0.4.0 → masterly-0.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "masterly"
-version = "0.4.0"
+version = "0.5.0"
 description = "Python client for Masterly — extract governed data products and ingest records from your lakehouse"
 readme = "README.md"
 requires-python = ">=3.10"

{masterly-0.4.0 → masterly-0.5.0}/src/masterly/__init__.py

@@ -20,10 +20,10 @@ personas" says how to get either one.
 
 from masterly._client import ApiError, Client, Persona
 from masterly._extract import ChangeFeed, RowPages
-from masterly._ingest import FullLoad, IngestMode, IngestReceipt, IngestReport
+from masterly._ingest import FullLoad, IngestMode, IngestReceipt, IngestReport, SourceTarget
 from masterly._precondition import Conflict, Precondition
 
-__version__ = "0.4.0"
+__version__ = "0.5.0"
 
 __all__ = [
     "ApiError",
@@ -37,5 +37,6 @@ __all__ = [
     "Persona",
     "Precondition",
     "RowPages",
+    "SourceTarget",
     "__version__",
 ]

{masterly-0.4.0 → masterly-0.5.0}/src/masterly/_client.py

@@ -283,7 +283,16 @@ class Client:
         response = self._http.request(method, path, params=params, json=json, headers=sent or None)
         return self._answer(response)
 
-    def _keyed_request(self, method: str, path: str, *, json: Any) -> Any:
+    def _keyed_request(
+        self,
+        method: str,
+        path: str,
+        *,
+        json: Any = None,
+        params: dict[str, Any] | None = None,
+        content: bytes | None = None,
+        headers: Mapping[str, str] | None = None,
+    ) -> Any:
         """Send a write the server de-duplicates by ``Idempotency-Key``, retrying it with the
         same key when the connection fails before an answer arrives.
 
@@ -291,12 +300,15 @@ class Client:
         reached us is indistinguishable, here, from one that never got through. Re-sending with
         the same key makes that ambiguity harmless: the server replays the answer it already
         gave instead of applying the write a second time. An answer — any status — is never
-        retried; it raises or returns as usual.
+        retried; it raises or returns as usual. The body is ``json``, or raw ``content`` (a
+        CSV file) with its own ``Content-Type`` in ``headers``.
         """
-        headers = {_IDEMPOTENCY_KEY: str(uuid.uuid4())}
+        sent = {**(headers or {}), _IDEMPOTENCY_KEY: str(uuid.uuid4())}
         for backoff in (*_KEYED_RETRY_BACKOFF, None):
             try:
-                response = self._http.request(method, path, json=json, headers=headers)
+                response = self._http.request(
+                    method, path, params=params, json=json, content=content, headers=sent
+                )
             except httpx.TransportError:
                 if backoff is None:
                     raise