masterly 0.4.0__tar.gz → 0.5.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- masterly-0.5.1/.github/dependency-review-config.yml +23 -0
- {masterly-0.4.0 → masterly-0.5.1}/.github/workflows/ci.yml +1 -1
- masterly-0.5.1/.github/workflows/dependency-review.yml +41 -0
- {masterly-0.4.0 → masterly-0.5.1}/.github/workflows/release.yml +1 -1
- masterly-0.5.1/AGENTS.md +1 -0
- masterly-0.5.1/CLAUDE.md +72 -0
- {masterly-0.4.0 → masterly-0.5.1}/PKG-INFO +117 -2
- {masterly-0.4.0 → masterly-0.5.1}/README.md +116 -1
- masterly-0.5.1/REVIEW.md +47 -0
- {masterly-0.4.0 → masterly-0.5.1}/pyproject.toml +1 -1
- {masterly-0.4.0 → masterly-0.5.1}/src/masterly/__init__.py +3 -2
- {masterly-0.4.0 → masterly-0.5.1}/src/masterly/_client.py +16 -4
- masterly-0.5.1/src/masterly/_ingest.py +878 -0
- masterly-0.5.1/tests/test_source_ignored_fields.py +163 -0
- masterly-0.5.1/tests/test_source_lifecycle.py +334 -0
- masterly-0.5.1/tests/test_source_targets.py +551 -0
- {masterly-0.4.0 → masterly-0.5.1}/uv.lock +255 -189
- masterly-0.4.0/src/masterly/_ingest.py +0 -466
- {masterly-0.4.0 → masterly-0.5.1}/.claude/PROJECT-CLAUDE.md +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/.claude/hooks/project-claude-md.sh +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/.claude/settings.json +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/.github/dependabot.yml +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/.github/workflows/workflow-hardening.yml +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/.github/zizmor.yml +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/.gitignore +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/LICENSE +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/RELEASING.md +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/SECURITY.md +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/examples/README.md +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/examples/demo_data.py +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/scripts/check_release_record.py +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/src/masterly/_config.py +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/src/masterly/_extract.py +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/src/masterly/_jobs.py +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/src/masterly/_paging.py +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/src/masterly/_precondition.py +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/src/masterly/cli/__init__.py +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/src/masterly/cli/__main__.py +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/src/masterly/cli/_engine.py +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/src/masterly/cli/_schema.py +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/src/masterly/py.typed +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/tests/test_cli_migrate.py +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/tests/test_client.py +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/tests/test_config.py +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/tests/test_demo_data.py +0 -0
- {masterly-0.4.0 → masterly-0.5.1}/tests/test_receipts_jobs_loads.py +0 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Licence allow-list for the dependency-review job (.github/workflows/dependency-review.yml).
|
|
2
|
+
#
|
|
3
|
+
# Source: the permissive licences that can be combined with this package's own Apache-2.0
|
|
4
|
+
# licence and redistributed from PyPI -- Apache-2.0 itself, and the permissive families
|
|
5
|
+
# (MIT, BSD, ISC, PSF, Zlib, Unlicense/CC0, MPL-2.0 as file-level copyleft that does not reach the
|
|
6
|
+
# SDK's own code). Strong or network copyleft (GPL, AGPL, LGPL-3 static) and non-commercial or
|
|
7
|
+
# source-available licences are deliberately absent: a dependency under one fails the check.
|
|
8
|
+
# Identifiers are SPDX. Adding one is an owner decision.
|
|
9
|
+
allow-licenses:
|
|
10
|
+
- Apache-2.0
|
|
11
|
+
- MIT
|
|
12
|
+
- MIT-0
|
|
13
|
+
- BSD-2-Clause
|
|
14
|
+
- BSD-3-Clause
|
|
15
|
+
- 0BSD
|
|
16
|
+
- ISC
|
|
17
|
+
- PSF-2.0
|
|
18
|
+
- Python-2.0
|
|
19
|
+
- Unlicense
|
|
20
|
+
- CC0-1.0
|
|
21
|
+
- Zlib
|
|
22
|
+
- MPL-2.0
|
|
23
|
+
- BlueOak-1.0.0
|
|
@@ -44,7 +44,7 @@ jobs:
|
|
|
44
44
|
- uses: actions/checkout@v7
|
|
45
45
|
with:
|
|
46
46
|
persist-credentials: false
|
|
47
|
-
- uses: astral-sh/setup-uv@
|
|
47
|
+
- uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10, pinned by digest
|
|
48
48
|
with:
|
|
49
49
|
python-version: ${{ matrix.python }}
|
|
50
50
|
- run: uv sync
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# Fails a pull request that introduces a dependency with a known high-or-worse advisory, or one
|
|
2
|
+
# whose licence is outside .github/dependency-review-config.yml (MAS-2155 is the backend twin).
|
|
3
|
+
# This package ships to PyPI, so what it depends on reaches customers' environments.
|
|
4
|
+
#
|
|
5
|
+
# The required-status-check name for the ruleset is the job name: `dependency-review`. Adding it
|
|
6
|
+
# to the ruleset is the owner's change.
|
|
7
|
+
name: dependency-review
|
|
8
|
+
|
|
9
|
+
# Quoted because bare `on` is a YAML 1.1 boolean.
|
|
10
|
+
"on":
|
|
11
|
+
pull_request:
|
|
12
|
+
# The merge queue tests the speculative merge. Without this trigger a required check never
|
|
13
|
+
# reports for a queued entry and the queue stalls until it times out.
|
|
14
|
+
merge_group:
|
|
15
|
+
|
|
16
|
+
# Deliberately no path filter. A required check that does not run on a pull request blocks it
|
|
17
|
+
# forever; with no dependency change the action reports an empty diff and exits 0 in seconds.
|
|
18
|
+
|
|
19
|
+
concurrency:
|
|
20
|
+
group: ${{ github.workflow }}-${{ github.ref }}
|
|
21
|
+
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
|
22
|
+
|
|
23
|
+
permissions: {}
|
|
24
|
+
|
|
25
|
+
jobs:
|
|
26
|
+
dependency-review:
|
|
27
|
+
runs-on: ubuntu-latest
|
|
28
|
+
permissions:
|
|
29
|
+
contents: read # actions/checkout, and the dependency-graph compare API
|
|
30
|
+
steps:
|
|
31
|
+
# persist-credentials: false -- nothing here pushes, so the token has no job after the checkout.
|
|
32
|
+
- uses: actions/checkout@v7
|
|
33
|
+
with:
|
|
34
|
+
persist-credentials: false
|
|
35
|
+
# Pinned by full commit SHA: MAS-2159 asks for it, which is stricter than the `actions/*`
|
|
36
|
+
# ref-pin policy in .github/zizmor.yml.
|
|
37
|
+
- uses: actions/dependency-review-action@5a2ce3f5b92ee19cbb1541a4984c76d921601d7c # v4.3.4, pinned by digest
|
|
38
|
+
with:
|
|
39
|
+
fail-on-severity: high
|
|
40
|
+
config-file: ./.github/dependency-review-config.yml
|
|
41
|
+
comment-summary-in-pr: never
|
|
@@ -75,7 +75,7 @@ jobs:
|
|
|
75
75
|
# entry poisoned by any earlier job holding this repository's GITHUB_TOKEN would feed the
|
|
76
76
|
# build backend straight into what ships. A cold cache costs seconds here; the wheel is
|
|
77
77
|
# built from the tag alone, with everything fetched fresh from PyPI.
|
|
78
|
-
- uses: astral-sh/setup-uv@
|
|
78
|
+
- uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10, pinned by digest
|
|
79
79
|
with:
|
|
80
80
|
enable-cache: false
|
|
81
81
|
- run: uv build
|
masterly-0.5.1/AGENTS.md
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
CLAUDE.md
|
masterly-0.5.1/CLAUDE.md
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# masterly-python-sdk
|
|
2
|
+
|
|
3
|
+
The public (Apache-2.0) Python client for Masterly, published to PyPI as `masterly`
|
|
4
|
+
(`pyproject.toml`). It is a hand-written client over the stable `/v1` REST API, not generated
|
|
5
|
+
code and not a second contract: it adds ergonomics to endpoints the server already
|
|
6
|
+
publishes (ADR 0068). Sync `httpx`, Python >= 3.10, typed (`src/masterly/py.typed`).
|
|
7
|
+
|
|
8
|
+
This repository is public. Write everything in it — code, comments, commit messages, PR
|
|
9
|
+
text — as something a customer will read.
|
|
10
|
+
|
|
11
|
+
## Layout
|
|
12
|
+
|
|
13
|
+
- `src/masterly/` — the package; public names are listed in `__all__` in `__init__.py`
|
|
14
|
+
- `src/masterly/cli/` — the `masterly` console script (`masterly config migrate`)
|
|
15
|
+
- `tests/` — pytest suite; `examples/demo_data.py` — demo master-data generator
|
|
16
|
+
- `scripts/check_release_record.py` — the release-record check (see `RELEASING.md`)
|
|
17
|
+
|
|
18
|
+
## Commands
|
|
19
|
+
|
|
20
|
+
Run from the repository root; `uv` manages the environment (`uv.lock` is committed).
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
uv sync # install, dev group included
|
|
24
|
+
uv run pytest -q # tests
|
|
25
|
+
uv run ruff check # lint (rules and line length in pyproject.toml)
|
|
26
|
+
uv run mypy src # types; strict, on the package only
|
|
27
|
+
uv build # sdist and wheel, built with hatchling
|
|
28
|
+
python3 scripts/check_release_record.py --selftest && python3 scripts/check_release_record.py
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
`.github/workflows/ci.yml` runs ruff, mypy and pytest on Python 3.10 and 3.13, plus the
|
|
32
|
+
release-record check. A change is gated on all of it.
|
|
33
|
+
|
|
34
|
+
## Versioning and release
|
|
35
|
+
|
|
36
|
+
- Semver. The version is set in two places that must agree: `pyproject.toml` (`version`)
|
|
37
|
+
and `src/masterly/__init__.py` (`__version__`, sent in the `User-Agent` header).
|
|
38
|
+
- The SDK's version tracks the `/v1` contract and is independent of the application's (ADR 0068).
|
|
39
|
+
- A release is a version-bump PR to `main`, then an annotated `vX.Y.Z` tag on the merged
|
|
40
|
+
commit. Pushing the tag starts `.github/workflows/release.yml`, which publishes to PyPI by
|
|
41
|
+
trusted publishing (OIDC, no tokens) through the `pypi` environment.
|
|
42
|
+
- PyPI versions are immutable, and the workflow refuses a tag that does not match
|
|
43
|
+
`pyproject.toml`, is not on the default branch, or claims 0.1.1 or 0.2.0. Never tag,
|
|
44
|
+
publish or re-use a version without being asked to. Details: `RELEASING.md`.
|
|
45
|
+
|
|
46
|
+
## Documentation
|
|
47
|
+
|
|
48
|
+
- The SDK reference on masterlydata.com (`Documentation` URL in `pyproject.toml`) documents
|
|
49
|
+
the **released** SDK. Do not describe unreleased behaviour there; `README.md` is what
|
|
50
|
+
PyPI shows, so it must also be true of the released package.
|
|
51
|
+
- There is no `CHANGELOG.md` in this repository. The changelog is the docs "What's new"
|
|
52
|
+
page (`Changelog` URL in `pyproject.toml`), which lives outside this repository.
|
|
53
|
+
|
|
54
|
+
## Don't
|
|
55
|
+
|
|
56
|
+
- Don't make a breaking change to the public API (anything in `__all__`, method signatures,
|
|
57
|
+
exception types, the `masterly` console script) without a major version.
|
|
58
|
+
- Don't add a runtime dependency beyond `httpx` without a stated reason. Optional features
|
|
59
|
+
go in an extra (`pandas`, `cli` in `pyproject.toml`), never in `dependencies`.
|
|
60
|
+
- Don't call endpoints outside stable `/v1` paths.
|
|
61
|
+
- Don't edit `uv.lock` by hand; change `pyproject.toml` and re-lock.
|
|
62
|
+
- Don't weaken a test, a ruff rule or a strict mypy setting to get a change through.
|
|
63
|
+
- Don't put credentials, tokens or customer data in code, tests, examples or comments.
|
|
64
|
+
|
|
65
|
+
## Done means
|
|
66
|
+
|
|
67
|
+
- `ruff check`, `mypy src` and `pytest -q` pass.
|
|
68
|
+
- New behaviour has a test; the existing tests were not weakened.
|
|
69
|
+
- `examples/demo_data.py` still runs: `uv run examples/demo_data.py --dry-run --customers 5`
|
|
70
|
+
(`examples/README.md`); `tests/test_demo_data.py` covers the generator.
|
|
71
|
+
- A user-visible change is noted for the changelog and `README.md` is updated if it is wrong.
|
|
72
|
+
- `REVIEW.md` is the checklist a reviewer applies.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: masterly
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.1
|
|
4
4
|
Summary: Python client for Masterly — extract governed data products and ingest records from your lakehouse
|
|
5
5
|
Project-URL: Homepage, https://masterlydata.com
|
|
6
6
|
Project-URL: Documentation, https://masterlydata.com/docs/reference/python-sdk/
|
|
@@ -222,7 +222,9 @@ recomputes without it, and its history stays readable. A delete for a key the so
|
|
|
222
222
|
hold does nothing.
|
|
223
223
|
|
|
224
224
|
When a system can only hand over everything it has, send it as a **full snapshot**: every live
|
|
225
|
-
record of the
|
|
225
|
+
record of the target that the snapshot does not carry is deleted after the upserts (a source
|
|
226
|
+
that [feeds several models](#a-source-that-feeds-several-models) names the target with
|
|
227
|
+
`model=`; its other targets are untouched).
|
|
226
228
|
|
|
227
229
|
```python
|
|
228
230
|
client.sources.ingest("crm", every_customer, mode="full", batch_size=5000)
|
|
@@ -413,10 +415,84 @@ source = client.sources.get("erp")
|
|
|
413
415
|
client.sources.update("erp", display_name="SAP ERP (Sweden)", if_match=source["version"])
|
|
414
416
|
```
|
|
415
417
|
|
|
418
|
+
A field the field map does not name passes through under its own name, so a field named like a
|
|
419
|
+
model attribute lands on it. When such a field is not meant for the model, name it in
|
|
420
|
+
`ignored_fields=` (on `create`, `SourceTarget` and `targets.add`): it is dropped before the field
|
|
421
|
+
map, lands on no attribute and is not reported as schema drift. It comes back as
|
|
422
|
+
`mapping["ignored_fields"]`; to change it later, edit the mapping you read and pass it to
|
|
423
|
+
`update` (or `targets.update`). An install on an API release before this field existed drops
|
|
424
|
+
`ignored_fields` without an error and returns no such key, so read the mapping back to confirm it
|
|
425
|
+
was kept.
|
|
426
|
+
|
|
416
427
|
Everything takes an id or an exact name, so a script reads the way the domain is discussed.
|
|
417
428
|
A model's `definition` carries its attributes, keys and constraints — ingest validates every
|
|
418
429
|
record against them, so widening or tightening one changes what the pipeline accepts.
|
|
419
430
|
|
|
431
|
+
### A source that feeds several models
|
|
432
|
+
|
|
433
|
+
A source feeds one or more models — its **targets** — and each target has its own source
|
|
434
|
+
key and field map. A CRM export that carries customers and their addresses is one source
|
|
435
|
+
with two targets, registered with `targets=` instead of `target_model=`:
|
|
436
|
+
|
|
437
|
+
```python
|
|
438
|
+
from masterly import SourceTarget
|
|
439
|
+
|
|
440
|
+
client.sources.create(
|
|
441
|
+
"crm",
|
|
442
|
+
display_name="Salesforce CRM",
|
|
443
|
+
targets=[
|
|
444
|
+
SourceTarget("Customer", source_key="customer_number"),
|
|
445
|
+
SourceTarget("Address", source_key="address_id", field_map={"ADDR_ID": "address_id"}),
|
|
446
|
+
],
|
|
447
|
+
)
|
|
448
|
+
```
|
|
449
|
+
|
|
450
|
+
A batch lands in **one** target, so a push to such a source names it with `model=` — one call
|
|
451
|
+
per model. On a source with one target `model` may be left out, and the call sends exactly
|
|
452
|
+
what it always sent; on a source with several, an unnamed batch is refused with
|
|
453
|
+
`SOURCE_TARGET_REQUIRED` before anything is queued, `error.details["targets"]` listing them:
|
|
454
|
+
|
|
455
|
+
```python
|
|
456
|
+
client.sources.ingest("crm", customers, model="Customer")
|
|
457
|
+
client.sources.ingest("crm", addresses, model="Address")
|
|
458
|
+
client.sources.upload_csv("crm", open("addresses.csv").read(), model="Address", delimiter=";")
|
|
459
|
+
|
|
460
|
+
with client.sources.full_load("crm", model="Address") as load: # one target's snapshot
|
|
461
|
+
load.send(every_address)
|
|
462
|
+
```
|
|
463
|
+
|
|
464
|
+
Records, quarantine and counts are per target too — a source key names one record within a
|
|
465
|
+
target, so the same key under two targets is two records:
|
|
466
|
+
|
|
467
|
+
```python
|
|
468
|
+
for row in client.sources.quarantine("crm", model="Address", status="open"):
|
|
469
|
+
print(row["model_name"], row["reason_code"], row["payload"])
|
|
470
|
+
|
|
471
|
+
stats = client.sources.stats("crm") # the whole source, plus `targets[]` per model
|
|
472
|
+
stats = client.sources.stats("crm", model="Address")
|
|
473
|
+
```
|
|
474
|
+
|
|
475
|
+
`client.sources.targets` lists, adds, reads, edits and removes a source's targets. A target
|
|
476
|
+
has a revision of its own, stated on its governed writes; adding one needs a model with a
|
|
477
|
+
published version, and only a target that holds no records — deleted ones included — can be
|
|
478
|
+
removed, never the last one:
|
|
479
|
+
|
|
480
|
+
```python
|
|
481
|
+
client.sources.targets.add("crm", "Address", source_key="address_id")
|
|
482
|
+
target = client.sources.targets.get("crm", "Address")
|
|
483
|
+
target["mapping"]["field_map"]["STREET"] = "street"
|
|
484
|
+
client.sources.targets.update("crm", "Address", mapping=target["mapping"], if_match=target["version"])
|
|
485
|
+
client.sources.targets.remove("crm", "Address", if_match=target["version"]) # only while empty
|
|
486
|
+
```
|
|
487
|
+
|
|
488
|
+
On what `client.sources.get` returns, `targets[]` carries every target; the top-level
|
|
489
|
+
`target_model`, `mapping`, `profile` and `drift` are **deprecated** and describe the first
|
|
490
|
+
target — on a one-target source, exactly what they always did. The same goes for `target_model=`
|
|
491
|
+
and `mapping=` on `client.sources.update`, which write the first target; edit any target through
|
|
492
|
+
`client.sources.targets.update`. These calls talk to routes that are not in a published release
|
|
493
|
+
of the API yet; an install on an earlier release answers them `404 NOT_FOUND`, and sends every
|
|
494
|
+
batch to its one target.
|
|
495
|
+
|
|
420
496
|
Editing a model or a source is a **governed write**: more than one person edits them, and the
|
|
421
497
|
edit is derived from a read, so `update` states the revision it replaces (ADR 0070) and a stale
|
|
422
498
|
one is refused rather than silently overwriting:
|
|
@@ -433,6 +509,45 @@ A refused write raises `ApiError`; `error.conflict` names the fields that moved.
|
|
|
433
509
|
without a typed method is reached through `client.request(...)`, which takes the same
|
|
434
510
|
`if_match`.
|
|
435
511
|
|
|
512
|
+
### Deactivate, reactivate or delete a source
|
|
513
|
+
|
|
514
|
+
A source has a `status`: `active`, or `inactive` once deactivated. An **inactive** source admits
|
|
515
|
+
nothing — a pushed batch, a CSV upload, a full load's next batch and a quarantine retry are all
|
|
516
|
+
refused with `409 SOURCE_INACTIVE` and nothing is queued. Everything it already landed stays,
|
|
517
|
+
and **its records keep contributing to golden records**: nothing is recomputed, in either
|
|
518
|
+
direction. Reactivating admits records again with nothing lost. Both are governed writes, so
|
|
519
|
+
they take the source's `version` as `if_match`:
|
|
520
|
+
|
|
521
|
+
```python
|
|
522
|
+
source = client.sources.get("erp")
|
|
523
|
+
client.sources.deactivate("erp", if_match=source["version"]) # status: "inactive"
|
|
524
|
+
|
|
525
|
+
source = client.sources.get("erp")
|
|
526
|
+
client.sources.reactivate("erp", if_match=source["version"]) # status: "active"
|
|
527
|
+
```
|
|
528
|
+
|
|
529
|
+
A **delete** removes the source and everything it landed — its records in every target and
|
|
530
|
+
their history, quarantine, profile, drift, run history, schedule and sealed connection — and
|
|
531
|
+
recomputes the golden records it contributed to without it; an entity left with no live
|
|
532
|
+
record clears. It is heavy work, so it runs as a job: the source is marked `deleting` at once,
|
|
533
|
+
the answer is the job receipt, and `client.jobs.wait` follows the removal to its end. Read
|
|
534
|
+
what it would touch first, and ask a person to type the source's `name` before you send it:
|
|
535
|
+
|
|
536
|
+
```python
|
|
537
|
+
impact = client.sources.deletion_impact("erp")
|
|
538
|
+
print(impact["targets"], impact["golden"]["would_change"], impact["golden"]["would_clear"])
|
|
539
|
+
if impact["deletable"] and typed_name == impact["name"]:
|
|
540
|
+
source = client.sources.get("erp")
|
|
541
|
+
receipt = client.sources.delete("erp", if_match=source["version"])
|
|
542
|
+
client.jobs.wait(receipt["job_id"])
|
|
543
|
+
```
|
|
544
|
+
|
|
545
|
+
The delete is refused with `409 SOURCE_IN_USE` while a data product reads the source through a
|
|
546
|
+
raw relation — `error.details["dependents"]` names each one; change or delete the product
|
|
547
|
+
first — and the model's built-in manual source can be neither deactivated nor deleted
|
|
548
|
+
(`409 SOURCE_BUILT_IN`). The four calls are session routes. They talk to routes that are not
|
|
549
|
+
in a published release yet; an install on an earlier release answers them `404 NOT_FOUND`.
|
|
550
|
+
|
|
436
551
|
## Command line: `masterly config migrate`
|
|
437
552
|
|
|
438
553
|
The configuration files in a GitOps repository follow a schema that Masterly versions. A
|
|
@@ -193,7 +193,9 @@ recomputes without it, and its history stays readable. A delete for a key the so
|
|
|
193
193
|
hold does nothing.
|
|
194
194
|
|
|
195
195
|
When a system can only hand over everything it has, send it as a **full snapshot**: every live
|
|
196
|
-
record of the
|
|
196
|
+
record of the target that the snapshot does not carry is deleted after the upserts (a source
|
|
197
|
+
that [feeds several models](#a-source-that-feeds-several-models) names the target with
|
|
198
|
+
`model=`; its other targets are untouched).
|
|
197
199
|
|
|
198
200
|
```python
|
|
199
201
|
client.sources.ingest("crm", every_customer, mode="full", batch_size=5000)
|
|
@@ -384,10 +386,84 @@ source = client.sources.get("erp")
|
|
|
384
386
|
client.sources.update("erp", display_name="SAP ERP (Sweden)", if_match=source["version"])
|
|
385
387
|
```
|
|
386
388
|
|
|
389
|
+
A field the field map does not name passes through under its own name, so a field named like a
|
|
390
|
+
model attribute lands on it. When such a field is not meant for the model, name it in
|
|
391
|
+
`ignored_fields=` (on `create`, `SourceTarget` and `targets.add`): it is dropped before the field
|
|
392
|
+
map, lands on no attribute and is not reported as schema drift. It comes back as
|
|
393
|
+
`mapping["ignored_fields"]`; to change it later, edit the mapping you read and pass it to
|
|
394
|
+
`update` (or `targets.update`). An install on an API release before this field existed drops
|
|
395
|
+
`ignored_fields` without an error and returns no such key, so read the mapping back to confirm it
|
|
396
|
+
was kept.
|
|
397
|
+
|
|
387
398
|
Everything takes an id or an exact name, so a script reads the way the domain is discussed.
|
|
388
399
|
A model's `definition` carries its attributes, keys and constraints — ingest validates every
|
|
389
400
|
record against them, so widening or tightening one changes what the pipeline accepts.
|
|
390
401
|
|
|
402
|
+
### A source that feeds several models
|
|
403
|
+
|
|
404
|
+
A source feeds one or more models — its **targets** — and each target has its own source
|
|
405
|
+
key and field map. A CRM export that carries customers and their addresses is one source
|
|
406
|
+
with two targets, registered with `targets=` instead of `target_model=`:
|
|
407
|
+
|
|
408
|
+
```python
|
|
409
|
+
from masterly import SourceTarget
|
|
410
|
+
|
|
411
|
+
client.sources.create(
|
|
412
|
+
"crm",
|
|
413
|
+
display_name="Salesforce CRM",
|
|
414
|
+
targets=[
|
|
415
|
+
SourceTarget("Customer", source_key="customer_number"),
|
|
416
|
+
SourceTarget("Address", source_key="address_id", field_map={"ADDR_ID": "address_id"}),
|
|
417
|
+
],
|
|
418
|
+
)
|
|
419
|
+
```
|
|
420
|
+
|
|
421
|
+
A batch lands in **one** target, so a push to such a source names it with `model=` — one call
|
|
422
|
+
per model. On a source with one target `model` may be left out, and the call sends exactly
|
|
423
|
+
what it always sent; on a source with several, an unnamed batch is refused with
|
|
424
|
+
`SOURCE_TARGET_REQUIRED` before anything is queued, `error.details["targets"]` listing them:
|
|
425
|
+
|
|
426
|
+
```python
|
|
427
|
+
client.sources.ingest("crm", customers, model="Customer")
|
|
428
|
+
client.sources.ingest("crm", addresses, model="Address")
|
|
429
|
+
client.sources.upload_csv("crm", open("addresses.csv").read(), model="Address", delimiter=";")
|
|
430
|
+
|
|
431
|
+
with client.sources.full_load("crm", model="Address") as load: # one target's snapshot
|
|
432
|
+
load.send(every_address)
|
|
433
|
+
```
|
|
434
|
+
|
|
435
|
+
Records, quarantine and counts are per target too — a source key names one record within a
|
|
436
|
+
target, so the same key under two targets is two records:
|
|
437
|
+
|
|
438
|
+
```python
|
|
439
|
+
for row in client.sources.quarantine("crm", model="Address", status="open"):
|
|
440
|
+
print(row["model_name"], row["reason_code"], row["payload"])
|
|
441
|
+
|
|
442
|
+
stats = client.sources.stats("crm") # the whole source, plus `targets[]` per model
|
|
443
|
+
stats = client.sources.stats("crm", model="Address")
|
|
444
|
+
```
|
|
445
|
+
|
|
446
|
+
`client.sources.targets` lists, adds, reads, edits and removes a source's targets. A target
|
|
447
|
+
has a revision of its own, stated on its governed writes; adding one needs a model with a
|
|
448
|
+
published version, and only a target that holds no records — deleted ones included — can be
|
|
449
|
+
removed, never the last one:
|
|
450
|
+
|
|
451
|
+
```python
|
|
452
|
+
client.sources.targets.add("crm", "Address", source_key="address_id")
|
|
453
|
+
target = client.sources.targets.get("crm", "Address")
|
|
454
|
+
target["mapping"]["field_map"]["STREET"] = "street"
|
|
455
|
+
client.sources.targets.update("crm", "Address", mapping=target["mapping"], if_match=target["version"])
|
|
456
|
+
client.sources.targets.remove("crm", "Address", if_match=target["version"]) # only while empty
|
|
457
|
+
```
|
|
458
|
+
|
|
459
|
+
On what `client.sources.get` returns, `targets[]` carries every target; the top-level
|
|
460
|
+
`target_model`, `mapping`, `profile` and `drift` are **deprecated** and describe the first
|
|
461
|
+
target — on a one-target source, exactly what they always did. The same goes for `target_model=`
|
|
462
|
+
and `mapping=` on `client.sources.update`, which write the first target; edit any target through
|
|
463
|
+
`client.sources.targets.update`. These calls talk to routes that are not in a published release
|
|
464
|
+
of the API yet; an install on an earlier release answers them `404 NOT_FOUND`, and sends every
|
|
465
|
+
batch to its one target.
|
|
466
|
+
|
|
391
467
|
Editing a model or a source is a **governed write**: more than one person edits them, and the
|
|
392
468
|
edit is derived from a read, so `update` states the revision it replaces (ADR 0070) and a stale
|
|
393
469
|
one is refused rather than silently overwriting:
|
|
@@ -404,6 +480,45 @@ A refused write raises `ApiError`; `error.conflict` names the fields that moved.
|
|
|
404
480
|
without a typed method is reached through `client.request(...)`, which takes the same
|
|
405
481
|
`if_match`.
|
|
406
482
|
|
|
483
|
+
### Deactivate, reactivate or delete a source
|
|
484
|
+
|
|
485
|
+
A source has a `status`: `active`, or `inactive` once deactivated. An **inactive** source admits
|
|
486
|
+
nothing — a pushed batch, a CSV upload, a full load's next batch and a quarantine retry are all
|
|
487
|
+
refused with `409 SOURCE_INACTIVE` and nothing is queued. Everything it already landed stays,
|
|
488
|
+
and **its records keep contributing to golden records**: nothing is recomputed, in either
|
|
489
|
+
direction. Reactivating admits records again with nothing lost. Both are governed writes, so
|
|
490
|
+
they take the source's `version` as `if_match`:
|
|
491
|
+
|
|
492
|
+
```python
|
|
493
|
+
source = client.sources.get("erp")
|
|
494
|
+
client.sources.deactivate("erp", if_match=source["version"]) # status: "inactive"
|
|
495
|
+
|
|
496
|
+
source = client.sources.get("erp")
|
|
497
|
+
client.sources.reactivate("erp", if_match=source["version"]) # status: "active"
|
|
498
|
+
```
|
|
499
|
+
|
|
500
|
+
A **delete** removes the source and everything it landed — its records in every target and
|
|
501
|
+
their history, quarantine, profile, drift, run history, schedule and sealed connection — and
|
|
502
|
+
recomputes the golden records it contributed to without it; an entity left with no live
|
|
503
|
+
record clears. It is heavy work, so it runs as a job: the source is marked `deleting` at once,
|
|
504
|
+
the answer is the job receipt, and `client.jobs.wait` follows the removal to its end. Read
|
|
505
|
+
what it would touch first, and ask a person to type the source's `name` before you send it:
|
|
506
|
+
|
|
507
|
+
```python
|
|
508
|
+
impact = client.sources.deletion_impact("erp")
|
|
509
|
+
print(impact["targets"], impact["golden"]["would_change"], impact["golden"]["would_clear"])
|
|
510
|
+
if impact["deletable"] and typed_name == impact["name"]:
|
|
511
|
+
source = client.sources.get("erp")
|
|
512
|
+
receipt = client.sources.delete("erp", if_match=source["version"])
|
|
513
|
+
client.jobs.wait(receipt["job_id"])
|
|
514
|
+
```
|
|
515
|
+
|
|
516
|
+
The delete is refused with `409 SOURCE_IN_USE` while a data product reads the source through a
|
|
517
|
+
raw relation — `error.details["dependents"]` names each one; change or delete the product
|
|
518
|
+
first — and the model's built-in manual source can be neither deactivated nor deleted
|
|
519
|
+
(`409 SOURCE_BUILT_IN`). The four calls are session routes. They talk to routes that are not
|
|
520
|
+
in a published release yet; an install on an earlier release answers them `404 NOT_FOUND`.
|
|
521
|
+
|
|
407
522
|
## Command line: `masterly config migrate`
|
|
408
523
|
|
|
409
524
|
The configuration files in a GitOps repository follow a schema that Masterly versions. A
|
masterly-0.5.1/REVIEW.md
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Reviewing a change to masterly-python-sdk
|
|
2
|
+
|
|
3
|
+
`CLAUDE.md` says how to work in this repository; this file says what a review blocks on.
|
|
4
|
+
|
|
5
|
+
## What blocks
|
|
6
|
+
|
|
7
|
+
- **The change's stated acceptance criteria are not met.** Check the change against the stated acceptance
|
|
8
|
+
criteria, not against what the diff happens to do.
|
|
9
|
+
- **A public API break without a major version.** Removed or renamed names in `__all__`
|
|
10
|
+
(`src/masterly/__init__.py`), changed signatures or defaults, a new required argument, a
|
|
11
|
+
changed exception type, or a changed `masterly` console script. PyPI versions are
|
|
12
|
+
immutable, so a break cannot be taken back.
|
|
13
|
+
- **A new runtime dependency.** Any change to `dependencies` in `pyproject.toml` beyond
|
|
14
|
+
`httpx`. Optional features belong in an extra.
|
|
15
|
+
- **A missing changelog entry** for a user-visible change. The changelog is the docs
|
|
16
|
+
"What's new" page (`Changelog` URL in `pyproject.toml`), not a file here, so look for the
|
|
17
|
+
entry in the PR description or the linked docs change.
|
|
18
|
+
- **A weakened check:** a skipped or loosened test, a new ruff ignore, a relaxed strict
|
|
19
|
+
mypy setting.
|
|
20
|
+
- **A release that breaks the record:** a version bump that differs between
|
|
21
|
+
`pyproject.toml` and `src/masterly/__init__.py`, or one that reuses 0.1.1 or 0.2.0
|
|
22
|
+
(`RELEASING.md`, `scripts/check_release_record.py`).
|
|
23
|
+
- **Secrets or customer data** anywhere in the diff. The repository is public.
|
|
24
|
+
|
|
25
|
+
## What does not block
|
|
26
|
+
|
|
27
|
+
- Style, formatting and import order that ruff owns (`uv run ruff check`). If the linter
|
|
28
|
+
accepts it, do not comment on it.
|
|
29
|
+
- Typing details mypy accepts under its strict configuration.
|
|
30
|
+
- Dependabot bumps of dev tools and actions that CI passes (`.github/dependabot.yml`).
|
|
31
|
+
- Wording preferences in comments and docs that are accurate.
|
|
32
|
+
|
|
33
|
+
## Known traps
|
|
34
|
+
|
|
35
|
+
- **The version lives in two places**, `pyproject.toml` and `src/masterly/__init__.py`.
|
|
36
|
+
Updating one passes CI's checks but ships a wrong `User-Agent` and `--version`.
|
|
37
|
+
- **`README.md` ships to PyPI** as the project page, so a README edit is a published change
|
|
38
|
+
once released, and must describe the released package only.
|
|
39
|
+
- **Two published versions have no tag here** (0.1.1, 0.2.0). Do not "fix" that by tagging
|
|
40
|
+
an unrelated commit; `RELEASING.md` explains why.
|
|
41
|
+
- **mypy runs on `src` only.** Tests and `examples/` are not type-checked, so a typing
|
|
42
|
+
problem there will not fail CI.
|
|
43
|
+
- **`examples/` is not covered by `pytest`** except through `tests/test_demo_data.py`;
|
|
44
|
+
run the example with `--dry-run` when a change touches the client surface it uses.
|
|
45
|
+
- **Governed writes need a revision.** `data_models.update` requires the precondition
|
|
46
|
+
version; do not make it optional to simplify a call site.
|
|
47
|
+
- **`uv.lock` is regenerated, not edited.** A hand-edited lockfile is a review finding.
|
|
@@ -20,10 +20,10 @@ personas" says how to get either one.
|
|
|
20
20
|
|
|
21
21
|
from masterly._client import ApiError, Client, Persona
|
|
22
22
|
from masterly._extract import ChangeFeed, RowPages
|
|
23
|
-
from masterly._ingest import FullLoad, IngestMode, IngestReceipt, IngestReport
|
|
23
|
+
from masterly._ingest import FullLoad, IngestMode, IngestReceipt, IngestReport, SourceTarget
|
|
24
24
|
from masterly._precondition import Conflict, Precondition
|
|
25
25
|
|
|
26
|
-
__version__ = "0.
|
|
26
|
+
__version__ = "0.5.1"
|
|
27
27
|
|
|
28
28
|
__all__ = [
|
|
29
29
|
"ApiError",
|
|
@@ -37,5 +37,6 @@ __all__ = [
|
|
|
37
37
|
"Persona",
|
|
38
38
|
"Precondition",
|
|
39
39
|
"RowPages",
|
|
40
|
+
"SourceTarget",
|
|
40
41
|
"__version__",
|
|
41
42
|
]
|
|
@@ -283,7 +283,16 @@ class Client:
|
|
|
283
283
|
response = self._http.request(method, path, params=params, json=json, headers=sent or None)
|
|
284
284
|
return self._answer(response)
|
|
285
285
|
|
|
286
|
-
def _keyed_request(
|
|
286
|
+
def _keyed_request(
|
|
287
|
+
self,
|
|
288
|
+
method: str,
|
|
289
|
+
path: str,
|
|
290
|
+
*,
|
|
291
|
+
json: Any = None,
|
|
292
|
+
params: dict[str, Any] | None = None,
|
|
293
|
+
content: bytes | None = None,
|
|
294
|
+
headers: Mapping[str, str] | None = None,
|
|
295
|
+
) -> Any:
|
|
287
296
|
"""Send a write the server de-duplicates by ``Idempotency-Key``, retrying it with the
|
|
288
297
|
same key when the connection fails before an answer arrives.
|
|
289
298
|
|
|
@@ -291,12 +300,15 @@ class Client:
|
|
|
291
300
|
reached us is indistinguishable, here, from one that never got through. Re-sending with
|
|
292
301
|
the same key makes that ambiguity harmless: the server replays the answer it already
|
|
293
302
|
gave instead of applying the write a second time. An answer — any status — is never
|
|
294
|
-
retried; it raises or returns as usual.
|
|
303
|
+
retried; it raises or returns as usual. The body is ``json``, or raw ``content`` (a
|
|
304
|
+
CSV file) with its own ``Content-Type`` in ``headers``.
|
|
295
305
|
"""
|
|
296
|
-
|
|
306
|
+
sent = {**(headers or {}), _IDEMPOTENCY_KEY: str(uuid.uuid4())}
|
|
297
307
|
for backoff in (*_KEYED_RETRY_BACKOFF, None):
|
|
298
308
|
try:
|
|
299
|
-
response = self._http.request(
|
|
309
|
+
response = self._http.request(
|
|
310
|
+
method, path, params=params, json=json, content=content, headers=sent
|
|
311
|
+
)
|
|
300
312
|
except httpx.TransportError:
|
|
301
313
|
if backoff is None:
|
|
302
314
|
raise
|