aice-data-quality 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. aice_data_quality-0.1.0/.gitignore +4 -0
  2. aice_data_quality-0.1.0/CHANGELOG.md +22 -0
  3. aice_data_quality-0.1.0/PKG-INFO +79 -0
  4. aice_data_quality-0.1.0/README.md +41 -0
  5. aice_data_quality-0.1.0/contract/CHANGELOG.md +15 -0
  6. aice_data_quality-0.1.0/contract/DECISIONS.md +44 -0
  7. aice_data_quality-0.1.0/contract/README.md +106 -0
  8. aice_data_quality-0.1.0/contract/v1/examples/invalid/bad_check_id.json +18 -0
  9. aice_data_quality-0.1.0/contract/v1/examples/invalid/extra_field.json +23 -0
  10. aice_data_quality-0.1.0/contract/v1/examples/invalid/local_timestamp.json +18 -0
  11. aice_data_quality-0.1.0/contract/v1/examples/invalid/missing_owner.json +17 -0
  12. aice_data_quality-0.1.0/contract/v1/examples/invalid/run_outcome_without_finished.json +11 -0
  13. aice_data_quality-0.1.0/contract/v1/examples/invalid/too_many_keys.json +121 -0
  14. aice_data_quality-0.1.0/contract/v1/examples/invalid/unknown_severity.json +18 -0
  15. aice_data_quality-0.1.0/contract/v1/examples/valid/check_error.json +19 -0
  16. aice_data_quality-0.1.0/contract/v1/examples/valid/dbt_block_pass.json +18 -0
  17. aice_data_quality-0.1.0/contract/v1/examples/valid/dbt_warn.json +18 -0
  18. aice_data_quality-0.1.0/contract/v1/examples/valid/history_skipped.json +23 -0
  19. aice_data_quality-0.1.0/contract/v1/examples/valid/ingestion_block_fail.json +35 -0
  20. aice_data_quality-0.1.0/contract/v1/examples/valid/ingestion_flag_pass.json +27 -0
  21. aice_data_quality-0.1.0/contract/v1/examples/valid/manual_accuracy.json +25 -0
  22. aice_data_quality-0.1.0/contract/v1/examples/valid/run_blocked.json +14 -0
  23. aice_data_quality-0.1.0/contract/v1/examples/valid/run_open.json +10 -0
  24. aice_data_quality-0.1.0/contract/v1/result.schema.json +49 -0
  25. aice_data_quality-0.1.0/contract/v1/run.schema.json +31 -0
  26. aice_data_quality-0.1.0/docker/initdb/01-sample-data.sql +8 -0
  27. aice_data_quality-0.1.0/docker-compose.yml +20 -0
  28. aice_data_quality-0.1.0/docs/phase1-acceptance.md +20 -0
  29. aice_data_quality-0.1.0/docs/user-guide.md +85 -0
  30. aice_data_quality-0.1.0/examples/dataframe_example.py +49 -0
  31. aice_data_quality-0.1.0/pyproject.toml +48 -0
  32. aice_data_quality-0.1.0/src/dq/__init__.py +19 -0
  33. aice_data_quality-0.1.0/src/dq/checks/__init__.py +30 -0
  34. aice_data_quality-0.1.0/src/dq/checks/accepted_values.py +57 -0
  35. aice_data_quality-0.1.0/src/dq/checks/base.py +102 -0
  36. aice_data_quality-0.1.0/src/dq/checks/not_null.py +41 -0
  37. aice_data_quality-0.1.0/src/dq/checks/parses_as.py +87 -0
  38. aice_data_quality-0.1.0/src/dq/checks/row_count.py +66 -0
  39. aice_data_quality-0.1.0/src/dq/checks/schema.py +58 -0
  40. aice_data_quality-0.1.0/src/dq/checks/unique.py +51 -0
  41. aice_data_quality-0.1.0/src/dq/cli.py +70 -0
  42. aice_data_quality-0.1.0/src/dq/client.py +302 -0
  43. aice_data_quality-0.1.0/src/dq/config.py +56 -0
  44. aice_data_quality-0.1.0/src/dq/contract.py +196 -0
  45. aice_data_quality-0.1.0/src/dq/dataset.py +30 -0
  46. aice_data_quality-0.1.0/src/dq/dbt/__init__.py +3 -0
  47. aice_data_quality-0.1.0/src/dq/dbt/artifacts.py +114 -0
  48. aice_data_quality-0.1.0/src/dq/dbt/publish.py +117 -0
  49. aice_data_quality-0.1.0/src/dq/errors.py +35 -0
  50. aice_data_quality-0.1.0/src/dq/history.py +17 -0
  51. aice_data_quality-0.1.0/src/dq/privacy.py +57 -0
  52. aice_data_quality-0.1.0/src/dq/py.typed +0 -0
  53. aice_data_quality-0.1.0/src/dq/sinks/__init__.py +8 -0
  54. aice_data_quality-0.1.0/src/dq/sinks/base.py +12 -0
  55. aice_data_quality-0.1.0/src/dq/sinks/http.py +55 -0
  56. aice_data_quality-0.1.0/src/dq/sinks/log.py +26 -0
  57. aice_data_quality-0.1.0/src/dq/sinks/spool.py +85 -0
  58. aice_data_quality-0.1.0/src/dq/sources/__init__.py +22 -0
  59. aice_data_quality-0.1.0/src/dq/sources/_frame.py +102 -0
  60. aice_data_quality-0.1.0/src/dq/sources/base.py +14 -0
  61. aice_data_quality-0.1.0/src/dq/sources/csv.py +52 -0
  62. aice_data_quality-0.1.0/src/dq/sources/dataframe.py +22 -0
  63. aice_data_quality-0.1.0/src/dq/sources/excel.py +98 -0
  64. aice_data_quality-0.1.0/src/dq/sources/sql.py +127 -0
  65. aice_data_quality-0.1.0/src/dq/suites.py +192 -0
  66. aice_data_quality-0.1.0/src/dq/telemetry.py +34 -0
  67. aice_data_quality-0.1.0/tests/conftest.py +119 -0
  68. aice_data_quality-0.1.0/tests/fixtures/dbt/manifest.json +147 -0
  69. aice_data_quality-0.1.0/tests/fixtures/dbt/run_results.json +151 -0
  70. aice_data_quality-0.1.0/tests/test_checks_frame.py +149 -0
  71. aice_data_quality-0.1.0/tests/test_client.py +281 -0
  72. aice_data_quality-0.1.0/tests/test_contract.py +79 -0
  73. aice_data_quality-0.1.0/tests/test_contract_models.py +54 -0
  74. aice_data_quality-0.1.0/tests/test_dbt.py +102 -0
  75. aice_data_quality-0.1.0/tests/test_http_sink.py +70 -0
  76. aice_data_quality-0.1.0/tests/test_privacy_suites_cli.py +135 -0
  77. aice_data_quality-0.1.0/tests/test_sources_files.py +120 -0
  78. aice_data_quality-0.1.0/tests/test_sql_source.py +108 -0
  79. aice_data_quality-0.1.0/uv.lock +1353 -0
@@ -0,0 +1,4 @@
1
+ .venv/
2
+ .dq-spool/
3
+ __pycache__/
4
+ .pytest_cache/
@@ -0,0 +1,22 @@
1
+ # Changelog
2
+
3
+ SDK versions are independent of contract versions. Each release states the contract it produces.
4
+
5
+ ## 0.1.0 (unreleased)
6
+
7
+ **Produces contract:** `1.0` (`contract-v1.0.0`, pending sign-off)
8
+
9
+ - Core: `QualityClient`, `Run`, `Report`, `Dataset`; contract models in pydantic.
10
+ - Sources: `DataFrameSource`, `CsvSource`, `ExcelSource`, `SqlSource`.
11
+ - Checks: `schema`, `not_null`, `parses_as`, `accepted_values`, `unique`, `row_count` (including `vs_previous`).
12
+ - YAML suites validated with pydantic; `dq suite validate`.
13
+ - Sinks: `LogSink`, `HttpSink` (client of the server); local spool with ordered replay. The SDK stores nothing and ships no results DDL.
14
+ - `dq dbt publish`; dbt artifacts parsed with pydantic models.
15
+ - Sample policy (`keys`, `none`), HMAC key hashing for sensitive datasets, OpenTelemetry spans.
16
+
17
+ Differences from the specification, to confirm:
18
+
19
+ - No `PostgresSink` and no results DDL (spec P2 dropped): the server is the only results store. Acceptance criteria revised in `docs/phase1-acceptance.md`.
20
+ - `vs_previous` history (served by the server) counts `pass` **and `skipped`** results; counting only passing results can never bootstrap.
21
+ - Constructor/suite default severity is `flag`; the spec did not state a default.
22
+ - `HttpSink` and a proof-of-concept server are part of phase 1 (spec said phase 3).
@@ -0,0 +1,79 @@
1
+ Metadata-Version: 2.5
2
+ Name: aice-data-quality
3
+ Version: 0.1.0
4
+ Summary: Data quality SDK: checks, one result contract, one destination
5
+ Author-email: Adanian Labs <engineering@adanianlabs.io>
6
+ License: MIT
7
+ Classifier: Development Status :: 4 - Beta
8
+ Classifier: Intended Audience :: Developers
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: Typing :: Typed
11
+ Requires-Python: <3.14,>=3.10
12
+ Requires-Dist: pydantic>=2
13
+ Requires-Dist: pyyaml>=6
14
+ Provides-Extra: dbt
15
+ Provides-Extra: dev
16
+ Requires-Dist: httpx>=0.27; extra == 'dev'
17
+ Requires-Dist: jsonschema[format]>=4.21; extra == 'dev'
18
+ Requires-Dist: openpyxl>=3.1; extra == 'dev'
19
+ Requires-Dist: opentelemetry-api>=1.20; extra == 'dev'
20
+ Requires-Dist: opentelemetry-sdk>=1.20; extra == 'dev'
21
+ Requires-Dist: pandas>=2; extra == 'dev'
22
+ Requires-Dist: psycopg[binary]>=3.1; extra == 'dev'
23
+ Requires-Dist: pytest>=8.3; extra == 'dev'
24
+ Requires-Dist: sqlalchemy>=2; extra == 'dev'
25
+ Provides-Extra: excel
26
+ Requires-Dist: openpyxl>=3.1; extra == 'excel'
27
+ Requires-Dist: pandas>=2; extra == 'excel'
28
+ Provides-Extra: http
29
+ Requires-Dist: httpx>=0.27; extra == 'http'
30
+ Provides-Extra: otel
31
+ Requires-Dist: opentelemetry-api>=1.20; extra == 'otel'
32
+ Provides-Extra: pandas
33
+ Requires-Dist: pandas>=2; extra == 'pandas'
34
+ Provides-Extra: sql
35
+ Requires-Dist: psycopg[binary]>=3.1; extra == 'sql'
36
+ Requires-Dist: sqlalchemy>=2; extra == 'sql'
37
+ Description-Content-Type: text/markdown
38
+
39
+ # aice-data-quality (SDK)
40
+
41
+ Check data, record the outcome in the result contract, send it to one destination.
42
+ Import name: `dq`. Produces records of contract `1.0` ([`contract/`](contract/README.md)).
43
+
44
+ ```python
45
+ from dq import QualityClient, Dataset, Severity, checks as c
46
+ from dq.sources import DataFrameSource
47
+
48
+ client = QualityClient(project="my-project", environment="production") # or DQ_* env vars
49
+ with client.run(pipeline="uploads") as run:
50
+ report = run.validate(DataFrameSource(df), dataset=Dataset("finance_uploads", "statement", "bronze"),
51
+ owner="finance",
52
+ checks=[c.not_null("amount", name="amount_set", severity=Severity.BLOCK)])
53
+ if report.blocked:
54
+ raise UploadRejected(report.messages())
55
+ ```
56
+
57
+ - Full guide: [`docs/user-guide.md`](docs/user-guide.md)
58
+ - Runnable example: [`examples/dataframe_example.py`](examples/dataframe_example.py)
59
+ - Specification (shared with the server): [`../README.md`](../README.md)
60
+
61
+ ## What belongs here, and what does not
62
+
63
+ | In the SDK (runs inside each project) | In the server (`../server/`, never here) |
64
+ |---|---|
65
+ | Sources, checks, suites, privacy rules | Storing results: schema, upserts, retention |
66
+ | Contract models, run outcome, dbt mapping | Authentication and token scopes |
67
+ | Sinks: `LogSink`, `HttpSink` (the server's client), the local spool | Queries, history for `vs_previous`, dataset and check listings |
68
+ | `SqlSource`, which reads the project's own data tables | Alert routing, UI |
69
+
70
+ The SDK has no results database and no results DDL. `SqlSource` is the only code that talks to a database,
71
+ and only to read data being checked.
72
+
73
+ ## Develop
74
+
75
+ ```bash
76
+ docker compose up -d --wait # sample Postgres on :55432 for SqlSource tests only
77
+ uv sync --extra dev
78
+ uv run pytest # SqlSource tests skip themselves if the database is down
79
+ ```
@@ -0,0 +1,41 @@
1
+ # aice-data-quality (SDK)
2
+
3
+ Check data, record the outcome in the result contract, send it to one destination.
4
+ Import name: `dq`. Produces records of contract `1.0` ([`contract/`](contract/README.md)).
5
+
6
+ ```python
7
+ from dq import QualityClient, Dataset, Severity, checks as c
8
+ from dq.sources import DataFrameSource
9
+
10
+ client = QualityClient(project="my-project", environment="production") # or DQ_* env vars
11
+ with client.run(pipeline="uploads") as run:
12
+ report = run.validate(DataFrameSource(df), dataset=Dataset("finance_uploads", "statement", "bronze"),
13
+ owner="finance",
14
+ checks=[c.not_null("amount", name="amount_set", severity=Severity.BLOCK)])
15
+ if report.blocked:
16
+ raise UploadRejected(report.messages())
17
+ ```
18
+
19
+ - Full guide: [`docs/user-guide.md`](docs/user-guide.md)
20
+ - Runnable example: [`examples/dataframe_example.py`](examples/dataframe_example.py)
21
+ - Specification (shared with the server): [`../README.md`](../README.md)
22
+
23
+ ## What belongs here, and what does not
24
+
25
+ | In the SDK (runs inside each project) | In the server (`../server/`, never here) |
26
+ |---|---|
27
+ | Sources, checks, suites, privacy rules | Storing results: schema, upserts, retention |
28
+ | Contract models, run outcome, dbt mapping | Authentication and token scopes |
29
+ | Sinks: `LogSink`, `HttpSink` (the server's client), the local spool | Queries, history for `vs_previous`, dataset and check listings |
30
+ | `SqlSource`, which reads the project's own data tables | Alert routing, UI |
31
+
32
+ The SDK has no results database and no results DDL. `SqlSource` is the only code that talks to a database,
33
+ and only to read data being checked.
34
+
35
+ ## Develop
36
+
37
+ ```bash
38
+ docker compose up -d --wait # sample Postgres on :55432 for SqlSource tests only
39
+ uv sync --extra dev
40
+ uv run pytest # SqlSource tests skip themselves if the database is down
41
+ ```
@@ -0,0 +1,15 @@
1
+ # Contract changelog
2
+
3
+ Versioning rules: see [README.md](README.md#versioning).
4
+
5
+ ## 1.0.0 (unreleased, pending sign-off and tag `contract-v1.0.0`)
6
+
7
+ First published version.
8
+
9
+ - `run.schema.json` and `result.schema.json` (JSON Schema 2020-12).
10
+ - `origin` includes `manual` (decision C8) so reconciliation results use the same records.
11
+ - Timestamps are UTC with `Z` only (decision C9), enforced by pattern.
12
+ - `owner` required and non-empty on every result (C1).
13
+ - `check_id` pattern `<source>.<name>:<check_name>` (C6); at most 100 `failing_keys` (C4).
14
+ - Valid and invalid examples, validation test in `tests/test_contract.py`.
15
+
@@ -0,0 +1,44 @@
1
+ # Phase 0 decision log
2
+
3
+ **Status: drafted, not yet reviewed.** Every outcome below is the *proposed
4
+ default* from the phase plan. "Decided by" stays `pending` until the review
5
+ session (day 2) and is filled in by the SDK owner. Nothing here has been agreed
6
+ by anyone yet.
7
+
8
+ Timebox rule: anything not agreed by day 5 is left out of 1.0 and recorded under
9
+ "Proposals for a later minor version".
10
+
11
+ | # | Question | Outcome (proposed) | Decided by | Reflected in |
12
+ |---|---|---|---|---|
13
+ | C1 | Is `owner` required, and what is the fallback? | Required, non-empty; fallback `data-platform` | pending | `result.schema.json` (`required`, `minLength`) |
14
+ | C2 | Is the `layer` list complete? | `raw`, `bronze`, `silver`, `gold`, `app`, `other` | pending | `dataset.layer` enum |
15
+ | C3 | Is the `dimension` list complete; is `conformity` separate from `validity`? | Ten dimensions as in the specification; `conformity` stays separate | pending | `dimension` enum |
16
+ | C4 | Maximum `failing_keys` | 100 | pending | `maxItems: 100`; `too_many_keys.json` |
17
+ | C5 | Which datasets are sensitive; is sensitivity a result field? | SDK configuration, not a field. Default: all employee-level and finance transaction datasets (spec Q6) | pending | README (privacy); no schema field |
18
+ | C6 | `check_id` format | `<source>.<name>:<check_name>`, lowercase snake case | pending | `check_id` pattern; `bad_check_id.json` |
19
+ | C7 | dbt `warn` mapping | `status: fail` with `severity: warn` | pending | README (check status); `dbt_warn.json` |
20
+ | C8 | Is `accuracy` recorded with the same record, with `origin: manual`? | Yes | pending | `origin` enum; `manual_accuracy.json` |
21
+ | C9 | Timestamp format | UTC with `Z` only | pending | `utc_timestamp` pattern; `local_timestamp.json` |
22
+
23
+ ## Drafting choices to confirm in review
24
+
25
+ Small tightenings made while turning the draft into schemas. Raise any you disagree with.
26
+
27
+ 1. `owner`, `check_type`, `sdk_version` and `external_run_id` have `minLength: 1`, so an empty string cannot satisfy a required field (otherwise C1 is meaningless).
28
+ 2. Timestamps use both `format: date-time` and a `Z`-only `pattern`. The pattern is what enforces C9, because `format` is advisory in many validators.
29
+ 3. The timestamp rule is shared through `$defs/utc_timestamp` in each schema (schemas are self-contained; no cross-file references).
30
+ 4. `observed_value` allows `null` (as in the specification draft).
31
+ 5. Not enforced by schema, by design: `check_id` matching `dataset` + `check_name`, and `failing_count` vs `failing_keys` length. Both belong to SDK tests in phase 1.
32
+ 6. v1 uses `additionalProperties: false`, which makes "ignore unknown optional fields" (spec section 14) a consumer-side duty rather than something the schema allows. Confirm this is the intended reading.
33
+
34
+ ## Sign-off
35
+
36
+ | Role | Name | Date | Signed off |
37
+ |---|---|---|---|
38
+ | SDK owner | TBC (spec Q1 is still open) | | no |
39
+ | Data Platform | | | no |
40
+ | Second project representative | | | no |
41
+
42
+ ## Proposals for a later minor version
43
+
44
+ None yet. Items not agreed by day 5 are recorded here.
@@ -0,0 +1,106 @@
1
+ # Data quality result contract (v1)
2
+
3
+ The format every project uses to record data quality results, and that every
4
+ destination (warehouse table, later a server) accepts.
5
+
6
+ **Source of truth:** the JSON Schemas in [`v1/`](v1/). This document covers only
7
+ what a schema cannot express. Background and rationale: [`../../README.md`](../../README.md).
8
+
9
+ | File | Describes |
10
+ |---|---|
11
+ | [`v1/run.schema.json`](v1/run.schema.json) | One execution of a pipeline step |
12
+ | [`v1/result.schema.json`](v1/result.schema.json) | The outcome of one check in one run |
13
+ | [`v1/examples/valid/`](v1/examples/valid/) | Records that must pass (files named `run_*` are runs, the rest results) |
14
+ | [`v1/examples/invalid/`](v1/examples/invalid/) | Records that must be rejected |
15
+
16
+ Validate with `uv run --extra dev pytest` (see `tests/test_contract.py`).
17
+
18
+ ## Enumerations
19
+
20
+ | Field | Values |
21
+ |---|---|
22
+ | `environment` | `development`, `staging`, `production` |
23
+ | `origin` (run) | `sdk`, `dbt`, `manual` (reconciliation recorded by hand) |
24
+ | `layer` | `raw`, `bronze`, `silver`, `gold`, `app`, `other` |
25
+ | `dimension` | `completeness`, `validity`, `uniqueness`, `consistency`, `referential_integrity`, `timeliness`, `volume`, `distribution`, `conformity`, `accuracy` |
26
+ | `severity` | `block`, `flag`, `warn` |
27
+ | `status` | `pass`, `fail`, `error`, `skipped` |
28
+ | `outcome` (run) | `passed`, `flagged`, `blocked`, `error` |
29
+
30
+ ## Rules the schema cannot express
31
+
32
+ These are enforced by the SDK (phase 1), not by the schemas.
33
+
34
+ ### Severity
35
+
36
+ | Severity | Effect when the check has status `fail` or `error` |
37
+ |---|---|
38
+ | `block` | The run is `blocked`. The caller must stop (reject the upload, skip downstream steps). The SDK never exits the process |
39
+ | `flag` | Data is kept; the run is `flagged`; the owner is notified |
40
+ | `warn` | Recorded only |
41
+
42
+ Automated sources are never rejected: at ingestion they use `flag` at most. Only
43
+ human uploads, which can be fixed and resubmitted, use `block`.
44
+
45
+ ### Run outcome
46
+
47
+ Computed when the run closes, first match wins:
48
+
49
+ 1. `blocked`: any `block` result has status `fail` or `error`
50
+ 2. `flagged`: any `flag` result has status `fail` or `error`
51
+ 3. `passed`: otherwise (`warn` failures and `skipped` results included)
52
+
53
+ `error` is set instead when the run itself could not complete (for example the
54
+ source could not be read). An open run has neither `finished_at` nor `outcome`;
55
+ `outcome` requires `finished_at` (the schema enforces this).
56
+
57
+ ### Check status
58
+
59
+ - `pass`: rule satisfied.
60
+ - `fail`: rule not satisfied.
61
+ - `error`: the check could not run. It counts as a failure for severity and outcome. A broken check is never a pass.
62
+ - `skipped`: deliberately not run (for example no history to compare against). Has no effect on the outcome.
63
+ - dbt `warn` is recorded as `status: fail` with `severity: warn`, so it is visible but never blocks.
64
+
65
+ ### Idempotency
66
+
67
+ A result is unique on (`run_id`, `check_id`). Sending it again replaces the
68
+ earlier copy; it is never counted twice. A run is unique on `run_id`; closing a
69
+ run replaces the open record.
70
+
71
+ ### Dataset naming and `check_id`
72
+
73
+ - `dataset.source`, `dataset.name`: lowercase snake case (`^[a-z0-9_]+$`). `source` is the system or channel (`finance_uploads`, `bd_sheet`, `warehouse`); `name` is the entity, file type or table.
74
+ - dbt models use `source: warehouse` and `name` = model name.
75
+ - `check_id` = `<source>.<name>:<check_name>`, all lowercase snake case, and must be consistent with `dataset` and `check_name`. The schema checks the pattern only; the SDK checks the match.
76
+ - Names must stay stable: renaming a dataset or check breaks its history.
77
+
78
+ ### Timestamps
79
+
80
+ UTC with a `Z` suffix only, for example `2027-01-12T07:41:03Z` (fractional
81
+ seconds allowed). Offsets such as `+03:00` are rejected; convert before sending.
82
+
83
+ ### Owner
84
+
85
+ `owner` is required on every result. When none is given, the SDK and the dbt
86
+ publisher use `data-platform`. `owner` is a team or role, never a person's email.
87
+
88
+ ### Sample policy and privacy (`failing_keys`, `message`, `error_detail`)
89
+
90
+ | Policy | `failing_keys` contains | Allowed in |
91
+ |---|---|---|
92
+ | `none` | Nothing | All environments |
93
+ | `keys` (default) | Row identifiers only: a primary key, or `row:<n>` for files | All environments |
94
+ | `rows` | Full failing rows | `development` only; the SDK refuses it in `staging` and `production` |
95
+
96
+ 1. `failing_keys` holds at most 100 entries (the schema enforces this). `failing_count` carries the true count.
97
+ 2. `message` and `error_detail` describe the failure (column, rule, count). They never contain data values from personal or financial fields.
98
+ 3. Keys of datasets marked sensitive are hashed before leaving the process. Sensitivity is SDK configuration, not a field on the result.
99
+ 4. Credentials, connection strings and tokens never appear in any field.
100
+
101
+ ## Versioning
102
+
103
+ 1. `spec_version` is on every record. This contract is `1.0`.
104
+ 2. **Minor** (`1.x`): new optional fields, new check types, new enum values for `layer`. Consumers must ignore unknown optional fields. (Note that the v1 schemas set `additionalProperties: false`, so a producer may not send a field until its minor version is published here, and validators must be updated with it.)
105
+ 3. **Breaking** (`2.0`): renamed or removed fields, changed meanings, new required fields. Sinks and the server accept both versions for at least one release cycle. Schemas live in a new `contract/v2/` folder.
106
+ 4. Every change goes through a pull request to this folder, with a [`CHANGELOG.md`](CHANGELOG.md) entry and a version bump, before any code changes. Release tags are `contract-vX.Y.Z`.
@@ -0,0 +1,18 @@
1
+ {
2
+ "spec_version": "1.0",
3
+ "run_id": "0b6c2f5e-9a51-4d4f-8a7e-3f1d2b9c7a10",
4
+ "check_id": "Finance-Uploads/date parses",
5
+ "check_name": "date_parses",
6
+ "check_type": "parses_as",
7
+ "dataset": {
8
+ "source": "finance_uploads",
9
+ "name": "bank_statement_kcb",
10
+ "layer": "bronze"
11
+ },
12
+ "dimension": "validity",
13
+ "severity": "block",
14
+ "status": "fail",
15
+ "owner": "finance",
16
+ "evaluated_at": "2027-01-12T07:41:03Z",
17
+ "failing_count": 1
18
+ }
@@ -0,0 +1,23 @@
1
+ {
2
+ "spec_version": "1.0",
3
+ "run_id": "0b6c2f5e-9a51-4d4f-8a7e-3f1d2b9c7a10",
4
+ "check_id": "finance_uploads.bank_statement_kcb:date_parses",
5
+ "check_name": "date_parses",
6
+ "check_type": "parses_as",
7
+ "dataset": {
8
+ "source": "finance_uploads",
9
+ "name": "bank_statement_kcb",
10
+ "layer": "bronze"
11
+ },
12
+ "dimension": "validity",
13
+ "severity": "block",
14
+ "status": "fail",
15
+ "owner": "finance",
16
+ "evaluated_at": "2027-01-12T07:41:03Z",
17
+ "failing_count": 1,
18
+ "sample_rows": [
19
+ {
20
+ "amount": 100
21
+ }
22
+ ]
23
+ }
@@ -0,0 +1,18 @@
1
+ {
2
+ "spec_version": "1.0",
3
+ "run_id": "0b6c2f5e-9a51-4d4f-8a7e-3f1d2b9c7a10",
4
+ "check_id": "finance_uploads.bank_statement_kcb:date_parses",
5
+ "check_name": "date_parses",
6
+ "check_type": "parses_as",
7
+ "dataset": {
8
+ "source": "finance_uploads",
9
+ "name": "bank_statement_kcb",
10
+ "layer": "bronze"
11
+ },
12
+ "dimension": "validity",
13
+ "severity": "block",
14
+ "status": "fail",
15
+ "owner": "finance",
16
+ "evaluated_at": "2027-01-12T10:41:03+03:00",
17
+ "failing_count": 1
18
+ }
@@ -0,0 +1,17 @@
1
+ {
2
+ "spec_version": "1.0",
3
+ "run_id": "0b6c2f5e-9a51-4d4f-8a7e-3f1d2b9c7a10",
4
+ "check_id": "finance_uploads.bank_statement_kcb:date_parses",
5
+ "check_name": "date_parses",
6
+ "check_type": "parses_as",
7
+ "dataset": {
8
+ "source": "finance_uploads",
9
+ "name": "bank_statement_kcb",
10
+ "layer": "bronze"
11
+ },
12
+ "dimension": "validity",
13
+ "severity": "block",
14
+ "status": "fail",
15
+ "evaluated_at": "2027-01-12T07:41:03Z",
16
+ "failing_count": 1
17
+ }
@@ -0,0 +1,11 @@
1
+ {
2
+ "spec_version": "1.0",
3
+ "run_id": "0b6c2f5e-9a51-4d4f-8a7e-3f1d2b9c7a10",
4
+ "project": "organization-data-model",
5
+ "environment": "production",
6
+ "pipeline": "finance-uploads",
7
+ "origin": "sdk",
8
+ "started_at": "2027-01-12T07:40:58Z",
9
+ "outcome": "blocked",
10
+ "sdk_version": "0.1.0"
11
+ }
@@ -0,0 +1,121 @@
1
+ {
2
+ "spec_version": "1.0",
3
+ "run_id": "0b6c2f5e-9a51-4d4f-8a7e-3f1d2b9c7a10",
4
+ "check_id": "finance_uploads.bank_statement_kcb:date_parses",
5
+ "check_name": "date_parses",
6
+ "check_type": "parses_as",
7
+ "dataset": {
8
+ "source": "finance_uploads",
9
+ "name": "bank_statement_kcb",
10
+ "layer": "bronze"
11
+ },
12
+ "dimension": "validity",
13
+ "severity": "block",
14
+ "status": "fail",
15
+ "owner": "finance",
16
+ "evaluated_at": "2027-01-12T07:41:03Z",
17
+ "failing_count": 101,
18
+ "failing_keys": [
19
+ "row:1",
20
+ "row:2",
21
+ "row:3",
22
+ "row:4",
23
+ "row:5",
24
+ "row:6",
25
+ "row:7",
26
+ "row:8",
27
+ "row:9",
28
+ "row:10",
29
+ "row:11",
30
+ "row:12",
31
+ "row:13",
32
+ "row:14",
33
+ "row:15",
34
+ "row:16",
35
+ "row:17",
36
+ "row:18",
37
+ "row:19",
38
+ "row:20",
39
+ "row:21",
40
+ "row:22",
41
+ "row:23",
42
+ "row:24",
43
+ "row:25",
44
+ "row:26",
45
+ "row:27",
46
+ "row:28",
47
+ "row:29",
48
+ "row:30",
49
+ "row:31",
50
+ "row:32",
51
+ "row:33",
52
+ "row:34",
53
+ "row:35",
54
+ "row:36",
55
+ "row:37",
56
+ "row:38",
57
+ "row:39",
58
+ "row:40",
59
+ "row:41",
60
+ "row:42",
61
+ "row:43",
62
+ "row:44",
63
+ "row:45",
64
+ "row:46",
65
+ "row:47",
66
+ "row:48",
67
+ "row:49",
68
+ "row:50",
69
+ "row:51",
70
+ "row:52",
71
+ "row:53",
72
+ "row:54",
73
+ "row:55",
74
+ "row:56",
75
+ "row:57",
76
+ "row:58",
77
+ "row:59",
78
+ "row:60",
79
+ "row:61",
80
+ "row:62",
81
+ "row:63",
82
+ "row:64",
83
+ "row:65",
84
+ "row:66",
85
+ "row:67",
86
+ "row:68",
87
+ "row:69",
88
+ "row:70",
89
+ "row:71",
90
+ "row:72",
91
+ "row:73",
92
+ "row:74",
93
+ "row:75",
94
+ "row:76",
95
+ "row:77",
96
+ "row:78",
97
+ "row:79",
98
+ "row:80",
99
+ "row:81",
100
+ "row:82",
101
+ "row:83",
102
+ "row:84",
103
+ "row:85",
104
+ "row:86",
105
+ "row:87",
106
+ "row:88",
107
+ "row:89",
108
+ "row:90",
109
+ "row:91",
110
+ "row:92",
111
+ "row:93",
112
+ "row:94",
113
+ "row:95",
114
+ "row:96",
115
+ "row:97",
116
+ "row:98",
117
+ "row:99",
118
+ "row:100",
119
+ "row:101"
120
+ ]
121
+ }
@@ -0,0 +1,18 @@
1
+ {
2
+ "spec_version": "1.0",
3
+ "run_id": "0b6c2f5e-9a51-4d4f-8a7e-3f1d2b9c7a10",
4
+ "check_id": "finance_uploads.bank_statement_kcb:date_parses",
5
+ "check_name": "date_parses",
6
+ "check_type": "parses_as",
7
+ "dataset": {
8
+ "source": "finance_uploads",
9
+ "name": "bank_statement_kcb",
10
+ "layer": "bronze"
11
+ },
12
+ "dimension": "validity",
13
+ "severity": "critical",
14
+ "status": "fail",
15
+ "owner": "finance",
16
+ "evaluated_at": "2027-01-12T07:41:03Z",
17
+ "failing_count": 1
18
+ }
@@ -0,0 +1,19 @@
1
+ {
2
+ "spec_version": "1.0",
3
+ "run_id": "0b6c2f5e-9a51-4d4f-8a7e-3f1d2b9c7a10",
4
+ "check_id": "finance_uploads.bank_statement_kcb:amount_not_null",
5
+ "check_name": "amount_not_null",
6
+ "check_type": "not_null",
7
+ "dataset": {
8
+ "source": "finance_uploads",
9
+ "name": "bank_statement_kcb",
10
+ "layer": "bronze"
11
+ },
12
+ "dimension": "completeness",
13
+ "severity": "block",
14
+ "status": "error",
15
+ "owner": "finance",
16
+ "evaluated_at": "2027-01-12T07:41:03Z",
17
+ "error_detail": "Column 'amount' not found in the uploaded sheet.",
18
+ "message": "Check could not run: required column is missing."
19
+ }
@@ -0,0 +1,18 @@
1
+ {
2
+ "spec_version": "1.0",
3
+ "run_id": "0b6c2f5e-9a51-4d4f-8a7e-3f1d2b9c7a10",
4
+ "check_id": "warehouse.fin_cash_activity:unique_fin_cash_activity_id",
5
+ "check_name": "unique_fin_cash_activity_id",
6
+ "check_type": "dbt:unique",
7
+ "dataset": {
8
+ "source": "warehouse",
9
+ "name": "fin_cash_activity",
10
+ "layer": "gold"
11
+ },
12
+ "dimension": "uniqueness",
13
+ "severity": "block",
14
+ "status": "pass",
15
+ "owner": "finance",
16
+ "evaluated_at": "2027-01-12T07:41:03Z",
17
+ "failing_count": 0
18
+ }
@@ -0,0 +1,18 @@
1
+ {
2
+ "spec_version": "1.0",
3
+ "run_id": "0b6c2f5e-9a51-4d4f-8a7e-3f1d2b9c7a10",
4
+ "check_id": "warehouse.fin_cash_activity:not_null_fin_cash_activity_amount",
5
+ "check_name": "not_null_fin_cash_activity_amount",
6
+ "check_type": "dbt:not_null",
7
+ "dataset": {
8
+ "source": "warehouse",
9
+ "name": "fin_cash_activity",
10
+ "layer": "gold"
11
+ },
12
+ "dimension": "completeness",
13
+ "severity": "warn",
14
+ "status": "fail",
15
+ "owner": "finance",
16
+ "evaluated_at": "2027-01-12T07:41:03Z",
17
+ "failing_count": 7
18
+ }
@@ -0,0 +1,23 @@
1
+ {
2
+ "spec_version": "1.0",
3
+ "run_id": "0b6c2f5e-9a51-4d4f-8a7e-3f1d2b9c7a10",
4
+ "check_id": "finance_uploads.bank_statement_kcb:row_count_vs_previous",
5
+ "check_name": "row_count_vs_previous",
6
+ "check_type": "row_count",
7
+ "dataset": {
8
+ "source": "finance_uploads",
9
+ "name": "bank_statement_kcb",
10
+ "layer": "bronze"
11
+ },
12
+ "dimension": "volume",
13
+ "severity": "flag",
14
+ "status": "skipped",
15
+ "owner": "finance",
16
+ "evaluated_at": "2027-01-12T07:41:03Z",
17
+ "expected": {
18
+ "vs_previous": {
19
+ "tolerance_pct": 30
20
+ }
21
+ },
22
+ "message": "No previous run to compare against."
23
+ }