dactopus 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. dactopus-0.2.0/.github/dependabot.yml +10 -0
  2. dactopus-0.2.0/.github/workflows/ci.yml +37 -0
  3. dactopus-0.2.0/.github/workflows/publish.yml +72 -0
  4. dactopus-0.2.0/.gitignore +14 -0
  5. dactopus-0.2.0/AGENTS.md +218 -0
  6. dactopus-0.2.0/CHANGELOG.md +139 -0
  7. dactopus-0.2.0/CLAUDE.md +1 -0
  8. dactopus-0.2.0/CONTRIBUTING.md +67 -0
  9. dactopus-0.2.0/LICENSE +202 -0
  10. dactopus-0.2.0/PKG-INFO +286 -0
  11. dactopus-0.2.0/README.md +256 -0
  12. dactopus-0.2.0/docs/design.md +182 -0
  13. dactopus-0.2.0/pyproject.toml +69 -0
  14. dactopus-0.2.0/src/dactopus/__init__.py +1 -0
  15. dactopus-0.2.0/src/dactopus/build.py +221 -0
  16. dactopus-0.2.0/src/dactopus/checks.py +671 -0
  17. dactopus-0.2.0/src/dactopus/cli.py +182 -0
  18. dactopus-0.2.0/src/dactopus/clickhouse.py +32 -0
  19. dactopus-0.2.0/src/dactopus/config.py +353 -0
  20. dactopus-0.2.0/src/dactopus/ga4.py +253 -0
  21. dactopus-0.2.0/src/dactopus/library.py +210 -0
  22. dactopus-0.2.0/src/dactopus/report.py +129 -0
  23. dactopus-0.2.0/src/dactopus/serve.py +59 -0
  24. dactopus-0.2.0/src/dactopus/shopify.py +475 -0
  25. dactopus-0.2.0/tests/conftest.py +81 -0
  26. dactopus-0.2.0/tests/fixtures/dbt/shopify/manifest.json +562 -0
  27. dactopus-0.2.0/tests/fixtures/dbt/shopify/run_results_build_error.json +210 -0
  28. dactopus-0.2.0/tests/fixtures/dbt/shopify/run_results_build_pass.json +210 -0
  29. dactopus-0.2.0/tests/fixtures/dbt/shopify/run_results_build_schema_change.json +210 -0
  30. dactopus-0.2.0/tests/fixtures/dbt/shopify/run_results_build_skipped.json +210 -0
  31. dactopus-0.2.0/tests/fixtures/dbt/shopify/run_results_test_fail.json +186 -0
  32. dactopus-0.2.0/tests/fixtures/dbt/shopify/sources_error.json +11 -0
  33. dactopus-0.2.0/tests/fixtures/dbt/shopify/sources_pass.json +29 -0
  34. dactopus-0.2.0/tests/fixtures/dbt/shopify/sources_stale.json +29 -0
  35. dactopus-0.2.0/tests/fixtures/ga4_export_schema.arrow +0 -0
  36. dactopus-0.2.0/tests/fixtures/library/sources/broken/deployment.yml +5 -0
  37. dactopus-0.2.0/tests/fixtures/library/sources/ga4/deployment.yml +20 -0
  38. dactopus-0.2.0/tests/fixtures/library/sources/shopify/deployment.yml +18 -0
  39. dactopus-0.2.0/tests/fixtures/shopify_order_imported.json +5 -0
  40. dactopus-0.2.0/tests/fixtures/shopify_orders.json +13 -0
  41. dactopus-0.2.0/tests/fixtures/shopify_products.json +196 -0
  42. dactopus-0.2.0/tests/test_build.py +157 -0
  43. dactopus-0.2.0/tests/test_checks.py +595 -0
  44. dactopus-0.2.0/tests/test_cli.py +288 -0
  45. dactopus-0.2.0/tests/test_config.py +217 -0
  46. dactopus-0.2.0/tests/test_ga4.py +244 -0
  47. dactopus-0.2.0/tests/test_library.py +187 -0
  48. dactopus-0.2.0/tests/test_report.py +131 -0
  49. dactopus-0.2.0/tests/test_serve.py +219 -0
  50. dactopus-0.2.0/tests/test_shopify.py +331 -0
  51. dactopus-0.2.0/uv.lock +2326 -0
@@ -0,0 +1,10 @@
1
+ version: 2
2
+ updates:
3
+ - package-ecosystem: uv
4
+ directory: /
5
+ schedule:
6
+ interval: monthly
7
+ - package-ecosystem: github-actions
8
+ directory: /
9
+ schedule:
10
+ interval: monthly
@@ -0,0 +1,37 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ jobs:
9
+ test:
10
+ runs-on: ubuntu-latest
11
+ strategy:
12
+ fail-fast: false
13
+ matrix:
14
+ python: ["3.11", "3.12", "3.13"]
15
+ env:
16
+ UV_PYTHON: ${{ matrix.python }}
17
+ steps:
18
+ - uses: actions/checkout@v7
19
+ - uses: astral-sh/setup-uv@v7
20
+ - run: uv sync --locked
21
+ - run: uv run ruff check src tests
22
+ - run: uv run ruff format --check src tests
23
+ - run: uv run pytest -q -m "not integration"
24
+
25
+ package:
26
+ runs-on: ubuntu-latest
27
+ steps:
28
+ - uses: actions/checkout@v7
29
+ - uses: astral-sh/setup-uv@v7
30
+ - run: uv build
31
+ - run: uvx twine check --strict dist/* # the README is the PyPI page
32
+ - name: The wheel works on its own, outside the checkout
33
+ run: |
34
+ uv venv "$RUNNER_TEMP/wheel"
35
+ uv pip install --python "$RUNNER_TEMP/wheel" "$(ls dist/*.whl)"
36
+ cd "$RUNNER_TEMP" && wheel/bin/dactopus init --config sample.yaml
37
+ wheel/bin/python -c "import dactopus.serve, dactopus_ossie_clickhouse.mcp_server"
@@ -0,0 +1,72 @@
1
+ name: Publish
2
+
3
+ # A release on GitHub publishes its tag to PyPI. Run by hand, the same build goes to
4
+ # TestPyPI under a dev version (X.Y.Z.devN, N the run number, since an index takes
5
+ # each version once): a rehearsal on main before the tag. Both indexes trust this
6
+ # workflow and its environment (trusted publishing), so no token is stored here.
7
+ on:
8
+ release:
9
+ types: [published]
10
+ workflow_dispatch:
11
+
12
+ jobs:
13
+ build:
14
+ runs-on: ubuntu-latest
15
+ steps:
16
+ - uses: actions/checkout@v7
17
+ with:
18
+ fetch-depth: 0 # the history of main, for the ancestry check below
19
+ - uses: astral-sh/setup-uv@v7
20
+ - name: The tag is the version in pyproject.toml
21
+ if: github.event_name == 'release'
22
+ run: test "$GITHUB_REF_NAME" = "v$(uv version --short)"
23
+ - name: The tagged commit is on main
24
+ if: github.event_name == 'release'
25
+ run: |
26
+ git merge-base --is-ancestor HEAD origin/main || {
27
+ echo "::error::$GITHUB_REF_NAME is not on main: tag the commit main got from the merge"
28
+ exit 1
29
+ }
30
+ - name: A dev version for TestPyPI
31
+ if: github.event_name == 'workflow_dispatch'
32
+ run: uv version --frozen "$(uv version --short).dev$GITHUB_RUN_NUMBER"
33
+ - run: uv build
34
+ - run: uvx twine check --strict dist/*
35
+ - uses: actions/upload-artifact@v4
36
+ with:
37
+ name: dist
38
+ path: dist/
39
+
40
+ pypi:
41
+ if: github.event_name == 'release'
42
+ needs: build
43
+ runs-on: ubuntu-latest
44
+ environment:
45
+ name: pypi
46
+ url: https://pypi.org/project/dactopus/
47
+ permissions:
48
+ id-token: write
49
+ steps:
50
+ - uses: actions/download-artifact@v4
51
+ with:
52
+ name: dist
53
+ path: dist/
54
+ - uses: pypa/gh-action-pypi-publish@release/v1
55
+
56
+ testpypi:
57
+ if: github.event_name == 'workflow_dispatch'
58
+ needs: build
59
+ runs-on: ubuntu-latest
60
+ environment:
61
+ name: testpypi
62
+ url: https://test.pypi.org/project/dactopus/
63
+ permissions:
64
+ id-token: write
65
+ steps:
66
+ - uses: actions/download-artifact@v4
67
+ with:
68
+ name: dist
69
+ path: dist/
70
+ - uses: pypa/gh-action-pypi-publish@release/v1
71
+ with:
72
+ repository-url: https://test.pypi.org/legacy/
@@ -0,0 +1,14 @@
1
+ CLAUDE.local.md
2
+ .claude/
3
+ .DS_Store
4
+ .knowledge_private/
5
+ .venv/
6
+ __pycache__/
7
+ .pytest_cache/
8
+ .ruff_cache/
9
+ dist/
10
+ .coverage
11
+
12
+ # Local installations and their output
13
+ /dactopus.yaml
14
+ .dactopus/
@@ -0,0 +1,218 @@
1
+ # AGENTS.md
2
+
3
+ Guidance for AI coding agents working in this repository. What the product
4
+ is and why: [README.md](README.md).
5
+
6
+ ## Project
7
+
8
+ dactopus is the runner of Dactopus: the process a client installs next to
9
+ their ClickHouse. It loads sources into that ClickHouse, runs the source
10
+ packages of [dactopus-data-models](https://github.com/Dactopus/dactopus-data-models)
11
+ (dbt projects) to build the canonical tables, runs checks, and serves the
12
+ Ossie models of those tables to AI agents over MCP through
13
+ [dactopus-ossie-clickhouse](https://github.com/Dactopus/dactopus-ossie-clickhouse).
14
+ The command line is the whole runner: no control plane, no scheduler.
15
+
16
+ The word is *runner*. Never call it an agent: an agent is the AI that
17
+ talks to the MCP server.
18
+
19
+ Licensed under Apache 2.0.
20
+
21
+ ## Language
22
+
23
+ All text in this repository is in English only: code identifiers, comments,
24
+ docstrings, commit messages, documentation, issues, and test names. Do not
25
+ introduce text in any other language, even in examples or fixtures.
26
+
27
+ ## Scope
28
+
29
+ In scope:
30
+
31
+ - One configuration file per installation: the ClickHouse connection and
32
+ the user `serve` connects as, the databases for raw input and canonical
33
+ tables, the store's time zone, the version of the models
34
+ library, the packages to run and their parameters.
35
+ - Loading GA4: the BigQuery export, read with the Storage Read API, into
36
+ the GA4 package's input table.
37
+ - Loading Shopify: orders with lines, shipping and refunds, and the
38
+ products with their deletions, read page by page from the GraphQL
39
+ Admin API, into the Shopify package's input tables.
40
+ - Running the packages with dbt.
41
+ - Checks behind a plugin interface, built-in checks, and one report a
42
+ store owner can read.
43
+ - Serving the models over MCP, as a thin call to dactopus-ossie-clickhouse.
44
+
45
+ Out of scope, do not build:
46
+
47
+ - Transport for other sources. A database (WooCommerce on MySQL) reaches
48
+ ClickHouse by CDC (PeerDB, ClickPipes) the client sets up; the runner
49
+ reads the tables it finds. No replication engine, no wrapper over
50
+ PeerDB, Debezium or dlt.
51
+ - Transformations and model descriptions. SQL that maps a source to an
52
+ entity belongs in a package of dactopus-data-models; a description
53
+ belongs in its Ossie model. The runner ships neither.
54
+ - Query execution and MCP tools. That is dactopus-ossie-clickhouse.
55
+ - Scheduling, alerts, run history, a control plane, a user interface.
56
+ Runs are started by hand or by the client's cron.
57
+ - Merging model files or overlaying fragments on them. The executor
58
+ reads several files as one model; the runner only lists them.
59
+ - Migration from a client's dbt or Looker.
60
+ - An installer, a Docker image, a single-file build. Installation is
61
+ `uv`, as for the executor.
62
+ - Placeholders or stubs for any of the above.
63
+
64
+ ## Design principles
65
+
66
+ - Thin. Every command is the Quick start of dactopus-data-models made
67
+ portable: the same steps, one tool, one configuration file. A new
68
+ concept needs a decision first.
69
+ - The library owns the input contract (`sources/<source>/input.sql`:
70
+ the tables a package reads) and the transformations; the runner owns
71
+ delivery to that contract and running. A gap between the two is fixed
72
+ in the library, never patched around in the runner.
73
+ - The runner knows a package only by its `input.sql` and its
74
+ `sources/<package>/deployment.yml`: the Ossie model its tables serve,
75
+ the dbt variables that take the input database and the time zone, and
76
+ the tables counted per day. A variable, table, column or model file
77
+ name of a package never appears in the runner's code. Two exceptions:
78
+ the loaders, which write the tables `input.sql` defines (the GA4
79
+ loader also rereads as many days as the package's
80
+ `ga4_lookback_days`), and the commented sample `init` writes.
81
+ - The executor knows nothing of Dactopus; the runner is a consumer like
82
+ any other. What the runner needs from it is proposed there as a
83
+ feature for every Ossie user.
84
+ - Data and secrets stay with the client. The runner writes only to the
85
+ client's ClickHouse. Credentials come from environment variables or
86
+ files the configuration names, and never appear in logs, reports or
87
+ error messages.
88
+ - No database name is written into a model or a package; the
89
+ configuration chooses them, the runner passes them to dbt as variables
90
+ and to `input.sql` as a query parameter.
91
+ - A run over unchanged input changes nothing, and a second load of the
92
+ same rows is harmless.
93
+ - One plugin interface: checks. Everything else is a direct call.
94
+ - Keep the public surface small: four commands, one configuration file.
95
+
96
+ ## Stack
97
+
98
+ Python 3.11+, managed with uv. The parts that constrain code:
99
+
100
+ - **dbt v1**: `dbt-core` 1.11 with `dbt-clickhouse`. Packages are dbt
101
+ projects and also run on dbt v2, but its ClickHouse adapter is in beta
102
+ (reads LZ4 only, `source freshness` broken), so the runner pins v1
103
+ until the adapter is generally available.
104
+ - **dactopus-ossie-clickhouse** for `serve` and for validating the
105
+ models against the built tables.
106
+ - **clickhouse-connect** with the `arrow` extra for loading: Arrow
107
+ tables go in with `insert_arrow`, which maps Arrow structs to `Tuple`
108
+ columns by field name and drops struct fields the table lacks.
109
+ - **google-cloud-bigquery-storage** for GA4: the Storage Read API returns
110
+ Arrow, selects columns and filters rows, and reads tables only, not
111
+ views. The daily tables are listed with one BigQuery REST call
112
+ (`tables.list`) through `google-auth`, which comes with it; no
113
+ `google-cloud-bigquery`.
114
+ - The Shopify client is the standard library (`urllib`) with `certifi`
115
+ for root certificates, as in the library's README recipe. No dlt: its Shopify source uses the legacy
116
+ REST Admin API.
117
+ - No ClickHouse binaries on the runner's machine: `clickhouse local` and
118
+ `clickhouse client` in the library's recipes are replaced by Python.
119
+ - **tzdata** on every platform: the names of time zones do not depend
120
+ on the age of the machine's tz database.
121
+
122
+ The models library is not a Python package. The configuration names a
123
+ version (`models.version`: a release tag, fetched as the GitHub archive
124
+ of the tag into `~/.cache/dactopus/`, no git needed) or a path
125
+ (`models.path`, for development). Packages run one at a time with
126
+ `dbt build --project-dir <library>/sources/<package>`, with a profile
127
+ the runner writes; the package names it (`profile: dactopus`). Packages
128
+ do not depend on each other; no root dbt project is generated.
129
+
130
+ ## Architecture rules
131
+
132
+ - Layering: configuration -> loaders (one per source) -> build (dbt) ->
133
+ checks -> report. `serve` is a call to the executor with a list of
134
+ files. Lower layers never import higher ones.
135
+ - Commands: `dactopus run` (load, build, check, report), `dactopus check`
136
+ (checks and report only), `dactopus serve` (MCP), `dactopus init`
137
+ (writes a commented sample configuration; no dialogue). Separate load
138
+ and build commands appear when someone needs them apart; until then
139
+ they are flags of `run`.
140
+ - A loader creates the package's input tables by running the package's
141
+ `input.sql` from the library, never from a copy, and writes nothing
142
+ else. GA4: select the top-level columns of the Arrow table and insert
143
+ it as is, no struct projection, no Parquet on disk; incremental loads
144
+ reread the daily tables from `max(event_date)` in the input minus the
145
+ package's `ga4_lookback_days` (default 3) and replace those days.
146
+ Shopify: pages of 25 orders with lines, shipping and refunds, then
147
+ pages of 100 products and the deletions of products from the store's
148
+ events; an incremental load filters `updated_at:>=` from the latest
149
+ loaded change of its kind minus one hour and appends the versions it
150
+ gets (the package keeps every version and dedupes). Bulk operations do
151
+ not deliver refund lines; do not use them for the full load.
152
+ - A check is a named object that takes a ClickHouse connection and the
153
+ installation's configuration and returns findings. A finding has one
154
+ shape for built-in and plugin checks alike: check, object, severity,
155
+ observed, expected, and a sentence for the owner. Checks register
156
+ through Python entry points; built-in ones are dbt test results (from
157
+ `target/run_results.json` with `manifest.json` for the test's kind,
158
+ column and parameters), input freshness, rows per day, and the time
159
+ zone of the daily tables against `store.timezone`. The report
160
+ is Markdown, to the terminal and to a file, three answers first (how
161
+ fresh the data is; whether every check passed and which did not, in
162
+ words; volumes per day this week against past weeks), details below.
163
+ The exit code is non-zero on a `fail` or `error` finding.
164
+ - `serve` passes the model file each configured package declares and
165
+ the connection; the executor combines them. The runner does not read,
166
+ merge or edit model files.
167
+ - Memory: a daily GA4 table of a mid-size brand is hundreds of megabytes
168
+ of Arrow. Keep it; read in `ReadRows` batches when a load exceeds it.
169
+ - Errors name the step, the object and the next action in plain words;
170
+ the owner reads the report, the engineer reads the log.
171
+
172
+ ## Testing
173
+
174
+ - A loader is accepted when the input it writes equals, row for row,
175
+ what the library's README recipe writes from the same source: the
176
+ public GA4 sample `ga4_obfuscated_sample_ecommerce`, the Shopify rows
177
+ in the library's `tests/shopify/fixture.sql` and a development store.
178
+ - Build and checks are accepted when the package's own `dbt build` and
179
+ the library's `tests/<source>/check_numbers.py` pass over tables the
180
+ runner loaded.
181
+ - Tests that use the `clickhouse` fixture are integration tests: they
182
+ need a server at `DACTOPUS_CLICKHOUSE_URL` (user, host and port, as
183
+ `clickhouse.url`; default `http://127.0.0.1:8123`), with the password
184
+ in `CLICKHOUSE_PASSWORD`, and skip otherwise. Tests that need BigQuery
185
+ or Shopify credentials skip without them; loader logic is tested on
186
+ recorded Arrow and JSON fixtures.
187
+ - Every change to library code comes with a test. Facts about the
188
+ BigQuery export, the Shopify API and ClickHouse behaviour come from
189
+ running them, not from memory; a version and a date go with each fact
190
+ written down.
191
+
192
+ ## Development
193
+
194
+ Layout: `src/dactopus/` (library and CLI), `tests/` (pytest, fixtures in
195
+ `tests/fixtures/`), `docs/`. Keep modules flat; add a package level only
196
+ when a module outgrows one file.
197
+
198
+ ```bash
199
+ uv sync # install with dev tools
200
+ uv run pytest # tests; ClickHouse-dependent ones skip when no server
201
+ uv run pytest -m "not integration" # unit tests only
202
+ uv run ruff check src tests # lint
203
+ uv run ruff format src tests # format
204
+ uv run dactopus init # write dactopus.yaml with comments
205
+ uv run dactopus run --config dactopus.yaml
206
+ uv run dactopus check --config dactopus.yaml
207
+ uv run dactopus serve --config dactopus.yaml # MCP over stdio
208
+ ```
209
+
210
+ Issues and pull requests in this repository are opened by the
211
+ maintainer, not by an agent on its own.
212
+
213
+ ## Repository status
214
+
215
+ A prototype; 0.1.0 is the first release. Versions follow
216
+ Semantic Versioning, the changelog follows Keep a Changelog, a release is
217
+ the tag `vX.Y.Z` with the GitHub release titled `X.Y.Z`. Pull requests
218
+ are merged with rebase.
@@ -0,0 +1,139 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project are recorded here. The format follows
4
+ [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versions follow
5
+ [Semantic Versioning](https://semver.org/). While the major version is 0,
6
+ a minor release may change the command line and the configuration file.
7
+
8
+ ## [Unreleased]
9
+
10
+ ## [0.2.0] - 2026-11-01
11
+
12
+ ### Added
13
+
14
+ - On PyPI: `uv tool install dactopus`. A release on GitHub publishes its
15
+ tag (`.github/workflows/publish.yml`, trusted publishing); run by hand,
16
+ the workflow uploads a dev version to TestPyPI as a rehearsal. CI builds
17
+ the package, checks that its README renders as the PyPI page, and runs
18
+ the wheel outside the checkout. The README's links are absolute, so
19
+ they work on PyPI.
20
+
21
+ ### Changed
22
+
23
+ - dactopus-ossie-clickhouse 0.13.0, from PyPI in place of a git
24
+ reference to 0.12.0. It carries its own copy of apache-ossie, so
25
+ apache-ossie is no longer installed with the runner. `dactopus serve`
26
+ gains the questions the executor answers since 0.12.1
27
+ ([its changelog](https://github.com/Dactopus/dactopus-ossie-clickhouse/blob/main/CHANGELOG.md)).
28
+
29
+ ## [0.1.1] - 2026-10-30
30
+
31
+ ### Added
32
+
33
+ - `docs/design.md`: the decisions behind the runner, the reasons for
34
+ each and what each was verified on. The README gains the scope of
35
+ 0.1.0: what the runner does not do and what it is untested on.
36
+
37
+ ## [0.1.0] - 2026-10-30
38
+
39
+ The first release, a prototype.
40
+
41
+ ### Added
42
+
43
+ - The `dactopus` command with `init`, `run`, `check` and `serve`,
44
+ installed with `uv tool install` from a release tag on GitHub.
45
+ - The configuration file `dactopus.yaml`: the ClickHouse connection, over
46
+ TLS for an `https://` URL (port 8443 by default), with the password from
47
+ an environment variable, the databases for input and
48
+ canonical tables, the store's time zone (`store.timezone`, required),
49
+ the models library by release or by path, and the GA4 and Shopify
50
+ packages with their dbt variables.
51
+ - `dactopus init` writes a commented sample configuration.
52
+ - A release of the models library is downloaded once from GitHub into
53
+ `~/.cache/dactopus/`.
54
+ - `dactopus run` loads GA4 and Shopify and builds their packages. GA4: the
55
+ daily tables of the BigQuery export through the Storage Read API, the
56
+ last `ga4_lookback_days` reread and replaced on every run, as the
57
+ Google account the load names; a gcloud login bills the reads to the
58
+ configured project. Shopify: orders with lines, shipping and refunds,
59
+ then the products, archived ones and drafts included, and the
60
+ deletions of products from the store's events, from the GraphQL Admin
61
+ API (2026-10), each kind from an hour before its latest loaded change,
62
+ with a token from the app's client credentials (`read_orders` and
63
+ `read_products`); a models library before v0.7.0, whose Shopify input
64
+ has no `products`, is refused before anything is loaded, with the
65
+ version to set. A refund carries its `processedAt`, the date Shopify's
66
+ reports count it on and the models library's since v0.7.2; a library
67
+ before it, or a refunds table an earlier library created, is refused
68
+ the same way, naming the library's changelog for adding the column.
69
+ Each package is built with
70
+ `dbt build` and a dbt profile the runner writes, not the one in the
71
+ package; the profile and dbt's output go to `.dactopus/` next to the
72
+ configuration.
73
+ - Checks and the report. A check is a function of a ClickHouse client and
74
+ the configuration that returns findings of one shape (check, object,
75
+ severity, observed, expected, a sentence for the owner), registered
76
+ through the entry point group `dactopus.checks`. Built in: the models,
77
+ tests and unit tests of the last dbt run, the freshness of the input
78
+ against the package's thresholds (`dbt source freshness`), and a day
79
+ without rows in the last seven days that are over of a table the
80
+ package marks as a gap (`events`, `orders`), unless the same weekday
81
+ had none in the four weeks before.
82
+ `dactopus run` ends with the report, `dactopus check` runs `dbt test`
83
+ and the checks without loading or building; both write it to the
84
+ terminal and to `.dactopus/report.md` (freshness, what did not pass,
85
+ rows per day for the last seven days that are over, in
86
+ `store.timezone` for Shopify, against the four weeks before, then every
87
+ finding) and exit with 1 on a `fail` or `error` or when no check ran. An unknown
88
+ `store.timezone`, and `shopify_timezone` among the package's vars, are
89
+ refused before any connection. A dbt run
90
+ with failing tests or models no longer stops the command: its results
91
+ are in the report.
92
+ - `dactopus run --full-refresh [PACKAGE ...]` rebuilds the canonical
93
+ tables of the named packages (every configured package when none is
94
+ named) from the input the runner holds, with `dbt build --full-refresh`
95
+ after the usual load; the input is not reloaded. For a release of the
96
+ models library that changes a model's columns, which the `dbt` check
97
+ reports together with this command, or a change of the store's time
98
+ zone.
99
+ - `dactopus serve` runs the MCP server of dactopus-ossie-clickhouse 0.12.0
100
+ over stdio with the models of the configured packages
101
+ (`entities/web_analytics.yaml` for GA4, `entities/commerce.yaml` for
102
+ Shopify) read as one model, on the canonical database, over TLS for an
103
+ `https://` URL on any port, as the ClickHouse user of the optional
104
+ `serve` section (`user`, `password_env`) or else the user of
105
+ `clickhouse.url`, with "today" in `store.timezone` (the executor's
106
+ `--time-zone`). Each command reads only the password it connects
107
+ with. The password reaches the executor inside the process, never on a
108
+ command line.
109
+ - One time zone per installation, `store.timezone`: the runner passes it
110
+ to every package that declares a time zone variable and to `serve`.
111
+ With Shopify configured, `dactopus run` reads the shop's zone
112
+ (`shop { ianaTimezone }`) before loading and refuses a different
113
+ `store.timezone`, naming both; another name of the same zone (`UTC`
114
+ for `Etc/UTC`) passes. Before loading it also counts the rows of
115
+ the daily tables of each package with a time zone variable whose date
116
+ is not the day of their time in `store.timezone`, and stops on any with
117
+ the `dactopus run --full-refresh <package>` that rebuilds them; a
118
+ package rebuilt with `--full-refresh` is not counted. The built-in check
119
+ `time_zone` runs the same count after the build: a `fail` for a package
120
+ with a time zone variable, a `warn` with the share of rows for one
121
+ without (GA4, whose days are the property's), over the days
122
+ `rows_per_day` reads. Zone names come from the `tzdata` package on
123
+ every platform as well, so a name newer than the machine's own tz
124
+ database (`Europe/Kyiv`) is known.
125
+ - The runner reads what it sets and reads of a package from the
126
+ package's `deployment.yml` in the models library: the Ossie model
127
+ `serve` passes, the dbt variables that take the input database and the
128
+ time zone, and the tables `rows_per_day` counts, with the ones where a
129
+ day without rows is a finding. A renamed variable, model file or
130
+ counted table needs no release of the runner. The models library must be
131
+ v0.6.0 or later, the first release with the file; the sample
132
+ configuration names v0.8.0, and an older `models.version` is refused
133
+ with the version to set. The vars of a package are checked against
134
+ the file before any connection.
135
+
136
+ [Unreleased]: https://github.com/Dactopus/dactopus/compare/v0.2.0...HEAD
137
+ [0.2.0]: https://github.com/Dactopus/dactopus/releases/tag/v0.2.0
138
+ [0.1.1]: https://github.com/Dactopus/dactopus/releases/tag/v0.1.1
139
+ [0.1.0]: https://github.com/Dactopus/dactopus/releases/tag/v0.1.0
@@ -0,0 +1 @@
1
+ @AGENTS.md
@@ -0,0 +1,67 @@
1
+ # Contributing
2
+
3
+ Bug reports and questions are welcome as issues. Everything in this
4
+ repository is in English: code, comments, commit messages, issues.
5
+
6
+ ## Setup
7
+
8
+ Python 3.11+ and [uv](https://docs.astral.sh/uv/).
9
+
10
+ ```bash
11
+ git clone https://github.com/Dactopus/dactopus
12
+ cd dactopus
13
+ uv sync
14
+ uv run pytest -m "not integration" # unit tests, no server needed
15
+ ```
16
+
17
+ Integration tests need a ClickHouse server reachable over HTTP at
18
+ `DACTOPUS_CLICKHOUSE_URL` (user, host and port, without the password;
19
+ default `http://127.0.0.1:8123`), with the password in
20
+ `CLICKHOUSE_PASSWORD`, and skip without one.
21
+
22
+ ```bash
23
+ uv run ruff check src tests
24
+ uv run ruff format src tests
25
+ ```
26
+
27
+ ## Rules that shape the code
28
+
29
+ [AGENTS.md](AGENTS.md) is the full list. The scope section says what the
30
+ runner does and what it leaves to the library, the executor and the
31
+ client's own tools; read it before proposing a feature.
32
+
33
+ ## Pull requests
34
+
35
+ Small and focused. CI runs lint, the format check and the tests. By
36
+ submitting a contribution you agree it is licensed under the Apache
37
+ License 2.0, like the rest of the project.
38
+
39
+ ## Releasing
40
+
41
+ For maintainers. A release is the tag `vX.Y.Z` on a commit on `main`; the
42
+ GitHub release is titled `X.Y.Z`.
43
+
44
+ 1. Move the `[Unreleased]` entries in `CHANGELOG.md` under
45
+ `## [X.Y.Z] - YYYY-MM-DD`, leaving `[Unreleased]` empty.
46
+ 2. Set `version` in `pyproject.toml` and run `uv lock`.
47
+ 3. Push, wait for CI, merge with "Rebase and merge".
48
+ 4. Rehearse on `main`: run the Publish workflow by hand. It uploads the
49
+ build to [TestPyPI](https://test.pypi.org/project/dactopus/) as
50
+ `X.Y.Z.devN`; install that from there, its dependencies from PyPI
51
+ (anyone may upload to TestPyPI, under any name), and look at the page.
52
+ PyPI takes each version once, so whatever goes wrong is fixed here,
53
+ before the tag.
54
+ 5. Tag the commit on `main`, never one on the branch: rebasing changes the
55
+ hashes. Publishing the release on GitHub runs the Publish workflow,
56
+ which checks that the tag is the version in `pyproject.toml` and its
57
+ commit is on `main`, then uploads to PyPI. The release notes are the changelog section plus the
58
+ install line pinned to the version.
59
+
60
+ ```bash
61
+ git switch main && git pull --ff-only
62
+ gh workflow run publish.yml --ref main # step 4, TestPyPI
63
+ uvx pip download --no-deps --index-url https://test.pypi.org/simple/ -d testpypi "dactopus==X.Y.Z.devN"
64
+ uv tool install --force testpypi/dactopus-X.Y.Z.devN-py3-none-any.whl # dependencies from PyPI
65
+ git tag -a vX.Y.Z -m "dactopus X.Y.Z" && git push origin vX.Y.Z
66
+ gh release create vX.Y.Z --title X.Y.Z --notes-file notes.md # publishes to PyPI
67
+ ```