dactopus 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dactopus-0.2.0 → dactopus-0.3.0}/.github/workflows/ci.yml +1 -1
- {dactopus-0.2.0 → dactopus-0.3.0}/AGENTS.md +29 -17
- {dactopus-0.2.0 → dactopus-0.3.0}/CHANGELOG.md +94 -1
- {dactopus-0.2.0 → dactopus-0.3.0}/CONTRIBUTING.md +2 -2
- {dactopus-0.2.0 → dactopus-0.3.0}/PKG-INFO +146 -58
- {dactopus-0.2.0 → dactopus-0.3.0}/README.md +144 -56
- dactopus-0.3.0/docs/design.md +356 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/pyproject.toml +3 -2
- dactopus-0.3.0/src/dactopus/build.py +355 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/src/dactopus/checks.py +155 -116
- {dactopus-0.2.0 → dactopus-0.3.0}/src/dactopus/cli.py +83 -34
- dactopus-0.3.0/src/dactopus/config.py +479 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/src/dactopus/ga4.py +13 -12
- {dactopus-0.2.0 → dactopus-0.3.0}/src/dactopus/library.py +154 -19
- {dactopus-0.2.0 → dactopus-0.3.0}/src/dactopus/report.py +43 -19
- {dactopus-0.2.0 → dactopus-0.3.0}/src/dactopus/serve.py +14 -12
- {dactopus-0.2.0 → dactopus-0.3.0}/src/dactopus/shopify.py +20 -12
- {dactopus-0.2.0 → dactopus-0.3.0}/tests/conftest.py +21 -1
- {dactopus-0.2.0 → dactopus-0.3.0}/tests/fixtures/library/sources/ga4/deployment.yml +4 -2
- dactopus-0.3.0/tests/fixtures/library/sources/sensor_community/deployment.yml +18 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/tests/fixtures/library/sources/shopify/deployment.yml +1 -0
- dactopus-0.3.0/tests/fixtures/library/sources/woocommerce/deployment.yml +21 -0
- dactopus-0.3.0/tests/fixtures/library/unions/commerce/deployment.yml +9 -0
- dactopus-0.3.0/tests/fixtures/library/unions/iot/deployment.yml +9 -0
- dactopus-0.3.0/tests/fixtures/library/unions/web_analytics/deployment.yml +9 -0
- dactopus-0.3.0/tests/test_build.py +400 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/tests/test_checks.py +221 -54
- dactopus-0.3.0/tests/test_cli.py +385 -0
- dactopus-0.3.0/tests/test_config.py +357 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/tests/test_ga4.py +10 -11
- dactopus-0.3.0/tests/test_library.py +422 -0
- dactopus-0.3.0/tests/test_report.py +232 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/tests/test_serve.py +91 -18
- {dactopus-0.2.0 → dactopus-0.3.0}/tests/test_shopify.py +32 -18
- {dactopus-0.2.0 → dactopus-0.3.0}/uv.lock +6 -503
- dactopus-0.2.0/docs/design.md +0 -182
- dactopus-0.2.0/src/dactopus/build.py +0 -221
- dactopus-0.2.0/src/dactopus/config.py +0 -353
- dactopus-0.2.0/tests/test_build.py +0 -157
- dactopus-0.2.0/tests/test_cli.py +0 -288
- dactopus-0.2.0/tests/test_config.py +0 -217
- dactopus-0.2.0/tests/test_library.py +0 -187
- dactopus-0.2.0/tests/test_report.py +0 -131
- {dactopus-0.2.0 → dactopus-0.3.0}/.github/dependabot.yml +0 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/.github/workflows/publish.yml +0 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/.gitignore +0 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/CLAUDE.md +0 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/LICENSE +0 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/src/dactopus/__init__.py +0 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/src/dactopus/clickhouse.py +0 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/tests/fixtures/dbt/shopify/manifest.json +0 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/tests/fixtures/dbt/shopify/run_results_build_error.json +0 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/tests/fixtures/dbt/shopify/run_results_build_pass.json +0 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/tests/fixtures/dbt/shopify/run_results_build_schema_change.json +0 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/tests/fixtures/dbt/shopify/run_results_build_skipped.json +0 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/tests/fixtures/dbt/shopify/run_results_test_fail.json +0 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/tests/fixtures/dbt/shopify/sources_error.json +0 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/tests/fixtures/dbt/shopify/sources_pass.json +0 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/tests/fixtures/dbt/shopify/sources_stale.json +0 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/tests/fixtures/ga4_export_schema.arrow +0 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/tests/fixtures/library/sources/broken/deployment.yml +0 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/tests/fixtures/shopify_order_imported.json +0 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/tests/fixtures/shopify_orders.json +0 -0
- {dactopus-0.2.0 → dactopus-0.3.0}/tests/fixtures/shopify_products.json +0 -0
|
@@ -31,7 +31,7 @@ jobs:
|
|
|
31
31
|
- run: uvx twine check --strict dist/* # the README is the PyPI page
|
|
32
32
|
- name: The wheel works on its own, outside the checkout
|
|
33
33
|
run: |
|
|
34
|
-
uv venv "$RUNNER_TEMP/wheel"
|
|
34
|
+
uv venv --python 3.13 "$RUNNER_TEMP/wheel"
|
|
35
35
|
uv pip install --python "$RUNNER_TEMP/wheel" "$(ls dist/*.whl)"
|
|
36
36
|
cd "$RUNNER_TEMP" && wheel/bin/dactopus init --config sample.yaml
|
|
37
37
|
wheel/bin/python -c "import dactopus.serve, dactopus_ossie_clickhouse.mcp_server"
|
|
@@ -29,15 +29,15 @@ introduce text in any other language, even in examples or fixtures.
|
|
|
29
29
|
In scope:
|
|
30
30
|
|
|
31
31
|
- One configuration file per installation: the ClickHouse connection and
|
|
32
|
-
the user `serve` connects as, the
|
|
33
|
-
|
|
34
|
-
|
|
32
|
+
the user `serve` connects as, the name the installation's databases
|
|
33
|
+
start with, its time zone, the version of the models library, and the
|
|
34
|
+
sources, each under a label with its package, zone and parameters.
|
|
35
35
|
- Loading GA4: the BigQuery export, read with the Storage Read API, into
|
|
36
36
|
the GA4 package's input table.
|
|
37
37
|
- Loading Shopify: orders with lines, shipping and refunds, and the
|
|
38
38
|
products with their deletions, read page by page from the GraphQL
|
|
39
39
|
Admin API, into the Shopify package's input tables.
|
|
40
|
-
- Running the packages with dbt.
|
|
40
|
+
- Running the packages and the unions with dbt.
|
|
41
41
|
- Checks behind a plugin interface, built-in checks, and one report a
|
|
42
42
|
store owner can read.
|
|
43
43
|
- Serving the models over MCP, as a thin call to dactopus-ossie-clickhouse.
|
|
@@ -58,7 +58,7 @@ Out of scope, do not build:
|
|
|
58
58
|
reads several files as one model; the runner only lists them.
|
|
59
59
|
- Migration from a client's dbt or Looker.
|
|
60
60
|
- An installer, a Docker image, a single-file build. Installation is
|
|
61
|
-
`uv
|
|
61
|
+
`uv tool install` from PyPI.
|
|
62
62
|
- Placeholders or stubs for any of the above.
|
|
63
63
|
|
|
64
64
|
## Design principles
|
|
@@ -72,9 +72,13 @@ Out of scope, do not build:
|
|
|
72
72
|
in the library, never patched around in the runner.
|
|
73
73
|
- The runner knows a package only by its `input.sql` and its
|
|
74
74
|
`sources/<package>/deployment.yml`: the Ossie model its tables serve,
|
|
75
|
-
the dbt variables that take the input database
|
|
76
|
-
the
|
|
77
|
-
|
|
75
|
+
the dbt variables that take the input database, the label and the
|
|
76
|
+
time zone, the zone its days are in when the package fixes it, and the
|
|
77
|
+
tables counted per day. A union, by its
|
|
78
|
+
`unions/<union>/deployment.yml`: the model it serves and the dbt
|
|
79
|
+
variable that takes the databases of its sources. A variable, table,
|
|
80
|
+
column or model file name of a package or a union never appears in
|
|
81
|
+
the runner's code. Two exceptions:
|
|
78
82
|
the loaders, which write the tables `input.sql` defines (the GA4
|
|
79
83
|
loader also rereads as many days as the package's
|
|
80
84
|
`ga4_lookback_days`), and the commented sample `init` writes.
|
|
@@ -95,7 +99,11 @@ Out of scope, do not build:
|
|
|
95
99
|
|
|
96
100
|
## Stack
|
|
97
101
|
|
|
98
|
-
Python 3.11
|
|
102
|
+
Python 3.11 to 3.13, managed with uv: dbt-core 1.11 does not start on
|
|
103
|
+
3.14, and dbt-core 1.12, which does, cannot be installed without access
|
|
104
|
+
to GitHub ([docs/design.md](docs/design.md)). Every dependency comes from
|
|
105
|
+
PyPI: PyPI refuses a package that depends on a git reference, so one
|
|
106
|
+
breaks the next release. The parts that constrain code:
|
|
99
107
|
|
|
100
108
|
- **dbt v1**: `dbt-core` 1.11 with `dbt-clickhouse`. Packages are dbt
|
|
101
109
|
projects and also run on dbt v2, but its ClickHouse adapter is in beta
|
|
@@ -123,9 +131,12 @@ The models library is not a Python package. The configuration names a
|
|
|
123
131
|
version (`models.version`: a release tag, fetched as the GitHub archive
|
|
124
132
|
of the tag into `~/.cache/dactopus/`, no git needed) or a path
|
|
125
133
|
(`models.path`, for development). Packages run one at a time with
|
|
126
|
-
`dbt build --project-dir <library>/sources/<package>`,
|
|
127
|
-
|
|
128
|
-
|
|
134
|
+
`dbt build --project-dir <library>/sources/<package>`, one per source,
|
|
135
|
+
then each union that has sources with `--project-dir
|
|
136
|
+
<library>/unions/<union>`, with a profile the runner writes; the package
|
|
137
|
+
names it (`profile: dactopus`). Packages do not depend on each other; a
|
|
138
|
+
union reads the databases of its sources; no root dbt project is
|
|
139
|
+
generated.
|
|
129
140
|
|
|
130
141
|
## Architecture rules
|
|
131
142
|
|
|
@@ -152,18 +163,19 @@ do not depend on each other; no root dbt project is generated.
|
|
|
152
163
|
- A check is a named object that takes a ClickHouse connection and the
|
|
153
164
|
installation's configuration and returns findings. A finding has one
|
|
154
165
|
shape for built-in and plugin checks alike: check, object, severity,
|
|
155
|
-
observed, expected,
|
|
166
|
+
observed, expected, a sentence for the owner, and optionally the
|
|
167
|
+
source it is about (a label, or `union-<name>` for a union). Checks register
|
|
156
168
|
through Python entry points; built-in ones are dbt test results (from
|
|
157
169
|
`target/run_results.json` with `manifest.json` for the test's kind,
|
|
158
170
|
column and parameters), input freshness, rows per day, and the time
|
|
159
|
-
zone of the daily tables against
|
|
171
|
+
zone of the daily tables against the source's zone. The report
|
|
160
172
|
is Markdown, to the terminal and to a file, three answers first (how
|
|
161
173
|
fresh the data is; whether every check passed and which did not, in
|
|
162
174
|
words; volumes per day this week against past weeks), details below.
|
|
163
175
|
The exit code is non-zero on a `fail` or `error` finding.
|
|
164
|
-
- `serve` passes the model file each
|
|
165
|
-
the connection; the executor combines them. The runner
|
|
166
|
-
merge or edit model files.
|
|
176
|
+
- `serve` passes the model file of each union that has configured
|
|
177
|
+
sources and the connection; the executor combines them. The runner
|
|
178
|
+
does not read, merge or edit model files.
|
|
167
179
|
- Memory: a daily GA4 table of a mid-size brand is hundreds of megabytes
|
|
168
180
|
of Arrow. Keep it; read in `ReadRows` batches when a load exceeds it.
|
|
169
181
|
- Errors name the step, the object and the next action in plain words;
|
|
@@ -7,6 +7,97 @@ a minor release may change the command line and the configuration file.
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.3.0] - 2026-11-03
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- Any package of the models library: a source names it in `package`. The
|
|
15
|
+
runner loads GA4 and Shopify; it builds, checks and serves the others
|
|
16
|
+
(WooCommerce, Sensor.Community) over the input the client's transport
|
|
17
|
+
(CDC) or the client writes, and leaves that database alone.
|
|
18
|
+
- Several stores, GA4 properties or sensor networks in one installation,
|
|
19
|
+
each a source under its own label: 1 to 64 of `a-z`, `0-9` and `_`,
|
|
20
|
+
passed to the package as its `source_id`. Each source builds into a
|
|
21
|
+
database of its own, `<clickhouse.database>_<label>`, from its input in
|
|
22
|
+
`<clickhouse.database>_<label>_raw` or the database its
|
|
23
|
+
`input_database` names. A label whose database is another source's
|
|
24
|
+
input, and two sources of one package reading one input database, are
|
|
25
|
+
refused.
|
|
26
|
+
- A source takes `timezone`, the zone its days are in: a store package is
|
|
27
|
+
built in it, and the time zone check counts the days of the source's
|
|
28
|
+
tables in it (for GA4 the property's zone). A package that fixes the
|
|
29
|
+
zone of its days declares it as `zone` in its `deployment.yml`
|
|
30
|
+
(Sensor.Community: UTC) and the runner uses it; a `timezone` on a
|
|
31
|
+
source of such a package is refused. The report says when a source's
|
|
32
|
+
days are in another zone than the installation's.
|
|
33
|
+
- A source dbt cannot build, test or measure (a package's required
|
|
34
|
+
variable missing, as `woocommerce_currency`) no longer stops the run:
|
|
35
|
+
the other sources go on, the report shows dbt's error with the
|
|
36
|
+
package's message, and the run exits with 1.
|
|
37
|
+
- A finding has an optional `source`, the label of the source it is
|
|
38
|
+
about. The report groups the findings and the rows per day by source,
|
|
39
|
+
in the order of the configuration, and names the source on each line of
|
|
40
|
+
its three answers.
|
|
41
|
+
- The unions of the models library: `dactopus run` builds each union
|
|
42
|
+
that has sources after the sources, as views in `clickhouse.database`
|
|
43
|
+
over the databases of its sources, and `dactopus check` tests them.
|
|
44
|
+
Their findings have `union-<name>` as their `source` (`union-commerce`),
|
|
45
|
+
and the report names the union in words ("the commerce union"). The
|
|
46
|
+
views belong to the user of `clickhouse.url`: to change that user, run
|
|
47
|
+
`dactopus run` as the new one first, then drop the old one.
|
|
48
|
+
|
|
49
|
+
### Changed
|
|
50
|
+
|
|
51
|
+
- The configuration file. `sources`, a mapping of labels to sources,
|
|
52
|
+
replaces `packages`; `timezone` at the top replaces `store.timezone`
|
|
53
|
+
and is the installation's zone, "today" in `dactopus serve` and the
|
|
54
|
+
zone of each source that names none; `clickhouse.input_database` is
|
|
55
|
+
gone, and `clickhouse.database` is the prefix of the sources' databases.
|
|
56
|
+
A configuration of 0.2 is refused: write it again from `dactopus init`.
|
|
57
|
+
- `dactopus run --full-refresh` takes labels.
|
|
58
|
+
- `dactopus serve` serves the model of each union that has sources, one
|
|
59
|
+
per area (commerce, web analytics), in place of each package's model.
|
|
60
|
+
The `serve` user needs `SELECT` on `clickhouse.database` alone: the
|
|
61
|
+
views read the sources' databases as their owner.
|
|
62
|
+
- `dactopus run` refuses, before loading, a `clickhouse.database` that
|
|
63
|
+
holds tables, such as the canonical tables of 0.2: the unions would
|
|
64
|
+
replace those of the same name with views. Give `clickhouse.database`
|
|
65
|
+
another name, or drop the old tables if nothing reads them.
|
|
66
|
+
- A source whose package's model no union of the library serves is
|
|
67
|
+
refused before anything runs.
|
|
68
|
+
- dbt's output for each source goes to `.dactopus/<label>/`, with the
|
|
69
|
+
profile that writes to the source's database.
|
|
70
|
+
- The models library v0.10.0 or later: an earlier release is refused
|
|
71
|
+
before anything runs, with the version to set. The sample configuration
|
|
72
|
+
and the tests use v0.10.0. The comment on `models.version` in the
|
|
73
|
+
sample says to run `dactopus run --full-refresh` once after moving to a
|
|
74
|
+
release that changes a model's columns.
|
|
75
|
+
- A package the library does not have is named with the packages it
|
|
76
|
+
has; a source's `vars` may not set the variables of the label, the
|
|
77
|
+
input database or the time zone.
|
|
78
|
+
|
|
79
|
+
### Fixed
|
|
80
|
+
|
|
81
|
+
- The rows per day of a GA4 source in the report: each table ended its
|
|
82
|
+
week at its own latest day, so the purchases of a site whose last
|
|
83
|
+
purchase came days before its last events were counted over another
|
|
84
|
+
week than the events. Every table of a source without a time zone now
|
|
85
|
+
ends at the latest day of any of them, and a day without rows is 0.
|
|
86
|
+
|
|
87
|
+
## [0.2.1] - 2026-11-01
|
|
88
|
+
|
|
89
|
+
### Fixed
|
|
90
|
+
|
|
91
|
+
- Python 3.11 to 3.13. On 3.14 the install succeeded and the first
|
|
92
|
+
`dactopus run` failed inside dbt: dbt-core 1.11 does not start there.
|
|
93
|
+
The README installs with `uv tool install --python 3.13 dactopus`;
|
|
94
|
+
every command refuses 3.14 and gives that command with `--force`;
|
|
95
|
+
`requires-python` is `>=3.11,<3.14`, which pip enforces and uv does
|
|
96
|
+
not.
|
|
97
|
+
- The README says how to update (`uv tool upgrade dactopus`) and how an
|
|
98
|
+
installation of 0.1.0 or 0.1.1 from GitHub moves to PyPI
|
|
99
|
+
(`uv tool install --force --python 3.13 dactopus`).
|
|
100
|
+
|
|
10
101
|
## [0.2.0] - 2026-11-01
|
|
11
102
|
|
|
12
103
|
### Added
|
|
@@ -133,7 +224,9 @@ The first release, a prototype.
|
|
|
133
224
|
with the version to set. The vars of a package are checked against
|
|
134
225
|
the file before any connection.
|
|
135
226
|
|
|
136
|
-
[Unreleased]: https://github.com/Dactopus/dactopus/compare/v0.
|
|
227
|
+
[Unreleased]: https://github.com/Dactopus/dactopus/compare/v0.3.0...HEAD
|
|
228
|
+
[0.3.0]: https://github.com/Dactopus/dactopus/releases/tag/v0.3.0
|
|
229
|
+
[0.2.1]: https://github.com/Dactopus/dactopus/releases/tag/v0.2.1
|
|
137
230
|
[0.2.0]: https://github.com/Dactopus/dactopus/releases/tag/v0.2.0
|
|
138
231
|
[0.1.1]: https://github.com/Dactopus/dactopus/releases/tag/v0.1.1
|
|
139
232
|
[0.1.0]: https://github.com/Dactopus/dactopus/releases/tag/v0.1.0
|
|
@@ -5,7 +5,7 @@ repository is in English: code, comments, commit messages, issues.
|
|
|
5
5
|
|
|
6
6
|
## Setup
|
|
7
7
|
|
|
8
|
-
Python 3.11
|
|
8
|
+
Python 3.11 to 3.13 and [uv](https://docs.astral.sh/uv/).
|
|
9
9
|
|
|
10
10
|
```bash
|
|
11
11
|
git clone https://github.com/Dactopus/dactopus
|
|
@@ -61,7 +61,7 @@ GitHub release is titled `X.Y.Z`.
|
|
|
61
61
|
git switch main && git pull --ff-only
|
|
62
62
|
gh workflow run publish.yml --ref main # step 4, TestPyPI
|
|
63
63
|
uvx pip download --no-deps --index-url https://test.pypi.org/simple/ -d testpypi "dactopus==X.Y.Z.devN"
|
|
64
|
-
uv tool install --force testpypi/dactopus-X.Y.Z.devN-py3-none-any.whl # dependencies from PyPI
|
|
64
|
+
uv tool install --force --python 3.13 testpypi/dactopus-X.Y.Z.devN-py3-none-any.whl # dependencies from PyPI
|
|
65
65
|
git tag -a vX.Y.Z -m "dactopus X.Y.Z" && git push origin vX.Y.Z
|
|
66
66
|
gh release create vX.Y.Z --title X.Y.Z --notes-file notes.md # publishes to PyPI
|
|
67
67
|
```
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: dactopus
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Runner of Dactopus: loads sources into ClickHouse, builds the canonical tables, runs checks, serves the models over MCP
|
|
5
5
|
Project-URL: Homepage, https://github.com/Dactopus/dactopus
|
|
6
6
|
Project-URL: Issues, https://github.com/Dactopus/dactopus/issues
|
|
@@ -16,7 +16,7 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
16
16
|
Classifier: Programming Language :: Python :: 3.12
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.13
|
|
18
18
|
Classifier: Topic :: Database
|
|
19
|
-
Requires-Python:
|
|
19
|
+
Requires-Python: <3.14,>=3.11
|
|
20
20
|
Requires-Dist: certifi>=2024.2
|
|
21
21
|
Requires-Dist: clickhouse-connect[arrow]>=0.8
|
|
22
22
|
Requires-Dist: dactopus-ossie-clickhouse[mcp]~=0.13.0
|
|
@@ -56,16 +56,23 @@ The executor knows nothing of Dactopus.
|
|
|
56
56
|
|
|
57
57
|
## Install
|
|
58
58
|
|
|
59
|
-
|
|
59
|
+
[uv](https://docs.astral.sh/uv/). The runner runs on Python 3.11 to 3.13;
|
|
60
|
+
`--python 3.13` makes uv use that version, and download it when the
|
|
61
|
+
machine has none.
|
|
60
62
|
|
|
61
63
|
```bash
|
|
62
|
-
uv tool install dactopus
|
|
64
|
+
uv tool install --python 3.13 dactopus
|
|
63
65
|
```
|
|
64
66
|
|
|
65
67
|
The `dactopus` command gets its own environment with dbt, the dbt
|
|
66
68
|
ClickHouse adapter and the executor. The machine needs no ClickHouse
|
|
67
69
|
binaries; the runner connects to the client's ClickHouse server over HTTP.
|
|
68
70
|
|
|
71
|
+
`uv tool upgrade dactopus` updates it. An installation of 0.1.0 or
|
|
72
|
+
0.1.1, which came from GitHub, moves to PyPI with
|
|
73
|
+
`uv tool install --force --python 3.13 dactopus`: `uv tool upgrade`
|
|
74
|
+
keeps it on the GitHub tag.
|
|
75
|
+
|
|
69
76
|
Then, in the folder of the installation:
|
|
70
77
|
|
|
71
78
|
```bash
|
|
@@ -79,34 +86,71 @@ dactopus serve # the MCP server for an AI agent, over stdio
|
|
|
79
86
|
| Command | What it does |
|
|
80
87
|
| --- | --- |
|
|
81
88
|
| `dactopus init` | Writes a commented sample configuration |
|
|
82
|
-
| `dactopus run` | Loads the sources, builds the
|
|
83
|
-
| `dactopus check` |
|
|
89
|
+
| `dactopus run` | Loads the sources, builds them and the unions, runs the checks, writes the report |
|
|
90
|
+
| `dactopus check` | Tests the sources and the unions, runs the checks and writes the report |
|
|
84
91
|
| `dactopus serve` | Serves the models over MCP |
|
|
85
92
|
|
|
86
93
|
One configuration file describes an installation: the ClickHouse
|
|
87
|
-
connection
|
|
88
|
-
time zone, the version of the models library, and the
|
|
94
|
+
connection and the name its databases start with, the installation's
|
|
95
|
+
time zone, the version of the models library, and the sources.
|
|
89
96
|
`dactopus init` writes it with a comment on every field. Secrets stay out
|
|
90
97
|
of it: the file names the environment variables that hold them.
|
|
91
98
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
99
|
+
## Sources
|
|
100
|
+
|
|
101
|
+
A source is a store, a GA4 property or a sensor network, under a label
|
|
102
|
+
of its own: 1 to 64 of `a-z`, `0-9` and `_`. Agents and BI see the
|
|
103
|
+
label, so give the name people call the store by (`main_store`,
|
|
104
|
+
`europe_store`, `website`). `package` names a package of the models
|
|
105
|
+
library; the runner passes the label to it as its `source_id`, which
|
|
106
|
+
tells the sources apart in the unions.
|
|
107
|
+
|
|
108
|
+
Every source has databases of its own, named after `clickhouse.database`
|
|
109
|
+
(`dactopus` in the sample):
|
|
110
|
+
|
|
111
|
+
| Database | Holds |
|
|
112
|
+
| --- | --- |
|
|
113
|
+
| `dactopus` | The views of the unions, which `serve` reads |
|
|
114
|
+
| `dactopus_<label>` | The canonical tables of the source |
|
|
115
|
+
| `dactopus_<label>_raw` | The input of the source, unless the source names another in `input_database` |
|
|
116
|
+
|
|
117
|
+
The runner loads GA4 and Shopify. The input of another package
|
|
118
|
+
(WooCommerce, Sensor.Community) is written by the client's transport
|
|
119
|
+
(CDC: PeerDB, ClickPipes) or by the client, into the database the
|
|
120
|
+
source names in `input_database`; the runner builds, checks and serves
|
|
121
|
+
it and leaves that database alone. A package's required variables, such
|
|
122
|
+
as `woocommerce_currency`, go in the source's `vars`.
|
|
123
|
+
|
|
124
|
+
`timezone` at the top of the file is the installation's time zone, an
|
|
125
|
+
IANA name such as `America/New_York`: "today" in an agent's question is
|
|
126
|
+
the date in it. A source whose days are in another zone names it in its
|
|
127
|
+
own `timezone`: for Shopify the store's zone (Settings > General in the
|
|
128
|
+
admin), for GA4 the property's (Admin > Property details). A package that
|
|
129
|
+
fixes the zone of its days declares it in its `deployment.yml` in the
|
|
130
|
+
models library (Sensor.Community: `UTC`); a source of such a package names
|
|
131
|
+
no `timezone`, and the runner refuses one. The rows of a source keep the
|
|
132
|
+
days of its zone; "today" is the installation's date for every source. The
|
|
133
|
+
runner passes a source's zone to its package when the package takes one
|
|
134
|
+
(`shopify_timezone` for Shopify, which the source's `vars` may not set).
|
|
135
|
+
|
|
136
|
+
For a Shopify source, `dactopus run` first reads the shop's zone and
|
|
137
|
+
refuses one that differs from the source's; another name of the same
|
|
138
|
+
zone (`UTC` for `Etc/UTC`) passes. It then counts, in the daily tables
|
|
139
|
+
of each source whose package takes a zone, the rows whose date is not
|
|
140
|
+
the day of their time in the source's zone. Such a row means the tables
|
|
141
|
+
were built in another zone: the Shopify models are incremental, and a
|
|
142
|
+
build after the zone changed would leave the orders it does not rebuild
|
|
143
|
+
on the days of the old one. The run stops before loading, with the
|
|
144
|
+
command that rebuilds the source, `dactopus run --full-refresh
|
|
145
|
+
main_store`; the agent keeps answering from the current tables until
|
|
146
|
+
then.
|
|
147
|
+
|
|
148
|
+
## Running
|
|
149
|
+
|
|
150
|
+
`dactopus run` loads each source it has a loader for into the input
|
|
151
|
+
tables of its package, then runs `dbt build` of each source's package
|
|
152
|
+
into the source's database, then of each union that has sources (see
|
|
153
|
+
"Serving the models"). A load reads only what changed:
|
|
110
154
|
|
|
111
155
|
| Source | Read from | Next load |
|
|
112
156
|
| --- | --- | --- |
|
|
@@ -119,7 +163,8 @@ application-default login`). The read sessions run in the configured
|
|
|
119
163
|
project, which pays for them. Shopify needs an app with `read_orders`
|
|
120
164
|
and `read_products`, installed on the store, and its client id and
|
|
121
165
|
secret; without `read_all_orders` Shopify returns the last 60 days of
|
|
122
|
-
orders only.
|
|
166
|
+
orders only. Two Shopify stores name different environment variables in
|
|
167
|
+
`client_id_env` and `client_secret_env`.
|
|
123
168
|
|
|
124
169
|
A load writes into the input tables the models library's `input.sql`
|
|
125
170
|
creates, and `input.sql` creates a table only if it is missing. When a
|
|
@@ -128,53 +173,80 @@ the load stops before reading anything and names the table, the column
|
|
|
128
173
|
and the library's changelog, which says how to add it: the Shopify
|
|
129
174
|
refunds' `processed_at` of v0.7.2, the date a refund counts on.
|
|
130
175
|
|
|
176
|
+
A source dbt cannot build, test or measure (a required variable of its
|
|
177
|
+
package missing, say) does not stop the others: the report shows dbt's
|
|
178
|
+
error with the package's message, and the run exits with 1.
|
|
179
|
+
|
|
131
180
|
The runner writes the dbt profile the packages name (`profile: dactopus`)
|
|
132
181
|
from the configuration, without the password, which dbt reads from the
|
|
133
|
-
environment. The profile
|
|
134
|
-
|
|
182
|
+
environment. The profile and dbt's output go to `.dactopus/<label>/` next
|
|
183
|
+
to the configuration file, those of a union to `.dactopus/union-<name>/`,
|
|
184
|
+
and the report to `.dactopus/report.md`.
|
|
135
185
|
|
|
136
|
-
`dactopus run --full-refresh [
|
|
137
|
-
then builds the named
|
|
138
|
-
|
|
186
|
+
`dactopus run --full-refresh [LABEL ...]` loads the sources as usual,
|
|
187
|
+
then builds the named sources (every source when none is named) with
|
|
188
|
+
`dbt build --full-refresh`: the canonical tables are rebuilt
|
|
139
189
|
from the input the runner already holds, which keeps every version of
|
|
140
190
|
every row. The input is not reloaded from the start: the usual
|
|
141
191
|
incremental load adds only what is new. Run it once when a
|
|
142
192
|
release of the models library changes a model's columns (the next
|
|
143
193
|
incremental build stops with an error, and the report names the table and
|
|
144
|
-
this command) or when
|
|
194
|
+
this command) or when a source's time zone changes. Each table exists
|
|
145
195
|
twice on disk until its rebuild ends: dbt builds the new table next to the
|
|
146
196
|
old one and swaps them when it is complete, so `serve` keeps answering
|
|
147
197
|
from the old table meanwhile.
|
|
148
198
|
|
|
199
|
+
## Upgrading from 0.2
|
|
200
|
+
|
|
201
|
+
The runner refuses a configuration of 0.2: write it again from
|
|
202
|
+
`dactopus init`, with each entry of `packages` as a source under a
|
|
203
|
+
label and `store.timezone` as `timezone`.
|
|
204
|
+
|
|
205
|
+
- The input already loaded stays where it is: give each source
|
|
206
|
+
`input_database: dactopus_raw` (the old `clickhouse.input_database`),
|
|
207
|
+
and the next load is incremental. Sources of different packages may
|
|
208
|
+
share an input database; two of one package may not.
|
|
209
|
+
- The canonical tables are built again, in the database of each source;
|
|
210
|
+
no `--full-refresh` is needed.
|
|
211
|
+
- The old canonical tables are in `dactopus`, the database of the
|
|
212
|
+
unions. `dactopus run` refuses to build views over them before loading
|
|
213
|
+
anything: give `clickhouse.database` another name, or drop the old
|
|
214
|
+
tables if nothing reads them.
|
|
215
|
+
|
|
149
216
|
## Checks and the report
|
|
150
217
|
|
|
151
|
-
After the build, `dactopus run` measures the freshness of
|
|
152
|
-
(`dbt source freshness`, with the thresholds the package sets), runs
|
|
153
|
-
checks and writes the report, to the terminal and to
|
|
154
|
-
`dactopus check` does the same over the tables as
|
|
155
|
-
|
|
218
|
+
After the build, `dactopus run` measures the freshness of each source's
|
|
219
|
+
input (`dbt source freshness`, with the thresholds the package sets), runs
|
|
220
|
+
the checks and writes the report, to the terminal and to
|
|
221
|
+
`.dactopus/report.md`. `dactopus check` does the same over the tables as
|
|
222
|
+
they are, with `dbt test` of the sources and the unions in place of the
|
|
223
|
+
build. The exit code is 1 when a check failed or could not
|
|
156
224
|
run, or when no check ran at all, for cron.
|
|
157
225
|
|
|
158
226
|
The report answers three questions first: how fresh the data is, whether
|
|
159
227
|
every check passed and in words which did not, and how many rows each
|
|
160
228
|
canonical table got per day over its last seven days that are over against
|
|
161
|
-
the four weeks before. A Shopify day is over at midnight in
|
|
162
|
-
a GA4 daily table is a day that is over.
|
|
163
|
-
finding follows below
|
|
229
|
+
the four weeks before. A Shopify day is over at midnight in the source's
|
|
230
|
+
zone; a GA4 daily table is a day that is over. The answers name the source
|
|
231
|
+
on each line. Every finding follows below, by source in the order of the
|
|
232
|
+
configuration, then the unions; the rows per day of a source whose days
|
|
233
|
+
are in another zone than the installation's say so.
|
|
164
234
|
|
|
165
235
|
The built-in checks:
|
|
166
236
|
|
|
167
237
|
| Check | Reads | Finding |
|
|
168
238
|
| --- | --- | --- |
|
|
169
|
-
| `dbt` | `run_results.json` and `manifest.json` of the last dbt run | Each model, test and unit test
|
|
239
|
+
| `dbt` | `run_results.json` and `manifest.json` of the last dbt run of each source and union | Each model, test and unit test, as a sentence with the count of failing rows |
|
|
170
240
|
| `freshness` | `sources.json` of `dbt source freshness` | The age of the latest input row against the package's `warn_after` and `error_after` |
|
|
171
241
|
| `rows_per_day` | The canonical tables | A day without rows in the last seven days that are over of a table the package marks as a gap in its `deployment.yml` (`events` for GA4, `orders` for Shopify), unless the same weekday had none in the four weeks before |
|
|
172
|
-
| `time_zone` | The daily tables with a time column in `deployment.yml` | Rows whose date is not the day of their time in
|
|
242
|
+
| `time_zone` | The daily tables with a time column in `deployment.yml` | Rows whose date is not the day of their time in the source's zone: a `fail` for a package built in it (Shopify, every row) or one that fixes it (Sensor.Community), a `warn` with the share of rows for one whose days come with the data (GA4, the property's zone); the last two counted over the days `rows_per_day` reads |
|
|
173
243
|
|
|
174
244
|
A check is a function of a ClickHouse client and the configuration that
|
|
175
245
|
returns findings; a finding names the check, the object, a severity
|
|
176
246
|
(`pass`, `skip`, `warn`, `fail`, `error`), what was observed, what was
|
|
177
|
-
expected,
|
|
247
|
+
expected, one sentence for the store owner, and optionally the source it
|
|
248
|
+
is about (`source`: a label, or `union-<name>` for a union, such as
|
|
249
|
+
`union-commerce`). A package registers a check
|
|
178
250
|
through the entry point group `dactopus.checks`; the entry point's name is
|
|
179
251
|
the check's name, and a check that raises becomes an `error` finding
|
|
180
252
|
without stopping the others:
|
|
@@ -188,13 +260,16 @@ reconciliation = "my_checks:reconcile_with_shopify"
|
|
|
188
260
|
|
|
189
261
|
`dactopus serve` runs the MCP server of dactopus-ossie-clickhouse over
|
|
190
262
|
stdio, for an AI agent to start as a subprocess. It serves the model
|
|
191
|
-
|
|
192
|
-
|
|
263
|
+
of each union of the library that has sources, read as one model. A
|
|
264
|
+
union is a set of views in `clickhouse.database` over the canonical
|
|
265
|
+
tables of every source of its kind, which `dactopus run` builds after
|
|
266
|
+
the sources:
|
|
193
267
|
|
|
194
|
-
|
|
|
195
|
-
| --- | --- |
|
|
196
|
-
| GA4 | `entities/web_analytics.yaml` |
|
|
197
|
-
| Shopify | `entities/commerce.yaml` |
|
|
268
|
+
| Union | Its sources | Model file of the library |
|
|
269
|
+
| --- | --- | --- |
|
|
270
|
+
| `web_analytics` | GA4 | `entities/web_analytics.yaml` |
|
|
271
|
+
| `commerce` | Shopify, WooCommerce | `entities/commerce.yaml` |
|
|
272
|
+
| `iot` | Sensor.Community | `entities/iot.yaml` |
|
|
198
273
|
|
|
199
274
|
The models name their tables without a database; the executor reads them
|
|
200
275
|
in `clickhouse.database`. The agent sees and runs only what its ClickHouse
|
|
@@ -213,9 +288,23 @@ CREATE USER agent IDENTIFIED BY '...';
|
|
|
213
288
|
GRANT SELECT ON dactopus.* TO agent;
|
|
214
289
|
```
|
|
215
290
|
|
|
216
|
-
|
|
217
|
-
the
|
|
218
|
-
`
|
|
291
|
+
One grant is enough. The views of the unions read the tables of the
|
|
292
|
+
sources as the user who built them, the user of `clickhouse.url`, so the
|
|
293
|
+
agent needs `SELECT` on `clickhouse.database` and none on the databases
|
|
294
|
+
of the sources or their inputs. What belongs to the agent's own user
|
|
295
|
+
still applies: its grants (the executor leaves out of the model what
|
|
296
|
+
they do not cover), its settings profile and its quotas. A row policy on
|
|
297
|
+
a source's table applies as the user of `clickhouse.url`, not as the
|
|
298
|
+
agent; a row restriction for one agent goes on the union's views, as a
|
|
299
|
+
policy `TO` that agent.
|
|
300
|
+
|
|
301
|
+
The views belong to the user of `clickhouse.url`, and ClickHouse does not
|
|
302
|
+
drop a user that owns views. To change that user, run `dactopus run` as the
|
|
303
|
+
new one first, which recreates the views under it; then the old one can
|
|
304
|
+
be dropped.
|
|
305
|
+
|
|
306
|
+
Each command reads only the password it connects with: `serve` needs the
|
|
307
|
+
variable `serve.password_env` names, `run` and `check` the one
|
|
219
308
|
`clickhouse.password_env` names. Without the `serve` section, `serve` runs
|
|
220
309
|
as the user of `clickhouse.url`.
|
|
221
310
|
|
|
@@ -251,10 +340,9 @@ of the executor.
|
|
|
251
340
|
|
|
252
341
|
The runner does not:
|
|
253
342
|
|
|
254
|
-
- load sources other than GA4 and Shopify
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
client sets up, with the runner reading the tables it finds;
|
|
343
|
+
- load sources other than GA4 and Shopify. A database (WooCommerce on
|
|
344
|
+
MySQL) is to reach ClickHouse by CDC the client sets up, with the
|
|
345
|
+
runner building its package over the tables it finds;
|
|
258
346
|
- schedule runs, send alerts or keep a run history; runs are started by
|
|
259
347
|
hand or by cron;
|
|
260
348
|
- link GA4 sessions to Shopify orders; the two models answer apart;
|
|
@@ -270,7 +358,7 @@ runner and what each was verified on:
|
|
|
270
358
|
|
|
271
359
|
## Development
|
|
272
360
|
|
|
273
|
-
Python 3.11
|
|
361
|
+
Python 3.11 to 3.13 and [uv](https://docs.astral.sh/uv/).
|
|
274
362
|
|
|
275
363
|
```bash
|
|
276
364
|
uv sync
|