fdl-cli 0.19.0__tar.gz → 0.20.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/CHANGELOG.md +14 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/PKG-INFO +1 -1
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/reference/cli.md +2 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/spec/index.md +1 -1
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/spec/layout.md +2 -1
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/spec/versions.md +45 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/spec/writing.md +2 -2
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/reference/cli.md +2 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/spec/index.md +1 -1
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/spec/layout.md +2 -1
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/spec/versions.md +45 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/spec/writing.md +2 -2
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/pyproject.toml +1 -1
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/src/fdl/cli.py +17 -3
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/src/fdl/layout.py +2 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/src/fdl/objects.py +12 -2
- fdl_cli-0.20.0/src/fdl/profiles.py +296 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/src/fdl/repo.py +59 -1
- fdl_cli-0.20.0/tests/test_profiles.py +131 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/.github/workflows/docs.yml +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/.github/workflows/publish.yml +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/.github/workflows/release.yml +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/.gitignore +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/.node-version +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/.python-version +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/CLAUDE.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/CONTRIBUTING.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/LICENSE +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/README.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/blume.config.ts +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/concepts/meta.ts +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/concepts/repositories.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/concepts/versions-and-refs.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/concepts/working-catalogs.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/getting-started.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/guides/building-a-host.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/guides/meta.ts +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/guides/mirrors.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/guides/public-and-private.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/guides/reading-without-duckdb.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/guides/serving-over-http.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/guides/stores.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/index.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/internals/from-write-to-read.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/internals/inside-a-repository.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/internals/meta.ts +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/concepts/meta.ts +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/concepts/repositories.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/concepts/versions-and-refs.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/concepts/working-catalogs.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/getting-started.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/guides/building-a-host.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/guides/meta.ts +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/guides/mirrors.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/guides/public-and-private.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/guides/reading-without-duckdb.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/guides/serving-over-http.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/guides/stores.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/index.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/internals/from-write-to-read.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/internals/inside-a-repository.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/internals/meta.ts +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/reference/meta.ts +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/reference/python.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/spec/data-files.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/spec/http.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/spec/meta.ts +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/spec/reading.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/spec/refs.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/reference/meta.ts +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/reference/python.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/spec/data-files.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/spec/http.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/spec/meta.ts +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/spec/reading.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/spec/refs.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/package-lock.json +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/package.json +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/public/CNAME +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/src/fdl/__init__.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/src/fdl/catalog.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/src/fdl/dataset.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/src/fdl/py.typed +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/src/fdl/refs.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/src/fdl/store.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/src/fdl/stores.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/src/fdl/version.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/tests/http_repo.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/tests/test_cli.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/tests/test_dataset.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/tests/test_http.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/tests/test_objects.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/tests/test_repo.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/tests/test_stores.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.0}/uv.lock +0 -0
|
@@ -1,6 +1,20 @@
|
|
|
1
1
|
# CHANGELOG
|
|
2
2
|
|
|
3
3
|
|
|
4
|
+
## v0.20.0 (2026-11-06)
|
|
5
|
+
|
|
6
|
+
### Documentation
|
|
7
|
+
|
|
8
|
+
- Drop the note that the draft spec changes as tools are built
|
|
9
|
+
([#36](https://github.com/queria-io/fdl/pull/36),
|
|
10
|
+
[`de531b8`](https://github.com/queria-io/fdl/commit/de531b806a3b1d65fe4e5347ce203d22573981d6))
|
|
11
|
+
|
|
12
|
+
### Features
|
|
13
|
+
|
|
14
|
+
- Write a profile of each table beside a version ([#39](https://github.com/queria-io/fdl/pull/39),
|
|
15
|
+
[`bdabd3f`](https://github.com/queria-io/fdl/commit/bdabd3fcd3f0b777609de78691b8ad64a1c40c26))
|
|
16
|
+
|
|
17
|
+
|
|
4
18
|
## v0.19.0 (2026-11-06)
|
|
5
19
|
|
|
6
20
|
### Features
|
|
@@ -53,6 +53,8 @@ fdl commit <working.sqlite> -m "What changed" --author alice [--dataset dataset.
|
|
|
53
53
|
|
|
54
54
|
コマンドは、説明の隣の `README.md` と、そこにある `docs/` より下のファイル(隠しファイルを除く)をバージョンの tree に入れます。そのフォルダからのパスで `tree.json` を書き、リポジトリに無いものを書いたままの中身で `assets/<sha256>` にアップロードします([tree.json](/ja/spec/versions#treejson))。
|
|
55
55
|
|
|
56
|
+
あわせてテーブルごとの profile を書きます([profiles/](/ja/spec/versions#profiles))。読むのは、データファイルか列が親から変わったテーブルだけです。読めなかったテーブルには profile を書かず、警告を出してコミットを続けます。
|
|
57
|
+
|
|
56
58
|
`asset: true` の印を付けた列は、ファイルの sha256 を持ちます([アセット](/ja/spec/versions#アセット))。コマンドはその列の値を読み、リポジトリに無いファイルを `--assets` のフォルダから `assets/<sha256>` にアップロードします。ファイルは中身で照合するので、フォルダの中のファイル名は問いません。
|
|
57
59
|
|
|
58
60
|
親と同じ中身になるとき([変更がないとき](/ja/spec/writing#変更がないとき))は `Nothing to commit` と表示し、ブランチを動かさずに終了コード 0 で終わります。同じ行を新しいファイルに書き直しただけの作り直しも、変更なしとみなします。`--allow-empty` を付けると、それでもバージョンを作ります。行を読めず判定できないときは、警告を出してバージョンを作ります。
|
|
@@ -5,7 +5,7 @@ sidebar:
|
|
|
5
5
|
description: FDL 0.1(草案)の規範となる説明。何を決め、何をホストに任せ、どんな用語を使うか。
|
|
6
6
|
---
|
|
7
7
|
|
|
8
|
-
FDL 0.1
|
|
8
|
+
FDL 0.1 の仕様の草案です。ツールはこの仕様に合わせて作り、ツールと仕様が食い違ったときは、先に仕様を直してからツールを合わせます。この日本語訳は参考で、英語版を正とします。
|
|
9
9
|
|
|
10
10
|
FDL は、[DuckLake](https://ducklake.select/) のカタログのバージョンを残すための、置き場のファイル配置です。Git がコミットを保存する形にならっています。どのファイルを置き場のどこに置き、読み手と書き手がそれをどう扱うかを決めます。サーバーは要りません。置き場に求めるのは、読む・書く・条件付きで書くの 3 つです。
|
|
11
11
|
|
|
@@ -23,9 +23,10 @@ description: リポジトリのファイル、そのパス、fdl.json の中身
|
|
|
23
23
|
dataset.json バージョンの中身の説明
|
|
24
24
|
files.json カタログから作った、表ごとのデータファイルの索引(任意)
|
|
25
25
|
tree.json README など、バージョンのほかのファイルのパスの一覧(任意)
|
|
26
|
+
profiles/<schema>/<table>.json データから測ったテーブルごとの要約(任意)
|
|
26
27
|
catalogs/<format>.duckdb 別の DuckLake 形式のカタログ(任意)
|
|
27
28
|
expired.json 保持期間を過ぎた印(過ぎたときだけ)
|
|
28
|
-
<others>
|
|
29
|
+
<others> ホストが足すファイル
|
|
29
30
|
<data_dir><schema>/<table>/ducklake-<uuid>.parquet データファイル(共有、変えない)
|
|
30
31
|
assets/<sha256> テーブルの行やバージョンの tree.json が指すファイル(共有、変更しない)
|
|
31
32
|
```
|
|
@@ -154,6 +154,51 @@ DuckLake を使わずに Parquet ファイルを読む読み手は、`deletes`
|
|
|
154
154
|
- 書き手は、ファイルを編集する場所(Git のリポジトリの中の YAML の隣など)に置き、書き手のツールが `tree.json` を書く。書き手が手で書くものではない
|
|
155
155
|
- `tree.json` はバージョンの一部で、そのバージョンの中では変わらない。保持期間から外すときはカタログと一緒に消す
|
|
156
156
|
|
|
157
|
+
## profiles/
|
|
158
|
+
|
|
159
|
+
profile は、`ducklake.snapshot_id` の時点の 1 つのテーブルの要約です。行数、列ごとの中身、先頭の行、図形の列を持つテーブルでは図形の範囲と見本を持ちます。読み手はこの小さなファイル 1 つで、カタログを開かずデータも読まずにテーブルを表示できます。ホストは検索の索引をこれから作れます。profile が持つのはデータから測った事実で、列が何を意味するかは書き手が `dataset.json` に書きます。`files.json` と同じくカタログとデータから作るもので、食い違うときはカタログとデータが正です。任意で、読み手は profile が無くても読めなければなりません。
|
|
160
|
+
|
|
161
|
+
テーブルごとの profile は `versions/<ID>/profiles/<schema>/<table>.json` に置きます。スキーマ名かテーブル名が空・`.`・`..` のとき、または `/` を含むときは置きません。ビューには置きません。
|
|
162
|
+
|
|
163
|
+
```json profiles/main/towns.json
|
|
164
|
+
{
|
|
165
|
+
"rows": 1741,
|
|
166
|
+
"columns": [
|
|
167
|
+
{ "name": "town_code", "type": "varchar", "null_count": 0, "distinct_count": 1741,
|
|
168
|
+
"values": [{ "value": "082201", "count": 1 }] },
|
|
169
|
+
{ "name": "prefecture", "type": "varchar", "null_count": 0, "distinct_count": 2,
|
|
170
|
+
"values": [{ "value": "茨城県", "count": 1200 }, { "value": "栃木県", "count": 541 }] },
|
|
171
|
+
{ "name": "population", "type": "int64", "null_count": 3, "distinct_count": 1650,
|
|
172
|
+
"min": 0, "max": 84211, "mean": 3120.5, "std": 4410.2, "q1": 420, "median": 1630, "q3": 3990,
|
|
173
|
+
"values": [{ "value": "0", "count": 12 }],
|
|
174
|
+
"histogram": [{ "start": 0, "end": 4210.55, "count": 1302 }] },
|
|
175
|
+
{ "name": "surveyed_on", "type": "date", "null_count": 0, "distinct_count": 1,
|
|
176
|
+
"min": "2025-10-01", "max": "2025-10-01", "span_days": 0,
|
|
177
|
+
"values": [{ "value": "2025-10-01", "count": 1741 }] },
|
|
178
|
+
{ "name": "coastal", "type": "boolean", "null_count": 0, "true_count": 210, "false_count": 1531 },
|
|
179
|
+
{ "name": "geom", "type": "geometry" }
|
|
180
|
+
],
|
|
181
|
+
"head": [{ "town_code": "082201", "prefecture": "茨城県", "population": 2210, "surveyed_on": "2025-10-01", "coastal": false }],
|
|
182
|
+
"geometry": {
|
|
183
|
+
"column": "geom",
|
|
184
|
+
"count": 1741,
|
|
185
|
+
"bbox": [139.69, 35.74, 140.85, 37.15],
|
|
186
|
+
"types": ["MULTIPOLYGON", "POLYGON"],
|
|
187
|
+
"sample": { "type": "FeatureCollection", "features": [] }
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
- `rows` はテーブルの行数。`columns` にはカタログの順にすべての列を並べ、`name` と `type`(DuckLake の型名)は `dataset.json` と同じにする
|
|
193
|
+
- 図形の列を除くすべての列が `null_count` を持つ。数値・テキスト・日付と時刻の列は `distinct_count` と `values` を持つ。大きなテーブルの `distinct_count` は推定値のことがある
|
|
194
|
+
- `values` は、値(テキスト)とその値を持つ行の数を、多い順、同数なら値の順に並べる。列が持つ異なる値が 1,000 以下なら全部を並べ、そのとき `distinct_count` は正確な数になる。読み手は、`values` の長さが `distinct_count` と等しいことで全部の一覧だと判定する。それより多い列は上位 10 件だけを並べる。ホストは全部の一覧を、地域コードや名前などの値で検索できる
|
|
195
|
+
- 数値の列は `min`・`max`・`mean`・`std`、四分位数の `q1`・`median`・`q3`、`min` から `max` までを等幅に区切った `histogram` を持つ。日付と時刻の列は `min` と `max` をテキストで持ち、日付とタイムスタンプは `span_days` も持つ。真偽値の列は `true_count` と `false_count` を持つ
|
|
196
|
+
- `head` は、テーブルが返す先頭の行を JSON のオブジェクトで持つ。図形の列と、値を JSON で表せない列は入れない
|
|
197
|
+
- `geometry` は、書き手が読めたときに、最初の図形の列を説明する。図形を持つ行の数、全体の `bbox`(列の座標で `[xmin, ymin, xmax, ymax]`)、含まれる図形の種類、地図を描くための、無作為に選んで簡略化した図形の GeoJSON `FeatureCollection` を持つ
|
|
198
|
+
- 書き手は計算できないキーを省いてよい。読み手は知らないキーを無視する
|
|
199
|
+
- 書き手は、データファイル(`files.json`)と列が親と同じテーブルには親の profile を使い回す。コミットで読むのは変わったテーブルだけになる
|
|
200
|
+
- profile はバージョンの一部で、そのバージョンの中では変わらない。保持期間から外すときはカタログと一緒に消す
|
|
201
|
+
|
|
157
202
|
## 固めたカタログ
|
|
158
203
|
|
|
159
204
|
`ducklake.duckdb` は、DuckDB のファイル形式の DuckLake カタログです。
|
|
@@ -17,7 +17,7 @@ description: チェックアウト、コミット、push。作業カタログ、
|
|
|
17
17
|
| --- | --- |
|
|
18
18
|
| チェックアウト | ブランチのバージョンのカタログから作業カタログを作り、そのバージョンを覚える。最初のバージョンの前は空のものを作る |
|
|
19
19
|
| コミット | 作業カタログの最後のスナップショットを新しいバージョンに固める。親は覚えておいたバージョン。中身を `dataset.json` で説明する |
|
|
20
|
-
| push | データファイルを確かめ、カタログ、`dataset.json`・`files.json`・`tree.json
|
|
20
|
+
| push | データファイルを確かめ、カタログ、`dataset.json`・`files.json`・`tree.json`・profile、`version.json` の順にアップロードし、条件付きで ref を動かす |
|
|
21
21
|
|
|
22
22
|
チェックアウトでは、DuckLake の管理用の表を、固めたカタログから SQLite へそのまま写します。固めるときはその逆で、最後のスナップショットまでの行を残します。どちらもデータファイルは書き換えません。
|
|
23
23
|
|
|
@@ -40,7 +40,7 @@ description: チェックアウト、コミット、push。作業カタログ、
|
|
|
40
40
|
|
|
41
41
|
1. データファイル・アセット
|
|
42
42
|
2. `versions/<ID>/ducklake.duckdb`(と `catalogs/`)
|
|
43
|
-
3. `versions/<ID>/dataset.json`・`versions/<ID>/files.json`・`versions/<ID>/tree.json
|
|
43
|
+
3. `versions/<ID>/dataset.json`・`versions/<ID>/files.json`・`versions/<ID>/tree.json`・`versions/<ID>/profiles/`
|
|
44
44
|
4. `versions/<ID>/version.json`
|
|
45
45
|
5. ref(条件付き)
|
|
46
46
|
|
|
@@ -53,6 +53,8 @@ Every version is described by a `dataset.json` (see [Versions](/spec/versions#da
|
|
|
53
53
|
|
|
54
54
|
The command puts `README.md` beside the description and every file below `docs/` there (hidden ones left out) in the version's tree: it writes `tree.json` with their paths from that folder and uploads those the repository lacks to `assets/<sha256>`, as written (see [tree.json](/spec/versions#treejson)).
|
|
55
55
|
|
|
56
|
+
It also writes a profile of each table (see [profiles/](/spec/versions#profiles)), reading only the tables whose data files or columns changed since the parent. A table it cannot read gets none, and the commit goes on with a warning.
|
|
57
|
+
|
|
56
58
|
Columns marked `asset: true` hold the sha256 of files (see [Assets](/spec/versions#assets)). The command reads the values those columns hold and uploads the files the repository lacks from the `--assets` folder to `assets/<sha256>`, by their content: file names in the folder do not matter.
|
|
57
59
|
|
|
58
60
|
When the version would hold what its parent holds (see [When nothing changed](/spec/writing#when-nothing-changed)), it prints `Nothing to commit`, leaves the branch where it is and exits 0. A rebuild that writes the same rows to new files counts as unchanged. `--allow-empty` makes the version anyway. If the rows cannot be read to tell, it warns and makes the version.
|
|
@@ -5,7 +5,7 @@ sidebar:
|
|
|
5
5
|
description: The normative description of FDL 0.1 (draft) - what it decides, what it leaves to hosts, and the terms it uses.
|
|
6
6
|
---
|
|
7
7
|
|
|
8
|
-
This is the specification of FDL 0.1, a draft.
|
|
8
|
+
This is the specification of FDL 0.1, a draft. Tools are written against this specification, and where a tool and this text disagree, the text is changed first and the tool follows.
|
|
9
9
|
|
|
10
10
|
FDL is a storage layout for keeping versions of a [DuckLake](https://ducklake.select/) catalog, modelled on how Git stores commits. It says which files go where in a store and how readers and writers handle them. No server is needed: the store has to read, write and write on condition.
|
|
11
11
|
|
|
@@ -23,9 +23,10 @@ Paths are relative to the root, lower case, and separated by `/`. The root is wh
|
|
|
23
23
|
dataset.json what the version holds, described
|
|
24
24
|
files.json each table's data files, indexed from the catalog (optional)
|
|
25
25
|
tree.json the other files of the version, such as a README, by path (optional)
|
|
26
|
+
profiles/<schema>/<table>.json a summary of each table, measured from the data (optional)
|
|
26
27
|
catalogs/<format>.duckdb the catalog in another DuckLake format (optional)
|
|
27
28
|
expired.json the past-retention mark (only once expired)
|
|
28
|
-
<others> files a host adds
|
|
29
|
+
<others> files a host adds
|
|
29
30
|
<data_dir><schema>/<table>/ducklake-<uuid>.parquet data files (shared, immutable)
|
|
30
31
|
assets/<sha256> files the rows of a table or a version's tree.json point at (shared, immutable)
|
|
31
32
|
```
|
|
@@ -154,6 +154,51 @@ When a version is taken out of retention, `files.json` is deleted with its catal
|
|
|
154
154
|
- A writer keeps the files where it edits them, such as beside its YAML in a Git repository, and its tool writes `tree.json`. The writer does not write it by hand
|
|
155
155
|
- `tree.json` is part of the version and does not change with it. It is deleted with the catalogs when the version is taken out of retention
|
|
156
156
|
|
|
157
|
+
## profiles/
|
|
158
|
+
|
|
159
|
+
A profile summarizes one table at `ducklake.snapshot_id`: its row count, what each column holds, its first rows and, for a table with a geometry column, the extent and a sample of its shapes. A reader can show a table, or a host can index it for search, from this one small file, without opening the catalog or reading the data. It holds what was measured from the data; what a column means is the writer's, in `dataset.json`. Like `files.json`, it is made from the catalog and the data, which win where the two differ, and it is optional: a reader must work without it.
|
|
160
|
+
|
|
161
|
+
Each table's profile is at `versions/<ID>/profiles/<schema>/<table>.json`. A table whose schema or name is empty, `.`, `..` or holds `/` has none. Views have none.
|
|
162
|
+
|
|
163
|
+
```json profiles/main/towns.json
|
|
164
|
+
{
|
|
165
|
+
"rows": 1741,
|
|
166
|
+
"columns": [
|
|
167
|
+
{ "name": "town_code", "type": "varchar", "null_count": 0, "distinct_count": 1741,
|
|
168
|
+
"values": [{ "value": "082201", "count": 1 }] },
|
|
169
|
+
{ "name": "prefecture", "type": "varchar", "null_count": 0, "distinct_count": 2,
|
|
170
|
+
"values": [{ "value": "Ibaraki", "count": 1200 }, { "value": "Tochigi", "count": 541 }] },
|
|
171
|
+
{ "name": "population", "type": "int64", "null_count": 3, "distinct_count": 1650,
|
|
172
|
+
"min": 0, "max": 84211, "mean": 3120.5, "std": 4410.2, "q1": 420, "median": 1630, "q3": 3990,
|
|
173
|
+
"values": [{ "value": "0", "count": 12 }],
|
|
174
|
+
"histogram": [{ "start": 0, "end": 4210.55, "count": 1302 }] },
|
|
175
|
+
{ "name": "surveyed_on", "type": "date", "null_count": 0, "distinct_count": 1,
|
|
176
|
+
"min": "2025-10-01", "max": "2025-10-01", "span_days": 0,
|
|
177
|
+
"values": [{ "value": "2025-10-01", "count": 1741 }] },
|
|
178
|
+
{ "name": "coastal", "type": "boolean", "null_count": 0, "true_count": 210, "false_count": 1531 },
|
|
179
|
+
{ "name": "geom", "type": "geometry" }
|
|
180
|
+
],
|
|
181
|
+
"head": [{ "town_code": "082201", "prefecture": "Ibaraki", "population": 2210, "surveyed_on": "2025-10-01", "coastal": false }],
|
|
182
|
+
"geometry": {
|
|
183
|
+
"column": "geom",
|
|
184
|
+
"count": 1741,
|
|
185
|
+
"bbox": [139.69, 35.74, 140.85, 37.15],
|
|
186
|
+
"types": ["MULTIPOLYGON", "POLYGON"],
|
|
187
|
+
"sample": { "type": "FeatureCollection", "features": [] }
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
- `rows` is the table's row count. `columns` lists every column in the catalog's order, with `name` and `type` (DuckLake's type name) as in `dataset.json`
|
|
193
|
+
- Every column but a geometry one has `null_count`. Numbers, text, dates and times have `distinct_count`, which may be estimated on a large table, and `values`
|
|
194
|
+
- `values` lists each value, as text, with the number of rows that hold it, most common first and then by value. When the column holds at most 1,000 distinct values, it lists them all and `distinct_count` is exact: a reader tells a whole list by its length matching `distinct_count`. Otherwise it lists the 10 most common. A host can search a whole list by value, such as a town's code or name
|
|
195
|
+
- Numbers have `min`, `max`, `mean`, `std`, the quartiles `q1`, `median` and `q3`, and a `histogram` of equal-width buckets between `min` and `max`. Dates and times have `min` and `max` as text, and dates and timestamps `span_days`. Booleans have `true_count` and `false_count`
|
|
196
|
+
- `head` holds the first rows the table returns, as JSON objects, leaving out geometry and columns whose values have no JSON form
|
|
197
|
+
- `geometry` describes the first geometry column, when the writer could read it: how many rows have a shape, the `bbox` of all of them (`[xmin, ymin, xmax, ymax]` in the column's coordinates), the geometry types present, and a GeoJSON `FeatureCollection` of a random sample of simplified shapes for drawing a map
|
|
198
|
+
- A writer may leave out any key it cannot compute. A reader ignores keys it does not know
|
|
199
|
+
- A writer reuses its parent's profile for a table whose data files (in `files.json`) and columns are the parent's, so a commit reads only the tables that changed
|
|
200
|
+
- Profiles are part of the version and do not change with it. They are deleted with the catalogs when the version is taken out of retention
|
|
201
|
+
|
|
157
202
|
## The frozen catalog
|
|
158
203
|
|
|
159
204
|
`ducklake.duckdb` is a DuckLake catalog in DuckDB's file format.
|
|
@@ -17,7 +17,7 @@ A writer writes to a working catalog: a DuckLake catalog in SQLite, outside the
|
|
|
17
17
|
| --- | --- |
|
|
18
18
|
| checkout | Make a working catalog from the catalog of the branch's version, and remember that version. Before the first version, make an empty one |
|
|
19
19
|
| commit | Freeze the working catalog's last snapshot into a new version, with the remembered version as its parent, and describe it in `dataset.json` |
|
|
20
|
-
| push | Check the data files, upload the catalog, `dataset.json`, `files.json
|
|
20
|
+
| push | Check the data files, upload the catalog, `dataset.json`, `files.json`, `tree.json` and the profiles, then `version.json`, then move the ref on condition |
|
|
21
21
|
|
|
22
22
|
Checking out copies DuckLake's bookkeeping tables from the frozen catalog into SQLite as they are. Freezing does the reverse, keeping the rows up to the last snapshot. Neither rewrites a data file.
|
|
23
23
|
|
|
@@ -40,7 +40,7 @@ A table rebuilt from the same input is written to new data files, so comparing f
|
|
|
40
40
|
|
|
41
41
|
1. Data files and assets
|
|
42
42
|
2. `versions/<ID>/ducklake.duckdb` (and any `catalogs/`)
|
|
43
|
-
3. `versions/<ID>/dataset.json`, `versions/<ID>/files.json
|
|
43
|
+
3. `versions/<ID>/dataset.json`, `versions/<ID>/files.json`, `versions/<ID>/tree.json` and `versions/<ID>/profiles/`
|
|
44
44
|
4. `versions/<ID>/version.json`
|
|
45
45
|
5. The ref, on condition
|
|
46
46
|
|
|
@@ -148,12 +148,16 @@ def commit(
|
|
|
148
148
|
typer.echo(f"warning: could not tell whether anything changed ({exc}); committing", err=True)
|
|
149
149
|
version = make(True)
|
|
150
150
|
local = Path(state["data"]) if state.get("data") else None
|
|
151
|
+
reading = Path(state["data"]).resolve().as_posix() + "/" if local else repo.data_url(store)
|
|
152
|
+
setup = _read_setup(store)
|
|
151
153
|
described = json.loads((out / version.id / layout.DATASET).read_text(encoding="utf-8"))
|
|
152
154
|
held = objects.asset_values(
|
|
153
|
-
working,
|
|
154
|
-
objects.asset_columns(described),
|
|
155
|
-
data_path=Path(state["data"]).resolve().as_posix() + "/" if local else repo.data_url(store),
|
|
155
|
+
working, objects.asset_columns(described), data_path=reading, setup=setup
|
|
156
156
|
)
|
|
157
|
+
try:
|
|
158
|
+
repo.write_profiles(store, out / version.id, parent, working, data_path=reading, setup=setup)
|
|
159
|
+
except Exception as exc: # noqa: BLE001 - a profile is never worth a failed commit
|
|
160
|
+
typer.echo(f"warning: tables were not profiled ({exc})", err=True)
|
|
157
161
|
local_objects = objects.ByContent(
|
|
158
162
|
{item["sha256"]: root / item["path"] for item in listed},
|
|
159
163
|
assets_dir,
|
|
@@ -180,6 +184,16 @@ def commit(
|
|
|
180
184
|
typer.echo(f"{state['ref']} -> {version.id} (version {version.seq})")
|
|
181
185
|
|
|
182
186
|
|
|
187
|
+
def _read_setup(store) -> list[str]:
|
|
188
|
+
"""What DuckDB needs to read the repository's data files: the token, over HTTP."""
|
|
189
|
+
token = getattr(store, "token", None)
|
|
190
|
+
if not token:
|
|
191
|
+
return []
|
|
192
|
+
scope = store.url.replace("'", "''")
|
|
193
|
+
secret = token.replace("'", "''")
|
|
194
|
+
return [f"CREATE SECRET fdl_remote (TYPE http, BEARER_TOKEN '{secret}', SCOPE '{scope}')"]
|
|
195
|
+
|
|
196
|
+
|
|
183
197
|
@app.command()
|
|
184
198
|
def assets(folder: Path = typer.Argument(..., help="The folder of files the rows will point at")) -> None:
|
|
185
199
|
"""List the files below a folder as CSV: path, sha256, bytes, media_type.
|
|
@@ -29,6 +29,8 @@ EXPIRED = "expired.json"
|
|
|
29
29
|
CATALOGS_DIR = "catalogs/"
|
|
30
30
|
#: The other files of a version (its README, say), by path to the asset that holds each (optional).
|
|
31
31
|
TREE = "tree.json"
|
|
32
|
+
#: A summary of each table, made from the data, under the version (optional).
|
|
33
|
+
PROFILES_DIR = "profiles/"
|
|
32
34
|
#: Files the rows of a table point at, by sha256 (shared).
|
|
33
35
|
ASSETS_DIR = "assets/"
|
|
34
36
|
|
|
@@ -174,9 +174,17 @@ def asset_columns(described: dict[str, Any]) -> list[tuple[str, str, str]]:
|
|
|
174
174
|
|
|
175
175
|
|
|
176
176
|
def asset_values(
|
|
177
|
-
working: Path,
|
|
177
|
+
working: Path,
|
|
178
|
+
columns: Iterable[tuple[str, str, str]],
|
|
179
|
+
*,
|
|
180
|
+
data_path: str,
|
|
181
|
+
setup: Iterable[str] = (),
|
|
178
182
|
) -> set[str]:
|
|
179
|
-
"""The sha256 the marked columns hold in the working catalog's last snapshot.
|
|
183
|
+
"""The sha256 the marked columns hold in the working catalog's last snapshot.
|
|
184
|
+
|
|
185
|
+
``setup`` runs before the catalog is attached: a ``CREATE SECRET`` for data
|
|
186
|
+
files in private storage, say.
|
|
187
|
+
"""
|
|
180
188
|
import duckdb
|
|
181
189
|
|
|
182
190
|
columns = list(columns)
|
|
@@ -189,6 +197,8 @@ def asset_values(
|
|
|
189
197
|
found: set[str] = set()
|
|
190
198
|
con = duckdb.connect()
|
|
191
199
|
try:
|
|
200
|
+
for statement in setup:
|
|
201
|
+
con.execute(statement)
|
|
192
202
|
for statement in catalog.attach_sql(Path(working), alias="lake", data_path=data_path, read_only=True):
|
|
193
203
|
if statement:
|
|
194
204
|
con.execute(statement)
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
"""Profiles: a summary of each table, written beside a version for readers that show it.
|
|
2
|
+
|
|
3
|
+
A profile holds a table's row count, statistics of each column, its first rows
|
|
4
|
+
and, for a table with a geometry column, a sample of its shapes. A reader can
|
|
5
|
+
show a table from this one small file without opening the catalog or reading
|
|
6
|
+
the data. It is made from the catalog and the data, which win where the two
|
|
7
|
+
differ, and a reader works without it (see the specification, "profiles").
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import math
|
|
14
|
+
import re
|
|
15
|
+
from collections.abc import Iterable
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
from . import catalog
|
|
20
|
+
|
|
21
|
+
#: Rows in `head`. A glance; more is what a query is for.
|
|
22
|
+
HEAD_ROWS = 10
|
|
23
|
+
#: Shapes in `geometry`, sampled from the whole table rather than its first rows.
|
|
24
|
+
GEOMETRY_SAMPLE = 500
|
|
25
|
+
#: Vertex tolerance of the sampled shapes, in the column's units (about 11 m in degrees).
|
|
26
|
+
SIMPLIFY_TOLERANCE = 0.0001
|
|
27
|
+
HISTOGRAM_BUCKETS = 20
|
|
28
|
+
#: A column with at most this many distinct values lists them all in `values`.
|
|
29
|
+
ALL_VALUES = 1000
|
|
30
|
+
#: Otherwise `values` lists the most common.
|
|
31
|
+
TOP_VALUES = 10
|
|
32
|
+
#: Above this many rows, distinct values are estimated rather than counted.
|
|
33
|
+
EXACT_DISTINCT_ROWS = 1_000_000
|
|
34
|
+
|
|
35
|
+
_NUMERIC = {
|
|
36
|
+
"TINYINT", "SMALLINT", "INTEGER", "BIGINT", "HUGEINT", "UTINYINT", "USMALLINT",
|
|
37
|
+
"UINTEGER", "UBIGINT", "UHUGEINT", "FLOAT", "DOUBLE", "DECIMAL", "REAL", "NUMERIC",
|
|
38
|
+
"INT8", "INT16", "INT32", "INT64", "INT128", "UINT8", "UINT16", "UINT32", "UINT64",
|
|
39
|
+
"UINT128", "FLOAT32", "FLOAT64",
|
|
40
|
+
} # fmt: skip
|
|
41
|
+
_TEMPORAL = {"DATE", "TIME", "TIMESTAMP", "TIMESTAMPTZ", "TIMESTAMP WITH TIME ZONE", "TIMETZ", "INTERVAL"}
|
|
42
|
+
_DATED = {"DATE", "TIMESTAMP", "TIMESTAMPTZ", "TIMESTAMP WITH TIME ZONE"}
|
|
43
|
+
_BOOLEAN = {"BOOLEAN"}
|
|
44
|
+
_TEXT = {"VARCHAR", "UUID"}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _base(data_type: str) -> str:
|
|
48
|
+
return re.sub(r"\(.*\)", "", data_type.upper()).strip()
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def kind(data_type: str) -> str:
|
|
52
|
+
"""The family of a DuckLake type, which decides the statistics a column gets."""
|
|
53
|
+
base = _base(data_type)
|
|
54
|
+
if base in _NUMERIC:
|
|
55
|
+
return "numeric"
|
|
56
|
+
if base in _TEMPORAL:
|
|
57
|
+
return "temporal"
|
|
58
|
+
if base in _BOOLEAN:
|
|
59
|
+
return "boolean"
|
|
60
|
+
if base in _TEXT:
|
|
61
|
+
return "text"
|
|
62
|
+
if base == "GEOMETRY":
|
|
63
|
+
return "geometry"
|
|
64
|
+
return "other"
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def path(schema: str, table: str) -> str | None:
|
|
68
|
+
"""The profile's path under the version, or None for a name a path cannot hold."""
|
|
69
|
+
if any(not part or part in (".", "..") or "/" in part for part in (schema, table)):
|
|
70
|
+
return None
|
|
71
|
+
return f"profiles/{schema}/{table}.json"
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def tables(described: dict[str, Any]) -> list[dict[str, Any]]:
|
|
75
|
+
"""The tables of a `dataset.json` that get a profile: every table, no view."""
|
|
76
|
+
return [t for t in described.get("tables", []) if t.get("kind") != "view" and path(t["schema"], t["name"])]
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def columns_of(table: dict[str, Any]) -> list[dict[str, str]]:
|
|
80
|
+
return [{"name": f["name"], "type": f.get("type", "")} for f in table.get("fields", [])]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def build(working: Path, wanted: Iterable[dict[str, Any]], *, data_path: str, setup: Iterable[str] = ()) -> dict[str, bytes]:
|
|
84
|
+
"""Profiles of ``wanted`` (tables of `dataset.json`) from the working catalog's last snapshot.
|
|
85
|
+
|
|
86
|
+
Returns path to content. A table that cannot be read gets no profile, and
|
|
87
|
+
a geometry sample that cannot be made (no spatial extension) is left out.
|
|
88
|
+
"""
|
|
89
|
+
import duckdb
|
|
90
|
+
|
|
91
|
+
wanted = list(wanted)
|
|
92
|
+
if not wanted:
|
|
93
|
+
return {}
|
|
94
|
+
con = duckdb.connect()
|
|
95
|
+
try:
|
|
96
|
+
for statement in setup:
|
|
97
|
+
con.execute(statement)
|
|
98
|
+
for statement in catalog.attach_sql(Path(working), alias="lake", data_path=data_path, read_only=True):
|
|
99
|
+
if statement:
|
|
100
|
+
con.execute(statement)
|
|
101
|
+
spatial = _load_spatial(con)
|
|
102
|
+
made: dict[str, bytes] = {}
|
|
103
|
+
for table in wanted:
|
|
104
|
+
try:
|
|
105
|
+
profile = _profile(con, table, spatial)
|
|
106
|
+
except duckdb.Error:
|
|
107
|
+
continue
|
|
108
|
+
made[path(table["schema"], table["name"])] = _dump(profile)
|
|
109
|
+
return made
|
|
110
|
+
finally:
|
|
111
|
+
con.close()
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def reusable(previous: bytes, table: dict[str, Any]) -> bool:
|
|
115
|
+
"""Whether a parent's profile describes ``table`` as it is: the same columns and types."""
|
|
116
|
+
try:
|
|
117
|
+
held = json.loads(previous).get("columns", [])
|
|
118
|
+
except ValueError:
|
|
119
|
+
return False
|
|
120
|
+
return [{"name": c.get("name"), "type": c.get("type")} for c in held] == columns_of(table)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _load_spatial(con) -> bool:
|
|
125
|
+
try:
|
|
126
|
+
con.execute("LOAD spatial")
|
|
127
|
+
return True
|
|
128
|
+
except Exception: # noqa: BLE001 - not installed: try once, then do without
|
|
129
|
+
try:
|
|
130
|
+
con.execute("INSTALL spatial")
|
|
131
|
+
con.execute("LOAD spatial")
|
|
132
|
+
return True
|
|
133
|
+
except Exception: # noqa: BLE001
|
|
134
|
+
return False
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _q(name: str) -> str:
|
|
138
|
+
return '"' + name.replace('"', '""') + '"'
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _profile(con, table: dict[str, Any], spatial: bool) -> dict[str, Any]:
|
|
142
|
+
relation = f"lake.{_q(table['schema'])}.{_q(table['name'])}"
|
|
143
|
+
fields = columns_of(table)
|
|
144
|
+
rows = int(con.execute(f"SELECT count(*) FROM {relation}").fetchone()[0])
|
|
145
|
+
|
|
146
|
+
measured = [f for f in fields if kind(f["type"]) != "geometry"]
|
|
147
|
+
exprs = [e for f in measured for e in _aggregates(f, rows)]
|
|
148
|
+
values = con.execute(f"SELECT {', '.join(exprs)} FROM {relation}").fetchone() if exprs else ()
|
|
149
|
+
stats: list[dict[str, Any]] = []
|
|
150
|
+
cursor = 0
|
|
151
|
+
for field in fields:
|
|
152
|
+
if kind(field["type"]) == "geometry":
|
|
153
|
+
stats.append(dict(field))
|
|
154
|
+
continue
|
|
155
|
+
width = len(_aggregates(field, rows))
|
|
156
|
+
stats.append(_column(con, relation, field, rows, values[cursor : cursor + width]))
|
|
157
|
+
cursor += width
|
|
158
|
+
|
|
159
|
+
shown = [f["name"] for f in fields if kind(f["type"]) not in ("geometry", "other")]
|
|
160
|
+
head: list[dict[str, Any]] = []
|
|
161
|
+
if shown:
|
|
162
|
+
encoded = con.execute(
|
|
163
|
+
f"SELECT to_json(list(t)) FROM (SELECT {', '.join(_q(c) for c in shown)} FROM {relation} LIMIT {HEAD_ROWS}) t"
|
|
164
|
+
).fetchone()
|
|
165
|
+
head = json.loads(encoded[0]) if encoded and encoded[0] else []
|
|
166
|
+
|
|
167
|
+
profile: dict[str, Any] = {"rows": rows, "columns": stats, "head": head}
|
|
168
|
+
geometry = next((f["name"] for f in fields if kind(f["type"]) == "geometry"), None)
|
|
169
|
+
if geometry is not None and spatial:
|
|
170
|
+
try:
|
|
171
|
+
profile["geometry"] = _geometry(con, relation, geometry)
|
|
172
|
+
except Exception: # noqa: BLE001 - the rest stands without the shapes
|
|
173
|
+
pass
|
|
174
|
+
return profile
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _aggregates(field: dict[str, str], rows: int) -> list[str]:
|
|
178
|
+
c = _q(field["name"])
|
|
179
|
+
distinct = f"count(DISTINCT {c})" if rows <= EXACT_DISTINCT_ROWS else f"approx_count_distinct({c})"
|
|
180
|
+
family = kind(field["type"])
|
|
181
|
+
head = [f"count({c})"]
|
|
182
|
+
if family == "numeric":
|
|
183
|
+
v = f"{c}::DOUBLE"
|
|
184
|
+
return head + [
|
|
185
|
+
distinct,
|
|
186
|
+
f"min({v})",
|
|
187
|
+
f"max({v})",
|
|
188
|
+
f"avg({v})",
|
|
189
|
+
f"stddev_samp({v})",
|
|
190
|
+
f"approx_quantile({v}, [0.25, 0.5, 0.75])",
|
|
191
|
+
]
|
|
192
|
+
if family == "text":
|
|
193
|
+
return head + [distinct]
|
|
194
|
+
if family == "temporal":
|
|
195
|
+
span = (
|
|
196
|
+
f"datediff('day', min({c})::TIMESTAMP, max({c})::TIMESTAMP)" if _base(field["type"]) in _DATED else "NULL"
|
|
197
|
+
)
|
|
198
|
+
return head + [f"min({c})::VARCHAR", f"max({c})::VARCHAR", span, distinct]
|
|
199
|
+
if family == "boolean":
|
|
200
|
+
return head + [f"count_if({c})", f"count_if(NOT {c})"]
|
|
201
|
+
return head
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _column(con, relation: str, field: dict[str, str], rows: int, values) -> dict[str, Any]:
|
|
205
|
+
out: dict[str, Any] = {**field, "null_count": rows - int(values[0])}
|
|
206
|
+
family = kind(field["type"])
|
|
207
|
+
if family == "numeric":
|
|
208
|
+
lo, hi, mean, std = (_number(v) for v in values[2:6])
|
|
209
|
+
q1, median, q3 = [_number(q) for q in values[6]] if values[6] else (None, None, None)
|
|
210
|
+
out["distinct_count"] = int(values[1] or 0)
|
|
211
|
+
out.update(min=lo, max=hi, mean=mean, std=std, q1=q1, median=median, q3=q3)
|
|
212
|
+
_values(con, relation, field["name"], out)
|
|
213
|
+
out["histogram"] = _histogram(con, relation, field["name"], lo, hi)
|
|
214
|
+
elif family == "text":
|
|
215
|
+
out["distinct_count"] = int(values[1] or 0)
|
|
216
|
+
_values(con, relation, field["name"], out)
|
|
217
|
+
elif family == "temporal":
|
|
218
|
+
lo, hi, span, distinct = values[1:5]
|
|
219
|
+
out.update(min=lo, max=hi, span_days=None if span is None else int(span), distinct_count=int(distinct or 0))
|
|
220
|
+
_values(con, relation, field["name"], out)
|
|
221
|
+
elif family == "boolean":
|
|
222
|
+
out.update(true_count=int(values[1] or 0), false_count=int(values[2] or 0))
|
|
223
|
+
return out
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _histogram(con, relation: str, name: str, lo: float | None, hi: float | None) -> list[dict[str, float]]:
|
|
227
|
+
if lo is None or hi is None:
|
|
228
|
+
return []
|
|
229
|
+
width = (hi - lo) / HISTOGRAM_BUCKETS if hi > lo else 1.0
|
|
230
|
+
found = con.execute(
|
|
231
|
+
f"SELECT least(floor(({_q(name)}::DOUBLE - ?) / ?)::BIGINT, ?), count(*) "
|
|
232
|
+
f"FROM {relation} WHERE {_q(name)} IS NOT NULL GROUP BY 1 ORDER BY 1",
|
|
233
|
+
[lo, width, HISTOGRAM_BUCKETS - 1],
|
|
234
|
+
).fetchall()
|
|
235
|
+
return [{"start": lo + int(b) * width, "end": lo + (int(b) + 1) * width, "count": int(n)} for b, n in found]
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _values(con, relation: str, name: str, out: dict[str, Any]) -> None:
|
|
239
|
+
"""Every value with its count when there are at most `ALL_VALUES`, else the most common.
|
|
240
|
+
|
|
241
|
+
Counted here, so `distinct_count` is exact whenever the list is whole: a
|
|
242
|
+
reader tells a whole list by its length matching `distinct_count`.
|
|
243
|
+
"""
|
|
244
|
+
found = con.execute(
|
|
245
|
+
f"SELECT {_q(name)}::VARCHAR, count(*) FROM {relation} "
|
|
246
|
+
f"WHERE {_q(name)} IS NOT NULL GROUP BY 1 ORDER BY 2 DESC, 1 LIMIT {ALL_VALUES + 1}"
|
|
247
|
+
).fetchall()
|
|
248
|
+
if len(found) <= ALL_VALUES:
|
|
249
|
+
out["distinct_count"] = len(found)
|
|
250
|
+
else:
|
|
251
|
+
found = found[:TOP_VALUES]
|
|
252
|
+
out["distinct_count"] = max(out.get("distinct_count", 0), ALL_VALUES + 1)
|
|
253
|
+
out["values"] = [{"value": v, "count": int(n)} for v, n in found]
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _geometry(con, relation: str, name: str) -> dict[str, Any]:
|
|
257
|
+
c = _q(name)
|
|
258
|
+
total, xmin, ymin, xmax, ymax = con.execute(
|
|
259
|
+
f"SELECT count({c}), min(ST_XMin({c})), min(ST_YMin({c})), max(ST_XMax({c})), max(ST_YMax({c})) "
|
|
260
|
+
f"FROM {relation}"
|
|
261
|
+
).fetchone()
|
|
262
|
+
kinds = [
|
|
263
|
+
k
|
|
264
|
+
for (k,) in con.execute(
|
|
265
|
+
f"SELECT DISTINCT ST_GeometryType({c})::VARCHAR FROM {relation} WHERE {c} IS NOT NULL ORDER BY 1"
|
|
266
|
+
).fetchall()
|
|
267
|
+
]
|
|
268
|
+
sampled = con.execute(
|
|
269
|
+
f"SELECT ST_AsGeoJSON(ST_Simplify(g, {SIMPLIFY_TOLERANCE})) FROM "
|
|
270
|
+
f"(SELECT {c} AS g FROM {relation} WHERE {c} IS NOT NULL) USING SAMPLE reservoir({GEOMETRY_SAMPLE} ROWS)"
|
|
271
|
+
).fetchall()
|
|
272
|
+
features = []
|
|
273
|
+
for (raw,) in sampled:
|
|
274
|
+
if not raw:
|
|
275
|
+
continue
|
|
276
|
+
shape = json.loads(raw)
|
|
277
|
+
parts = shape.get("geometries", []) if shape.get("type") == "GeometryCollection" else [shape]
|
|
278
|
+
features.extend({"type": "Feature", "geometry": part, "properties": {}} for part in parts)
|
|
279
|
+
return {
|
|
280
|
+
"column": name,
|
|
281
|
+
"count": int(total),
|
|
282
|
+
"bbox": None if xmin is None else [xmin, ymin, xmax, ymax],
|
|
283
|
+
"types": kinds,
|
|
284
|
+
"sample": {"type": "FeatureCollection", "features": features},
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _number(value: Any) -> float | None:
|
|
289
|
+
if value is None:
|
|
290
|
+
return None
|
|
291
|
+
number = float(value)
|
|
292
|
+
return None if math.isnan(number) or math.isinf(number) else number
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _dump(value: dict[str, Any]) -> bytes:
|
|
296
|
+
return json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode()
|
|
@@ -18,7 +18,7 @@ from dataclasses import dataclass
|
|
|
18
18
|
from pathlib import Path
|
|
19
19
|
from typing import Iterable, Iterator, Mapping
|
|
20
20
|
|
|
21
|
-
from . import catalog, layout, objects, refs
|
|
21
|
+
from . import catalog, layout, objects, profiles, refs
|
|
22
22
|
from . import dataset as dataset_json
|
|
23
23
|
from .store import Store, is_conflict, is_missing
|
|
24
24
|
from .version import Catalog, DuckLake, File, Version, VersionError, new_id, parse, utc_now
|
|
@@ -469,6 +469,58 @@ def update_refs_index(store: Store) -> None:
|
|
|
469
469
|
warnings.warn(f"the ref moved, but refs.json was not rewritten: {exc}", stacklevel=2)
|
|
470
470
|
|
|
471
471
|
|
|
472
|
+
def write_profiles(
|
|
473
|
+
store: Store,
|
|
474
|
+
local: Path,
|
|
475
|
+
parent: Version | None,
|
|
476
|
+
working: Path,
|
|
477
|
+
*,
|
|
478
|
+
data_path: str,
|
|
479
|
+
setup: Iterable[str] = (),
|
|
480
|
+
) -> int:
|
|
481
|
+
"""Write a profile of each table of a committed version under ``local/profiles/``.
|
|
482
|
+
|
|
483
|
+
A table whose data files and columns are the parent's keeps the parent's
|
|
484
|
+
profile, so a commit reads only the tables that changed. Returns how many
|
|
485
|
+
were made from the data.
|
|
486
|
+
"""
|
|
487
|
+
local = Path(local)
|
|
488
|
+
described_path = local / layout.DATASET
|
|
489
|
+
if not described_path.is_file():
|
|
490
|
+
return 0
|
|
491
|
+
described = json.loads(described_path.read_text(encoding="utf-8"))
|
|
492
|
+
files_now = json.loads((local / layout.FILES).read_text(encoding="utf-8")).get("tables", {})
|
|
493
|
+
files_before: dict = {}
|
|
494
|
+
if parent is not None and parent.files is not None:
|
|
495
|
+
try:
|
|
496
|
+
files_before = json.loads(store.get(layout.version_dir(parent.id) + parent.files.path)).get("tables", {})
|
|
497
|
+
except Exception as exc: # noqa: BLE001 - stores raise their own types
|
|
498
|
+
if not is_missing(exc):
|
|
499
|
+
raise
|
|
500
|
+
kept: dict[str, bytes] = {}
|
|
501
|
+
pending = []
|
|
502
|
+
for table in profiles.tables(described):
|
|
503
|
+
key = f"{table['schema']}.{table['name']}"
|
|
504
|
+
relative = profiles.path(table["schema"], table["name"])
|
|
505
|
+
previous = None
|
|
506
|
+
if parent is not None and key in files_before and files_before[key] == files_now.get(key):
|
|
507
|
+
try:
|
|
508
|
+
previous = store.get(layout.version_dir(parent.id) + relative)
|
|
509
|
+
except Exception as exc: # noqa: BLE001
|
|
510
|
+
if not is_missing(exc):
|
|
511
|
+
raise
|
|
512
|
+
if previous is not None and profiles.reusable(previous, table):
|
|
513
|
+
kept[relative] = previous
|
|
514
|
+
else:
|
|
515
|
+
pending.append(table)
|
|
516
|
+
made = profiles.build(working, pending, data_path=data_path, setup=setup)
|
|
517
|
+
for relative, content in {**kept, **made}.items():
|
|
518
|
+
target = local / relative
|
|
519
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
520
|
+
target.write_bytes(content)
|
|
521
|
+
return len(made)
|
|
522
|
+
|
|
523
|
+
|
|
472
524
|
def upload(store: Store, local: Path, version: Version) -> None:
|
|
473
525
|
"""Put a committed version's files, catalog first and the record last.
|
|
474
526
|
|
|
@@ -483,6 +535,11 @@ def upload(store: Store, local: Path, version: Version) -> None:
|
|
|
483
535
|
_put_once(store, layout.version_dir(version.id) + version.files.path, local / version.files.path)
|
|
484
536
|
if version.tree is not None:
|
|
485
537
|
_put_once(store, layout.version_dir(version.id) + version.tree.path, local / version.tree.path)
|
|
538
|
+
profiled = local / layout.PROFILES_DIR
|
|
539
|
+
if profiled.is_dir():
|
|
540
|
+
for written in sorted(profiled.rglob("*.json")):
|
|
541
|
+
relative = written.relative_to(local).as_posix()
|
|
542
|
+
_put_once(store, layout.version_dir(version.id) + relative, written)
|
|
486
543
|
_put_once(store, layout.version_record(version.id), local / layout.VERSION_RECORD)
|
|
487
544
|
|
|
488
545
|
|
|
@@ -553,6 +610,7 @@ def expire(store: Store, version_id: str, *, reason: str = "retention") -> None:
|
|
|
553
610
|
layout.version_dir(version_id) + layout.FILES,
|
|
554
611
|
layout.version_dir(version_id) + layout.TREE,
|
|
555
612
|
*store.list(layout.version_dir(version_id) + layout.CATALOGS_DIR),
|
|
613
|
+
*store.list(layout.version_dir(version_id) + layout.PROFILES_DIR),
|
|
556
614
|
]
|
|
557
615
|
)
|
|
558
616
|
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""Profiles: a summary of each table, made from the data at commit."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import duckdb
|
|
9
|
+
import pytest
|
|
10
|
+
from typer.testing import CliRunner
|
|
11
|
+
|
|
12
|
+
from fdl import catalog, layout, profiles, repo
|
|
13
|
+
from fdl.cli import app
|
|
14
|
+
from fdl.stores import LocalStore
|
|
15
|
+
|
|
16
|
+
runner = CliRunner()
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def run(*args) -> str:
|
|
20
|
+
result = runner.invoke(app, [str(a) for a in args])
|
|
21
|
+
assert result.exit_code == 0, result.output
|
|
22
|
+
return result.output
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def write(working: Path, store: LocalStore, sql: str) -> None:
|
|
26
|
+
con = duckdb.connect()
|
|
27
|
+
for statement in catalog.attach_sql(working, alias="lake", data_path=repo.data_url(store)):
|
|
28
|
+
if statement:
|
|
29
|
+
con.execute(statement)
|
|
30
|
+
con.execute(sql)
|
|
31
|
+
con.close()
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@pytest.fixture
|
|
35
|
+
def repository(tmp_path: Path, monkeypatch):
|
|
36
|
+
monkeypatch.chdir(tmp_path)
|
|
37
|
+
remote, working = tmp_path / "remote", tmp_path / "lake.sqlite"
|
|
38
|
+
run("init", remote)
|
|
39
|
+
run("checkout", remote, working)
|
|
40
|
+
(tmp_path / "dataset.yml").write_text("name: towns\ntitle: Towns\n")
|
|
41
|
+
return LocalStore(remote), working
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def profile_of(store: LocalStore, table: str = "towns") -> dict:
|
|
45
|
+
head = repo.head(store)
|
|
46
|
+
return json.loads(store.get(f"{layout.version_dir(head.id)}profiles/main/{table}.json"))
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def test_a_commit_profiles_each_table(repository) -> None:
|
|
50
|
+
store, working = repository
|
|
51
|
+
write(
|
|
52
|
+
working,
|
|
53
|
+
store,
|
|
54
|
+
"CREATE TABLE lake.main.towns AS SELECT i AS id, 'town ' || (i % 3) AS name, "
|
|
55
|
+
"DATE '2020-01-01' + i::INTEGER AS day, i % 2 = 0 AS even FROM range(20) t(i)",
|
|
56
|
+
)
|
|
57
|
+
run("commit", working, "-m", "m", "--author", "t")
|
|
58
|
+
got = profile_of(store)
|
|
59
|
+
|
|
60
|
+
assert got["rows"] == 20
|
|
61
|
+
assert len(got["head"]) == profiles.HEAD_ROWS
|
|
62
|
+
by_name = {c["name"]: c for c in got["columns"]}
|
|
63
|
+
assert by_name["id"]["min"] == 0 and by_name["id"]["max"] == 19
|
|
64
|
+
assert by_name["name"]["distinct_count"] == 3
|
|
65
|
+
assert {v["value"]: v["count"] for v in by_name["name"]["values"]} == {"town 0": 7, "town 1": 7, "town 2": 6}
|
|
66
|
+
assert by_name["day"]["min"] == "2020-01-01"
|
|
67
|
+
assert by_name["even"]["true_count"] == 10
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def test_many_values_list_only_the_most_common(repository) -> None:
|
|
71
|
+
store, working = repository
|
|
72
|
+
write(working, store, f"CREATE TABLE lake.main.towns AS SELECT 'v' || i AS code FROM range({profiles.ALL_VALUES + 5}) t(i)")
|
|
73
|
+
run("commit", working, "-m", "m", "--author", "t")
|
|
74
|
+
code = profile_of(store)["columns"][0]
|
|
75
|
+
assert len(code["values"]) == profiles.TOP_VALUES
|
|
76
|
+
assert code["distinct_count"] > profiles.ALL_VALUES
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def test_an_unchanged_table_keeps_its_parents_profile(repository, monkeypatch) -> None:
|
|
80
|
+
store, working = repository
|
|
81
|
+
write(working, store, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
|
|
82
|
+
write(working, store, "CREATE TABLE lake.main.other AS SELECT 1 AS id")
|
|
83
|
+
run("commit", working, "-m", "m", "--author", "t")
|
|
84
|
+
write(working, store, "INSERT INTO lake.main.other VALUES (2)")
|
|
85
|
+
|
|
86
|
+
built: list[str] = []
|
|
87
|
+
original = profiles.build
|
|
88
|
+
|
|
89
|
+
def spy(working, wanted, **kwargs):
|
|
90
|
+
wanted = list(wanted)
|
|
91
|
+
built.extend(t["name"] for t in wanted)
|
|
92
|
+
return original(working, wanted, **kwargs)
|
|
93
|
+
|
|
94
|
+
monkeypatch.setattr(profiles, "build", spy)
|
|
95
|
+
run("commit", working, "-m", "again", "--author", "t")
|
|
96
|
+
assert built == ["other"]
|
|
97
|
+
assert profile_of(store, "towns")["rows"] == 1
|
|
98
|
+
assert profile_of(store, "other")["rows"] == 2
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def test_expiry_deletes_the_profiles(repository) -> None:
|
|
102
|
+
store, working = repository
|
|
103
|
+
write(working, store, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
|
|
104
|
+
run("commit", working, "-m", "m", "--author", "t")
|
|
105
|
+
first = repo.head(store).id
|
|
106
|
+
write(working, store, "INSERT INTO lake.main.towns VALUES (2)")
|
|
107
|
+
run("commit", working, "-m", "again", "--author", "t")
|
|
108
|
+
repo.expire(store, first)
|
|
109
|
+
assert store.list(f"{layout.version_dir(first)}profiles/") == []
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def test_a_geometry_column_gets_its_extent_and_a_sample(repository) -> None:
|
|
113
|
+
store, working = repository
|
|
114
|
+
try:
|
|
115
|
+
duckdb.connect().execute("INSTALL spatial; LOAD spatial")
|
|
116
|
+
except duckdb.Error:
|
|
117
|
+
pytest.skip("no spatial extension")
|
|
118
|
+
write(
|
|
119
|
+
working,
|
|
120
|
+
store,
|
|
121
|
+
"INSTALL spatial; LOAD spatial; CREATE TABLE lake.main.towns AS "
|
|
122
|
+
"SELECT i AS id, ST_Point(139 + i, 35) AS geom FROM range(3) t(i)",
|
|
123
|
+
)
|
|
124
|
+
run("commit", working, "-m", "m", "--author", "t")
|
|
125
|
+
got = profile_of(store)
|
|
126
|
+
assert got["geometry"]["column"] == "geom"
|
|
127
|
+
assert got["geometry"]["count"] == 3
|
|
128
|
+
assert got["geometry"]["bbox"] == [139, 35, 141, 35]
|
|
129
|
+
assert got["geometry"]["types"] == ["POINT"]
|
|
130
|
+
assert len(got["geometry"]["sample"]["features"]) == 3
|
|
131
|
+
assert [r.keys() for r in got["head"]][0] == {"id"}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|