fdl-cli 0.19.0__tar.gz → 0.20.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/CHANGELOG.md +14 -0
  2. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/PKG-INFO +1 -1
  3. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/reference/cli.md +2 -0
  4. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/spec/index.md +1 -1
  5. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/spec/layout.md +2 -1
  6. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/spec/versions.md +45 -0
  7. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/spec/writing.md +2 -2
  8. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/reference/cli.md +2 -0
  9. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/spec/index.md +1 -1
  10. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/spec/layout.md +2 -1
  11. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/spec/versions.md +45 -0
  12. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/spec/writing.md +2 -2
  13. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/pyproject.toml +1 -1
  14. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/src/fdl/cli.py +17 -3
  15. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/src/fdl/layout.py +2 -0
  16. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/src/fdl/objects.py +12 -2
  17. fdl_cli-0.20.0/src/fdl/profiles.py +296 -0
  18. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/src/fdl/repo.py +59 -1
  19. fdl_cli-0.20.0/tests/test_profiles.py +131 -0
  20. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/.github/workflows/docs.yml +0 -0
  21. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/.github/workflows/publish.yml +0 -0
  22. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/.github/workflows/release.yml +0 -0
  23. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/.gitignore +0 -0
  24. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/.node-version +0 -0
  25. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/.python-version +0 -0
  26. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/CLAUDE.md +0 -0
  27. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/CONTRIBUTING.md +0 -0
  28. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/LICENSE +0 -0
  29. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/README.md +0 -0
  30. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/blume.config.ts +0 -0
  31. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/concepts/meta.ts +0 -0
  32. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/concepts/repositories.md +0 -0
  33. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/concepts/versions-and-refs.md +0 -0
  34. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/concepts/working-catalogs.md +0 -0
  35. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/getting-started.md +0 -0
  36. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/guides/building-a-host.md +0 -0
  37. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/guides/meta.ts +0 -0
  38. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/guides/mirrors.md +0 -0
  39. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/guides/public-and-private.md +0 -0
  40. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/guides/reading-without-duckdb.md +0 -0
  41. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/guides/serving-over-http.md +0 -0
  42. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/guides/stores.md +0 -0
  43. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/index.md +0 -0
  44. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/internals/ducklake-formats.md +0 -0
  45. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/internals/from-write-to-read.md +0 -0
  46. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/internals/inside-a-repository.md +0 -0
  47. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/internals/meta.ts +0 -0
  48. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/concepts/meta.ts +0 -0
  49. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/concepts/repositories.md +0 -0
  50. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/concepts/versions-and-refs.md +0 -0
  51. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/concepts/working-catalogs.md +0 -0
  52. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/getting-started.md +0 -0
  53. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/guides/building-a-host.md +0 -0
  54. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/guides/meta.ts +0 -0
  55. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/guides/mirrors.md +0 -0
  56. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/guides/public-and-private.md +0 -0
  57. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/guides/reading-without-duckdb.md +0 -0
  58. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/guides/serving-over-http.md +0 -0
  59. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/guides/stores.md +0 -0
  60. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/index.md +0 -0
  61. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/internals/ducklake-formats.md +0 -0
  62. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/internals/from-write-to-read.md +0 -0
  63. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/internals/inside-a-repository.md +0 -0
  64. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/internals/meta.ts +0 -0
  65. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/reference/meta.ts +0 -0
  66. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/reference/python.md +0 -0
  67. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/spec/data-files.md +0 -0
  68. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/spec/http.md +0 -0
  69. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/spec/meta.ts +0 -0
  70. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/spec/reading.md +0 -0
  71. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/ja/spec/refs.md +0 -0
  72. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/reference/meta.ts +0 -0
  73. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/reference/python.md +0 -0
  74. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/spec/data-files.md +0 -0
  75. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/spec/http.md +0 -0
  76. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/spec/meta.ts +0 -0
  77. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/spec/reading.md +0 -0
  78. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/docs/spec/refs.md +0 -0
  79. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/package-lock.json +0 -0
  80. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/package.json +0 -0
  81. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/public/CNAME +0 -0
  82. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/src/fdl/__init__.py +0 -0
  83. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/src/fdl/catalog.py +0 -0
  84. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/src/fdl/dataset.py +0 -0
  85. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/src/fdl/py.typed +0 -0
  86. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/src/fdl/refs.py +0 -0
  87. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/src/fdl/store.py +0 -0
  88. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/src/fdl/stores.py +0 -0
  89. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/src/fdl/version.py +0 -0
  90. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/tests/http_repo.py +0 -0
  91. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/tests/test_cli.py +0 -0
  92. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/tests/test_dataset.py +0 -0
  93. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/tests/test_http.py +0 -0
  94. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/tests/test_objects.py +0 -0
  95. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/tests/test_repo.py +0 -0
  96. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/tests/test_stores.py +0 -0
  97. {fdl_cli-0.19.0 → fdl_cli-0.20.0}/uv.lock +0 -0
@@ -1,6 +1,20 @@
1
1
  # CHANGELOG
2
2
 
3
3
 
4
+ ## v0.20.0 (2026-11-06)
5
+
6
+ ### Documentation
7
+
8
+ - Drop the note that the draft spec changes as tools are built
9
+ ([#36](https://github.com/queria-io/fdl/pull/36),
10
+ [`de531b8`](https://github.com/queria-io/fdl/commit/de531b806a3b1d65fe4e5347ce203d22573981d6))
11
+
12
+ ### Features
13
+
14
+ - Write a profile of each table beside a version ([#39](https://github.com/queria-io/fdl/pull/39),
15
+ [`bdabd3f`](https://github.com/queria-io/fdl/commit/bdabd3fcd3f0b777609de78691b8ad64a1c40c26))
16
+
17
+
4
18
  ## v0.19.0 (2026-11-06)
5
19
 
6
20
  ### Features
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: fdl-cli
3
- Version: 0.19.0
3
+ Version: 0.20.0
4
4
  Summary: Versions of a DuckLake catalog on object storage, kept the way Git keeps commits
5
5
  Project-URL: Homepage, https://fdl.flo8s.com/
6
6
  Project-URL: Documentation, https://fdl.flo8s.com/
@@ -53,6 +53,8 @@ fdl commit <working.sqlite> -m "What changed" --author alice [--dataset dataset.
53
53
 
54
54
  コマンドは、説明の隣の `README.md` と、そこにある `docs/` より下のファイル(隠しファイルを除く)をバージョンの tree に入れます。そのフォルダからのパスで `tree.json` を書き、リポジトリに無いものを書いたままの中身で `assets/<sha256>` にアップロードします([tree.json](/ja/spec/versions#treejson))。
55
55
 
56
+ あわせてテーブルごとの profile を書きます([profiles/](/ja/spec/versions#profiles))。読むのは、データファイルか列が親から変わったテーブルだけです。読めなかったテーブルには profile を書かず、警告を出してコミットを続けます。
57
+
56
58
  `asset: true` の印を付けた列は、ファイルの sha256 を持ちます([アセット](/ja/spec/versions#アセット))。コマンドはその列の値を読み、リポジトリに無いファイルを `--assets` のフォルダから `assets/<sha256>` にアップロードします。ファイルは中身で照合するので、フォルダの中のファイル名は問いません。
57
59
 
58
60
  親と同じ中身になるとき([変更がないとき](/ja/spec/writing#変更がないとき))は `Nothing to commit` と表示し、ブランチを動かさずに終了コード 0 で終わります。同じ行を新しいファイルに書き直しただけの作り直しも、変更なしとみなします。`--allow-empty` を付けると、それでもバージョンを作ります。行を読めず判定できないときは、警告を出してバージョンを作ります。
@@ -5,7 +5,7 @@ sidebar:
5
5
  description: FDL 0.1(草案)の規範となる説明。何を決め、何をホストに任せ、どんな用語を使うか。
6
6
  ---
7
7
 
8
- FDL 0.1 の仕様です。草案で、実装を作りながら変えていきます。ツールはこの仕様に合わせて作り、ツールと仕様が食い違ったときは、先に仕様を直してからツールを合わせます。この日本語訳は参考で、英語版を正とします。
8
+ FDL 0.1 の仕様の草案です。ツールはこの仕様に合わせて作り、ツールと仕様が食い違ったときは、先に仕様を直してからツールを合わせます。この日本語訳は参考で、英語版を正とします。
9
9
 
10
10
  FDL は、[DuckLake](https://ducklake.select/) のカタログのバージョンを残すための、置き場のファイル配置です。Git がコミットを保存する形にならっています。どのファイルを置き場のどこに置き、読み手と書き手がそれをどう扱うかを決めます。サーバーは要りません。置き場に求めるのは、読む・書く・条件付きで書くの 3 つです。
11
11
 
@@ -23,9 +23,10 @@ description: リポジトリのファイル、そのパス、fdl.json の中身
23
23
  dataset.json バージョンの中身の説明
24
24
  files.json カタログから作った、表ごとのデータファイルの索引(任意)
25
25
  tree.json README など、バージョンのほかのファイルのパスの一覧(任意)
26
+ profiles/<schema>/<table>.json データから測ったテーブルごとの要約(任意)
26
27
  catalogs/<format>.duckdb 別の DuckLake 形式のカタログ(任意)
27
28
  expired.json 保持期間を過ぎた印(過ぎたときだけ)
28
- <others> ホストが足すファイル(プレビューなど)
29
+ <others> ホストが足すファイル
29
30
  <data_dir><schema>/<table>/ducklake-<uuid>.parquet データファイル(共有、変えない)
30
31
  assets/<sha256> テーブルの行やバージョンの tree.json が指すファイル(共有、変更しない)
31
32
  ```
@@ -154,6 +154,51 @@ DuckLake を使わずに Parquet ファイルを読む読み手は、`deletes`
154
154
  - 書き手は、ファイルを編集する場所(Git のリポジトリの中の YAML の隣など)に置き、書き手のツールが `tree.json` を書く。書き手が手で書くものではない
155
155
  - `tree.json` はバージョンの一部で、そのバージョンの中では変わらない。保持期間から外すときはカタログと一緒に消す
156
156
 
157
+ ## profiles/
158
+
159
+ profile は、`ducklake.snapshot_id` の時点の 1 つのテーブルの要約です。行数、列ごとの中身、先頭の行、図形の列を持つテーブルでは図形の範囲と見本を持ちます。読み手はこの小さなファイル 1 つで、カタログを開かずデータも読まずにテーブルを表示できます。ホストは検索の索引をこれから作れます。profile が持つのはデータから測った事実で、列が何を意味するかは書き手が `dataset.json` に書きます。`files.json` と同じくカタログとデータから作るもので、食い違うときはカタログとデータが正です。任意で、読み手は profile が無くても読めなければなりません。
160
+
161
+ テーブルごとの profile は `versions/<ID>/profiles/<schema>/<table>.json` に置きます。スキーマ名かテーブル名が空・`.`・`..` のとき、または `/` を含むときは置きません。ビューには置きません。
162
+
163
+ ```json profiles/main/towns.json
164
+ {
165
+ "rows": 1741,
166
+ "columns": [
167
+ { "name": "town_code", "type": "varchar", "null_count": 0, "distinct_count": 1741,
168
+ "values": [{ "value": "082201", "count": 1 }] },
169
+ { "name": "prefecture", "type": "varchar", "null_count": 0, "distinct_count": 2,
170
+ "values": [{ "value": "茨城県", "count": 1200 }, { "value": "栃木県", "count": 541 }] },
171
+ { "name": "population", "type": "int64", "null_count": 3, "distinct_count": 1650,
172
+ "min": 0, "max": 84211, "mean": 3120.5, "std": 4410.2, "q1": 420, "median": 1630, "q3": 3990,
173
+ "values": [{ "value": "0", "count": 12 }],
174
+ "histogram": [{ "start": 0, "end": 4210.55, "count": 1302 }] },
175
+ { "name": "surveyed_on", "type": "date", "null_count": 0, "distinct_count": 1,
176
+ "min": "2025-10-01", "max": "2025-10-01", "span_days": 0,
177
+ "values": [{ "value": "2025-10-01", "count": 1741 }] },
178
+ { "name": "coastal", "type": "boolean", "null_count": 0, "true_count": 210, "false_count": 1531 },
179
+ { "name": "geom", "type": "geometry" }
180
+ ],
181
+ "head": [{ "town_code": "082201", "prefecture": "茨城県", "population": 2210, "surveyed_on": "2025-10-01", "coastal": false }],
182
+ "geometry": {
183
+ "column": "geom",
184
+ "count": 1741,
185
+ "bbox": [139.69, 35.74, 140.85, 37.15],
186
+ "types": ["MULTIPOLYGON", "POLYGON"],
187
+ "sample": { "type": "FeatureCollection", "features": [] }
188
+ }
189
+ }
190
+ ```
191
+
192
+ - `rows` はテーブルの行数。`columns` にはカタログの順にすべての列を並べ、`name` と `type`(DuckLake の型名)は `dataset.json` と同じにする
193
+ - 図形の列を除くすべての列が `null_count` を持つ。数値・テキスト・日付と時刻の列は `distinct_count` と `values` を持つ。大きなテーブルの `distinct_count` は推定値のことがある
194
+ - `values` は、値(テキスト)とその値を持つ行の数を、多い順、同数なら値の順に並べる。列が持つ異なる値が 1,000 以下なら全部を並べ、そのとき `distinct_count` は正確な数になる。読み手は、`values` の長さが `distinct_count` と等しいことで全部の一覧だと判定する。それより多い列は上位 10 件だけを並べる。ホストは全部の一覧を、地域コードや名前などの値で検索できる
195
+ - 数値の列は `min`・`max`・`mean`・`std`、四分位数の `q1`・`median`・`q3`、`min` から `max` までを等幅に区切った `histogram` を持つ。日付と時刻の列は `min` と `max` をテキストで持ち、日付とタイムスタンプは `span_days` も持つ。真偽値の列は `true_count` と `false_count` を持つ
196
+ - `head` は、テーブルが返す先頭の行を JSON のオブジェクトで持つ。図形の列と、値を JSON で表せない列は入れない
197
+ - `geometry` は、書き手が読めたときに、最初の図形の列を説明する。図形を持つ行の数、全体の `bbox`(列の座標で `[xmin, ymin, xmax, ymax]`)、含まれる図形の種類、地図を描くための、無作為に選んで簡略化した図形の GeoJSON `FeatureCollection` を持つ
198
+ - 書き手は計算できないキーを省いてよい。読み手は知らないキーを無視する
199
+ - 書き手は、データファイル(`files.json`)と列が親と同じテーブルには親の profile を使い回す。コミットで読むのは変わったテーブルだけになる
200
+ - profile はバージョンの一部で、そのバージョンの中では変わらない。保持期間から外すときはカタログと一緒に消す
201
+
157
202
  ## 固めたカタログ
158
203
 
159
204
  `ducklake.duckdb` は、DuckDB のファイル形式の DuckLake カタログです。
@@ -17,7 +17,7 @@ description: チェックアウト、コミット、push。作業カタログ、
17
17
  | --- | --- |
18
18
  | チェックアウト | ブランチのバージョンのカタログから作業カタログを作り、そのバージョンを覚える。最初のバージョンの前は空のものを作る |
19
19
  | コミット | 作業カタログの最後のスナップショットを新しいバージョンに固める。親は覚えておいたバージョン。中身を `dataset.json` で説明する |
20
- | push | データファイルを確かめ、カタログ、`dataset.json`・`files.json`・`tree.json`、`version.json` の順にアップロードし、条件付きで ref を動かす |
20
+ | push | データファイルを確かめ、カタログ、`dataset.json`・`files.json`・`tree.json`・profile、`version.json` の順にアップロードし、条件付きで ref を動かす |
21
21
 
22
22
  チェックアウトでは、DuckLake の管理用の表を、固めたカタログから SQLite へそのまま写します。固めるときはその逆で、最後のスナップショットまでの行を残します。どちらもデータファイルは書き換えません。
23
23
 
@@ -40,7 +40,7 @@ description: チェックアウト、コミット、push。作業カタログ、
40
40
 
41
41
  1. データファイル・アセット
42
42
  2. `versions/<ID>/ducklake.duckdb`(と `catalogs/`)
43
- 3. `versions/<ID>/dataset.json`・`versions/<ID>/files.json`・`versions/<ID>/tree.json`
43
+ 3. `versions/<ID>/dataset.json`・`versions/<ID>/files.json`・`versions/<ID>/tree.json`・`versions/<ID>/profiles/`
44
44
  4. `versions/<ID>/version.json`
45
45
  5. ref(条件付き)
46
46
 
@@ -53,6 +53,8 @@ Every version is described by a `dataset.json` (see [Versions](/spec/versions#da
53
53
 
54
54
  The command puts `README.md` beside the description and every file below `docs/` there (hidden ones left out) in the version's tree: it writes `tree.json` with their paths from that folder and uploads those the repository lacks to `assets/<sha256>`, as written (see [tree.json](/spec/versions#treejson)).
55
55
 
56
+ It also writes a profile of each table (see [profiles/](/spec/versions#profiles)), reading only the tables whose data files or columns changed since the parent. A table it cannot read gets none, and the commit goes on with a warning.
57
+
56
58
  Columns marked `asset: true` hold the sha256 of files (see [Assets](/spec/versions#assets)). The command reads the values those columns hold and uploads the files the repository lacks from the `--assets` folder to `assets/<sha256>`, by their content: file names in the folder do not matter.
57
59
 
58
60
  When the version would hold what its parent holds (see [When nothing changed](/spec/writing#when-nothing-changed)), it prints `Nothing to commit`, leaves the branch where it is and exits 0. A rebuild that writes the same rows to new files counts as unchanged. `--allow-empty` makes the version anyway. If the rows cannot be read to tell, it warns and makes the version.
@@ -5,7 +5,7 @@ sidebar:
5
5
  description: The normative description of FDL 0.1 (draft) - what it decides, what it leaves to hosts, and the terms it uses.
6
6
  ---
7
7
 
8
- This is the specification of FDL 0.1, a draft. It changes as the tools that implement it are built. Tools are written against this specification, and where a tool and this text disagree, the text is changed first and the tool follows.
8
+ This is the specification of FDL 0.1, a draft. Tools are written against this specification, and where a tool and this text disagree, the text is changed first and the tool follows.
9
9
 
10
10
  FDL is a storage layout for keeping versions of a [DuckLake](https://ducklake.select/) catalog, modelled on how Git stores commits. It says which files go where in a store and how readers and writers handle them. No server is needed: the store has to read, write and write on condition.
11
11
 
@@ -23,9 +23,10 @@ Paths are relative to the root, lower case, and separated by `/`. The root is wh
23
23
  dataset.json what the version holds, described
24
24
  files.json each table's data files, indexed from the catalog (optional)
25
25
  tree.json the other files of the version, such as a README, by path (optional)
26
+ profiles/<schema>/<table>.json a summary of each table, measured from the data (optional)
26
27
  catalogs/<format>.duckdb the catalog in another DuckLake format (optional)
27
28
  expired.json the past-retention mark (only once expired)
28
- <others> files a host adds, such as previews
29
+ <others> files a host adds
29
30
  <data_dir><schema>/<table>/ducklake-<uuid>.parquet data files (shared, immutable)
30
31
  assets/<sha256> files the rows of a table or a version's tree.json point at (shared, immutable)
31
32
  ```
@@ -154,6 +154,51 @@ When a version is taken out of retention, `files.json` is deleted with its catal
154
154
  - A writer keeps the files where it edits them, such as beside its YAML in a Git repository, and its tool writes `tree.json`. The writer does not write it by hand
155
155
  - `tree.json` is part of the version and does not change with it. It is deleted with the catalogs when the version is taken out of retention
156
156
 
157
+ ## profiles/
158
+
159
+ A profile summarizes one table at `ducklake.snapshot_id`: its row count, what each column holds, its first rows and, for a table with a geometry column, the extent and a sample of its shapes. A reader can show a table, or a host can index it for search, from this one small file, without opening the catalog or reading the data. It holds what was measured from the data; what a column means is the writer's, in `dataset.json`. Like `files.json`, it is made from the catalog and the data, which win where the two differ, and it is optional: a reader must work without it.
160
+
161
+ Each table's profile is at `versions/<ID>/profiles/<schema>/<table>.json`. A table whose schema or name is empty, `.`, `..` or holds `/` has none. Views have none.
162
+
163
+ ```json profiles/main/towns.json
164
+ {
165
+ "rows": 1741,
166
+ "columns": [
167
+ { "name": "town_code", "type": "varchar", "null_count": 0, "distinct_count": 1741,
168
+ "values": [{ "value": "082201", "count": 1 }] },
169
+ { "name": "prefecture", "type": "varchar", "null_count": 0, "distinct_count": 2,
170
+ "values": [{ "value": "Ibaraki", "count": 1200 }, { "value": "Tochigi", "count": 541 }] },
171
+ { "name": "population", "type": "int64", "null_count": 3, "distinct_count": 1650,
172
+ "min": 0, "max": 84211, "mean": 3120.5, "std": 4410.2, "q1": 420, "median": 1630, "q3": 3990,
173
+ "values": [{ "value": "0", "count": 12 }],
174
+ "histogram": [{ "start": 0, "end": 4210.55, "count": 1302 }] },
175
+ { "name": "surveyed_on", "type": "date", "null_count": 0, "distinct_count": 1,
176
+ "min": "2025-10-01", "max": "2025-10-01", "span_days": 0,
177
+ "values": [{ "value": "2025-10-01", "count": 1741 }] },
178
+ { "name": "coastal", "type": "boolean", "null_count": 0, "true_count": 210, "false_count": 1531 },
179
+ { "name": "geom", "type": "geometry" }
180
+ ],
181
+ "head": [{ "town_code": "082201", "prefecture": "Ibaraki", "population": 2210, "surveyed_on": "2025-10-01", "coastal": false }],
182
+ "geometry": {
183
+ "column": "geom",
184
+ "count": 1741,
185
+ "bbox": [139.69, 35.74, 140.85, 37.15],
186
+ "types": ["MULTIPOLYGON", "POLYGON"],
187
+ "sample": { "type": "FeatureCollection", "features": [] }
188
+ }
189
+ }
190
+ ```
191
+
192
+ - `rows` is the table's row count. `columns` lists every column in the catalog's order, with `name` and `type` (DuckLake's type name) as in `dataset.json`
193
+ - Every column but a geometry one has `null_count`. Numbers, text, dates and times have `distinct_count`, which may be estimated on a large table, and `values`
194
+ - `values` lists each value, as text, with the number of rows that hold it, most common first and then by value. When the column holds at most 1,000 distinct values, it lists them all and `distinct_count` is exact: a reader tells a whole list by its length matching `distinct_count`. Otherwise it lists the 10 most common. A host can search a whole list by value, such as a town's code or name
195
+ - Numbers have `min`, `max`, `mean`, `std`, the quartiles `q1`, `median` and `q3`, and a `histogram` of equal-width buckets between `min` and `max`. Dates and times have `min` and `max` as text, and dates and timestamps `span_days`. Booleans have `true_count` and `false_count`
196
+ - `head` holds the first rows the table returns, as JSON objects, leaving out geometry and columns whose values have no JSON form
197
+ - `geometry` describes the first geometry column, when the writer could read it: how many rows have a shape, the `bbox` of all of them (`[xmin, ymin, xmax, ymax]` in the column's coordinates), the geometry types present, and a GeoJSON `FeatureCollection` of a random sample of simplified shapes for drawing a map
198
+ - A writer may leave out any key it cannot compute. A reader ignores keys it does not know
199
+ - A writer reuses its parent's profile for a table whose data files (in `files.json`) and columns are the parent's, so a commit reads only the tables that changed
200
+ - Profiles are part of the version and do not change with it. They are deleted with the catalogs when the version is taken out of retention
201
+
157
202
  ## The frozen catalog
158
203
 
159
204
  `ducklake.duckdb` is a DuckLake catalog in DuckDB's file format.
@@ -17,7 +17,7 @@ A writer writes to a working catalog: a DuckLake catalog in SQLite, outside the
17
17
  | --- | --- |
18
18
  | checkout | Make a working catalog from the catalog of the branch's version, and remember that version. Before the first version, make an empty one |
19
19
  | commit | Freeze the working catalog's last snapshot into a new version, with the remembered version as its parent, and describe it in `dataset.json` |
20
- | push | Check the data files, upload the catalog, `dataset.json`, `files.json` and `tree.json`, then `version.json`, then move the ref on condition |
20
+ | push | Check the data files, upload the catalog, `dataset.json`, `files.json`, `tree.json` and the profiles, then `version.json`, then move the ref on condition |
21
21
 
22
22
  Checking out copies DuckLake's bookkeeping tables from the frozen catalog into SQLite as they are. Freezing does the reverse, keeping the rows up to the last snapshot. Neither rewrites a data file.
23
23
 
@@ -40,7 +40,7 @@ A table rebuilt from the same input is written to new data files, so comparing f
40
40
 
41
41
  1. Data files and assets
42
42
  2. `versions/<ID>/ducklake.duckdb` (and any `catalogs/`)
43
- 3. `versions/<ID>/dataset.json`, `versions/<ID>/files.json` and `versions/<ID>/tree.json`
43
+ 3. `versions/<ID>/dataset.json`, `versions/<ID>/files.json`, `versions/<ID>/tree.json` and `versions/<ID>/profiles/`
44
44
  4. `versions/<ID>/version.json`
45
45
  5. The ref, on condition
46
46
 
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "fdl-cli"
3
- version = "0.19.0"
3
+ version = "0.20.0"
4
4
  description = "Versions of a DuckLake catalog on object storage, kept the way Git keeps commits"
5
5
  readme = "README.md"
6
6
  license = "MIT"
@@ -148,12 +148,16 @@ def commit(
148
148
  typer.echo(f"warning: could not tell whether anything changed ({exc}); committing", err=True)
149
149
  version = make(True)
150
150
  local = Path(state["data"]) if state.get("data") else None
151
+ reading = Path(state["data"]).resolve().as_posix() + "/" if local else repo.data_url(store)
152
+ setup = _read_setup(store)
151
153
  described = json.loads((out / version.id / layout.DATASET).read_text(encoding="utf-8"))
152
154
  held = objects.asset_values(
153
- working,
154
- objects.asset_columns(described),
155
- data_path=Path(state["data"]).resolve().as_posix() + "/" if local else repo.data_url(store),
155
+ working, objects.asset_columns(described), data_path=reading, setup=setup
156
156
  )
157
+ try:
158
+ repo.write_profiles(store, out / version.id, parent, working, data_path=reading, setup=setup)
159
+ except Exception as exc: # noqa: BLE001 - a profile is never worth a failed commit
160
+ typer.echo(f"warning: tables were not profiled ({exc})", err=True)
157
161
  local_objects = objects.ByContent(
158
162
  {item["sha256"]: root / item["path"] for item in listed},
159
163
  assets_dir,
@@ -180,6 +184,16 @@ def commit(
180
184
  typer.echo(f"{state['ref']} -> {version.id} (version {version.seq})")
181
185
 
182
186
 
187
+ def _read_setup(store) -> list[str]:
188
+ """What DuckDB needs to read the repository's data files: the token, over HTTP."""
189
+ token = getattr(store, "token", None)
190
+ if not token:
191
+ return []
192
+ scope = store.url.replace("'", "''")
193
+ secret = token.replace("'", "''")
194
+ return [f"CREATE SECRET fdl_remote (TYPE http, BEARER_TOKEN '{secret}', SCOPE '{scope}')"]
195
+
196
+
183
197
  @app.command()
184
198
  def assets(folder: Path = typer.Argument(..., help="The folder of files the rows will point at")) -> None:
185
199
  """List the files below a folder as CSV: path, sha256, bytes, media_type.
@@ -29,6 +29,8 @@ EXPIRED = "expired.json"
29
29
  CATALOGS_DIR = "catalogs/"
30
30
  #: The other files of a version (its README, say), by path to the asset that holds each (optional).
31
31
  TREE = "tree.json"
32
+ #: A summary of each table, made from the data, under the version (optional).
33
+ PROFILES_DIR = "profiles/"
32
34
  #: Files the rows of a table point at, by sha256 (shared).
33
35
  ASSETS_DIR = "assets/"
34
36
 
@@ -174,9 +174,17 @@ def asset_columns(described: dict[str, Any]) -> list[tuple[str, str, str]]:
174
174
 
175
175
 
176
176
  def asset_values(
177
- working: Path, columns: Iterable[tuple[str, str, str]], *, data_path: str
177
+ working: Path,
178
+ columns: Iterable[tuple[str, str, str]],
179
+ *,
180
+ data_path: str,
181
+ setup: Iterable[str] = (),
178
182
  ) -> set[str]:
179
- """The sha256 the marked columns hold in the working catalog's last snapshot."""
183
+ """The sha256 the marked columns hold in the working catalog's last snapshot.
184
+
185
+ ``setup`` runs before the catalog is attached: a ``CREATE SECRET`` for data
186
+ files in private storage, say.
187
+ """
180
188
  import duckdb
181
189
 
182
190
  columns = list(columns)
@@ -189,6 +197,8 @@ def asset_values(
189
197
  found: set[str] = set()
190
198
  con = duckdb.connect()
191
199
  try:
200
+ for statement in setup:
201
+ con.execute(statement)
192
202
  for statement in catalog.attach_sql(Path(working), alias="lake", data_path=data_path, read_only=True):
193
203
  if statement:
194
204
  con.execute(statement)
@@ -0,0 +1,296 @@
1
+ """Profiles: a summary of each table, written beside a version for readers that show it.
2
+
3
+ A profile holds a table's row count, statistics of each column, its first rows
4
+ and, for a table with a geometry column, a sample of its shapes. A reader can
5
+ show a table from this one small file without opening the catalog or reading
6
+ the data. It is made from the catalog and the data, which win where the two
7
+ differ, and a reader works without it (see the specification, "profiles").
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import json
13
+ import math
14
+ import re
15
+ from collections.abc import Iterable
16
+ from pathlib import Path
17
+ from typing import Any
18
+
19
+ from . import catalog
20
+
21
+ #: Rows in `head`. A glance; more is what a query is for.
22
+ HEAD_ROWS = 10
23
+ #: Shapes in `geometry`, sampled from the whole table rather than its first rows.
24
+ GEOMETRY_SAMPLE = 500
25
+ #: Vertex tolerance of the sampled shapes, in the column's units (about 11 m in degrees).
26
+ SIMPLIFY_TOLERANCE = 0.0001
27
+ HISTOGRAM_BUCKETS = 20
28
+ #: A column with at most this many distinct values lists them all in `values`.
29
+ ALL_VALUES = 1000
30
+ #: Otherwise `values` lists the most common.
31
+ TOP_VALUES = 10
32
+ #: Above this many rows, distinct values are estimated rather than counted.
33
+ EXACT_DISTINCT_ROWS = 1_000_000
34
+
35
+ _NUMERIC = {
36
+ "TINYINT", "SMALLINT", "INTEGER", "BIGINT", "HUGEINT", "UTINYINT", "USMALLINT",
37
+ "UINTEGER", "UBIGINT", "UHUGEINT", "FLOAT", "DOUBLE", "DECIMAL", "REAL", "NUMERIC",
38
+ "INT8", "INT16", "INT32", "INT64", "INT128", "UINT8", "UINT16", "UINT32", "UINT64",
39
+ "UINT128", "FLOAT32", "FLOAT64",
40
+ } # fmt: skip
41
+ _TEMPORAL = {"DATE", "TIME", "TIMESTAMP", "TIMESTAMPTZ", "TIMESTAMP WITH TIME ZONE", "TIMETZ", "INTERVAL"}
42
+ _DATED = {"DATE", "TIMESTAMP", "TIMESTAMPTZ", "TIMESTAMP WITH TIME ZONE"}
43
+ _BOOLEAN = {"BOOLEAN"}
44
+ _TEXT = {"VARCHAR", "UUID"}
45
+
46
+
47
+ def _base(data_type: str) -> str:
48
+ return re.sub(r"\(.*\)", "", data_type.upper()).strip()
49
+
50
+
51
+ def kind(data_type: str) -> str:
52
+ """The family of a DuckLake type, which decides the statistics a column gets."""
53
+ base = _base(data_type)
54
+ if base in _NUMERIC:
55
+ return "numeric"
56
+ if base in _TEMPORAL:
57
+ return "temporal"
58
+ if base in _BOOLEAN:
59
+ return "boolean"
60
+ if base in _TEXT:
61
+ return "text"
62
+ if base == "GEOMETRY":
63
+ return "geometry"
64
+ return "other"
65
+
66
+
67
+ def path(schema: str, table: str) -> str | None:
68
+ """The profile's path under the version, or None for a name a path cannot hold."""
69
+ if any(not part or part in (".", "..") or "/" in part for part in (schema, table)):
70
+ return None
71
+ return f"profiles/{schema}/{table}.json"
72
+
73
+
74
+ def tables(described: dict[str, Any]) -> list[dict[str, Any]]:
75
+ """The tables of a `dataset.json` that get a profile: every table, no view."""
76
+ return [t for t in described.get("tables", []) if t.get("kind") != "view" and path(t["schema"], t["name"])]
77
+
78
+
79
+ def columns_of(table: dict[str, Any]) -> list[dict[str, str]]:
80
+ return [{"name": f["name"], "type": f.get("type", "")} for f in table.get("fields", [])]
81
+
82
+
83
+ def build(working: Path, wanted: Iterable[dict[str, Any]], *, data_path: str, setup: Iterable[str] = ()) -> dict[str, bytes]:
84
+ """Profiles of ``wanted`` (tables of `dataset.json`) from the working catalog's last snapshot.
85
+
86
+ Returns path to content. A table that cannot be read gets no profile, and
87
+ a geometry sample that cannot be made (no spatial extension) is left out.
88
+ """
89
+ import duckdb
90
+
91
+ wanted = list(wanted)
92
+ if not wanted:
93
+ return {}
94
+ con = duckdb.connect()
95
+ try:
96
+ for statement in setup:
97
+ con.execute(statement)
98
+ for statement in catalog.attach_sql(Path(working), alias="lake", data_path=data_path, read_only=True):
99
+ if statement:
100
+ con.execute(statement)
101
+ spatial = _load_spatial(con)
102
+ made: dict[str, bytes] = {}
103
+ for table in wanted:
104
+ try:
105
+ profile = _profile(con, table, spatial)
106
+ except duckdb.Error:
107
+ continue
108
+ made[path(table["schema"], table["name"])] = _dump(profile)
109
+ return made
110
+ finally:
111
+ con.close()
112
+
113
+
114
+ def reusable(previous: bytes, table: dict[str, Any]) -> bool:
115
+ """Whether a parent's profile describes ``table`` as it is: the same columns and types."""
116
+ try:
117
+ held = json.loads(previous).get("columns", [])
118
+ except ValueError:
119
+ return False
120
+ return [{"name": c.get("name"), "type": c.get("type")} for c in held] == columns_of(table)
121
+
122
+
123
+
124
+ def _load_spatial(con) -> bool:
125
+ try:
126
+ con.execute("LOAD spatial")
127
+ return True
128
+ except Exception: # noqa: BLE001 - not installed: try once, then do without
129
+ try:
130
+ con.execute("INSTALL spatial")
131
+ con.execute("LOAD spatial")
132
+ return True
133
+ except Exception: # noqa: BLE001
134
+ return False
135
+
136
+
137
+ def _q(name: str) -> str:
138
+ return '"' + name.replace('"', '""') + '"'
139
+
140
+
141
+ def _profile(con, table: dict[str, Any], spatial: bool) -> dict[str, Any]:
142
+ relation = f"lake.{_q(table['schema'])}.{_q(table['name'])}"
143
+ fields = columns_of(table)
144
+ rows = int(con.execute(f"SELECT count(*) FROM {relation}").fetchone()[0])
145
+
146
+ measured = [f for f in fields if kind(f["type"]) != "geometry"]
147
+ exprs = [e for f in measured for e in _aggregates(f, rows)]
148
+ values = con.execute(f"SELECT {', '.join(exprs)} FROM {relation}").fetchone() if exprs else ()
149
+ stats: list[dict[str, Any]] = []
150
+ cursor = 0
151
+ for field in fields:
152
+ if kind(field["type"]) == "geometry":
153
+ stats.append(dict(field))
154
+ continue
155
+ width = len(_aggregates(field, rows))
156
+ stats.append(_column(con, relation, field, rows, values[cursor : cursor + width]))
157
+ cursor += width
158
+
159
+ shown = [f["name"] for f in fields if kind(f["type"]) not in ("geometry", "other")]
160
+ head: list[dict[str, Any]] = []
161
+ if shown:
162
+ encoded = con.execute(
163
+ f"SELECT to_json(list(t)) FROM (SELECT {', '.join(_q(c) for c in shown)} FROM {relation} LIMIT {HEAD_ROWS}) t"
164
+ ).fetchone()
165
+ head = json.loads(encoded[0]) if encoded and encoded[0] else []
166
+
167
+ profile: dict[str, Any] = {"rows": rows, "columns": stats, "head": head}
168
+ geometry = next((f["name"] for f in fields if kind(f["type"]) == "geometry"), None)
169
+ if geometry is not None and spatial:
170
+ try:
171
+ profile["geometry"] = _geometry(con, relation, geometry)
172
+ except Exception: # noqa: BLE001 - the rest stands without the shapes
173
+ pass
174
+ return profile
175
+
176
+
177
+ def _aggregates(field: dict[str, str], rows: int) -> list[str]:
178
+ c = _q(field["name"])
179
+ distinct = f"count(DISTINCT {c})" if rows <= EXACT_DISTINCT_ROWS else f"approx_count_distinct({c})"
180
+ family = kind(field["type"])
181
+ head = [f"count({c})"]
182
+ if family == "numeric":
183
+ v = f"{c}::DOUBLE"
184
+ return head + [
185
+ distinct,
186
+ f"min({v})",
187
+ f"max({v})",
188
+ f"avg({v})",
189
+ f"stddev_samp({v})",
190
+ f"approx_quantile({v}, [0.25, 0.5, 0.75])",
191
+ ]
192
+ if family == "text":
193
+ return head + [distinct]
194
+ if family == "temporal":
195
+ span = (
196
+ f"datediff('day', min({c})::TIMESTAMP, max({c})::TIMESTAMP)" if _base(field["type"]) in _DATED else "NULL"
197
+ )
198
+ return head + [f"min({c})::VARCHAR", f"max({c})::VARCHAR", span, distinct]
199
+ if family == "boolean":
200
+ return head + [f"count_if({c})", f"count_if(NOT {c})"]
201
+ return head
202
+
203
+
204
+ def _column(con, relation: str, field: dict[str, str], rows: int, values) -> dict[str, Any]:
205
+ out: dict[str, Any] = {**field, "null_count": rows - int(values[0])}
206
+ family = kind(field["type"])
207
+ if family == "numeric":
208
+ lo, hi, mean, std = (_number(v) for v in values[2:6])
209
+ q1, median, q3 = [_number(q) for q in values[6]] if values[6] else (None, None, None)
210
+ out["distinct_count"] = int(values[1] or 0)
211
+ out.update(min=lo, max=hi, mean=mean, std=std, q1=q1, median=median, q3=q3)
212
+ _values(con, relation, field["name"], out)
213
+ out["histogram"] = _histogram(con, relation, field["name"], lo, hi)
214
+ elif family == "text":
215
+ out["distinct_count"] = int(values[1] or 0)
216
+ _values(con, relation, field["name"], out)
217
+ elif family == "temporal":
218
+ lo, hi, span, distinct = values[1:5]
219
+ out.update(min=lo, max=hi, span_days=None if span is None else int(span), distinct_count=int(distinct or 0))
220
+ _values(con, relation, field["name"], out)
221
+ elif family == "boolean":
222
+ out.update(true_count=int(values[1] or 0), false_count=int(values[2] or 0))
223
+ return out
224
+
225
+
226
+ def _histogram(con, relation: str, name: str, lo: float | None, hi: float | None) -> list[dict[str, float]]:
227
+ if lo is None or hi is None:
228
+ return []
229
+ width = (hi - lo) / HISTOGRAM_BUCKETS if hi > lo else 1.0
230
+ found = con.execute(
231
+ f"SELECT least(floor(({_q(name)}::DOUBLE - ?) / ?)::BIGINT, ?), count(*) "
232
+ f"FROM {relation} WHERE {_q(name)} IS NOT NULL GROUP BY 1 ORDER BY 1",
233
+ [lo, width, HISTOGRAM_BUCKETS - 1],
234
+ ).fetchall()
235
+ return [{"start": lo + int(b) * width, "end": lo + (int(b) + 1) * width, "count": int(n)} for b, n in found]
236
+
237
+
238
+ def _values(con, relation: str, name: str, out: dict[str, Any]) -> None:
239
+ """Every value with its count when there are at most `ALL_VALUES`, else the most common.
240
+
241
+ Counted here, so `distinct_count` is exact whenever the list is whole: a
242
+ reader tells a whole list by its length matching `distinct_count`.
243
+ """
244
+ found = con.execute(
245
+ f"SELECT {_q(name)}::VARCHAR, count(*) FROM {relation} "
246
+ f"WHERE {_q(name)} IS NOT NULL GROUP BY 1 ORDER BY 2 DESC, 1 LIMIT {ALL_VALUES + 1}"
247
+ ).fetchall()
248
+ if len(found) <= ALL_VALUES:
249
+ out["distinct_count"] = len(found)
250
+ else:
251
+ found = found[:TOP_VALUES]
252
+ out["distinct_count"] = max(out.get("distinct_count", 0), ALL_VALUES + 1)
253
+ out["values"] = [{"value": v, "count": int(n)} for v, n in found]
254
+
255
+
256
+ def _geometry(con, relation: str, name: str) -> dict[str, Any]:
257
+ c = _q(name)
258
+ total, xmin, ymin, xmax, ymax = con.execute(
259
+ f"SELECT count({c}), min(ST_XMin({c})), min(ST_YMin({c})), max(ST_XMax({c})), max(ST_YMax({c})) "
260
+ f"FROM {relation}"
261
+ ).fetchone()
262
+ kinds = [
263
+ k
264
+ for (k,) in con.execute(
265
+ f"SELECT DISTINCT ST_GeometryType({c})::VARCHAR FROM {relation} WHERE {c} IS NOT NULL ORDER BY 1"
266
+ ).fetchall()
267
+ ]
268
+ sampled = con.execute(
269
+ f"SELECT ST_AsGeoJSON(ST_Simplify(g, {SIMPLIFY_TOLERANCE})) FROM "
270
+ f"(SELECT {c} AS g FROM {relation} WHERE {c} IS NOT NULL) USING SAMPLE reservoir({GEOMETRY_SAMPLE} ROWS)"
271
+ ).fetchall()
272
+ features = []
273
+ for (raw,) in sampled:
274
+ if not raw:
275
+ continue
276
+ shape = json.loads(raw)
277
+ parts = shape.get("geometries", []) if shape.get("type") == "GeometryCollection" else [shape]
278
+ features.extend({"type": "Feature", "geometry": part, "properties": {}} for part in parts)
279
+ return {
280
+ "column": name,
281
+ "count": int(total),
282
+ "bbox": None if xmin is None else [xmin, ymin, xmax, ymax],
283
+ "types": kinds,
284
+ "sample": {"type": "FeatureCollection", "features": features},
285
+ }
286
+
287
+
288
+ def _number(value: Any) -> float | None:
289
+ if value is None:
290
+ return None
291
+ number = float(value)
292
+ return None if math.isnan(number) or math.isinf(number) else number
293
+
294
+
295
+ def _dump(value: dict[str, Any]) -> bytes:
296
+ return json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode()
@@ -18,7 +18,7 @@ from dataclasses import dataclass
18
18
  from pathlib import Path
19
19
  from typing import Iterable, Iterator, Mapping
20
20
 
21
- from . import catalog, layout, objects, refs
21
+ from . import catalog, layout, objects, profiles, refs
22
22
  from . import dataset as dataset_json
23
23
  from .store import Store, is_conflict, is_missing
24
24
  from .version import Catalog, DuckLake, File, Version, VersionError, new_id, parse, utc_now
@@ -469,6 +469,58 @@ def update_refs_index(store: Store) -> None:
469
469
  warnings.warn(f"the ref moved, but refs.json was not rewritten: {exc}", stacklevel=2)
470
470
 
471
471
 
472
+ def write_profiles(
473
+ store: Store,
474
+ local: Path,
475
+ parent: Version | None,
476
+ working: Path,
477
+ *,
478
+ data_path: str,
479
+ setup: Iterable[str] = (),
480
+ ) -> int:
481
+ """Write a profile of each table of a committed version under ``local/profiles/``.
482
+
483
+ A table whose data files and columns are the parent's keeps the parent's
484
+ profile, so a commit reads only the tables that changed. Returns how many
485
+ were made from the data.
486
+ """
487
+ local = Path(local)
488
+ described_path = local / layout.DATASET
489
+ if not described_path.is_file():
490
+ return 0
491
+ described = json.loads(described_path.read_text(encoding="utf-8"))
492
+ files_now = json.loads((local / layout.FILES).read_text(encoding="utf-8")).get("tables", {})
493
+ files_before: dict = {}
494
+ if parent is not None and parent.files is not None:
495
+ try:
496
+ files_before = json.loads(store.get(layout.version_dir(parent.id) + parent.files.path)).get("tables", {})
497
+ except Exception as exc: # noqa: BLE001 - stores raise their own types
498
+ if not is_missing(exc):
499
+ raise
500
+ kept: dict[str, bytes] = {}
501
+ pending = []
502
+ for table in profiles.tables(described):
503
+ key = f"{table['schema']}.{table['name']}"
504
+ relative = profiles.path(table["schema"], table["name"])
505
+ previous = None
506
+ if parent is not None and key in files_before and files_before[key] == files_now.get(key):
507
+ try:
508
+ previous = store.get(layout.version_dir(parent.id) + relative)
509
+ except Exception as exc: # noqa: BLE001
510
+ if not is_missing(exc):
511
+ raise
512
+ if previous is not None and profiles.reusable(previous, table):
513
+ kept[relative] = previous
514
+ else:
515
+ pending.append(table)
516
+ made = profiles.build(working, pending, data_path=data_path, setup=setup)
517
+ for relative, content in {**kept, **made}.items():
518
+ target = local / relative
519
+ target.parent.mkdir(parents=True, exist_ok=True)
520
+ target.write_bytes(content)
521
+ return len(made)
522
+
523
+
472
524
  def upload(store: Store, local: Path, version: Version) -> None:
473
525
  """Put a committed version's files, catalog first and the record last.
474
526
 
@@ -483,6 +535,11 @@ def upload(store: Store, local: Path, version: Version) -> None:
483
535
  _put_once(store, layout.version_dir(version.id) + version.files.path, local / version.files.path)
484
536
  if version.tree is not None:
485
537
  _put_once(store, layout.version_dir(version.id) + version.tree.path, local / version.tree.path)
538
+ profiled = local / layout.PROFILES_DIR
539
+ if profiled.is_dir():
540
+ for written in sorted(profiled.rglob("*.json")):
541
+ relative = written.relative_to(local).as_posix()
542
+ _put_once(store, layout.version_dir(version.id) + relative, written)
486
543
  _put_once(store, layout.version_record(version.id), local / layout.VERSION_RECORD)
487
544
 
488
545
 
@@ -553,6 +610,7 @@ def expire(store: Store, version_id: str, *, reason: str = "retention") -> None:
553
610
  layout.version_dir(version_id) + layout.FILES,
554
611
  layout.version_dir(version_id) + layout.TREE,
555
612
  *store.list(layout.version_dir(version_id) + layout.CATALOGS_DIR),
613
+ *store.list(layout.version_dir(version_id) + layout.PROFILES_DIR),
556
614
  ]
557
615
  )
558
616
 
@@ -0,0 +1,131 @@
1
+ """Profiles: a summary of each table, made from the data at commit."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from pathlib import Path
7
+
8
+ import duckdb
9
+ import pytest
10
+ from typer.testing import CliRunner
11
+
12
+ from fdl import catalog, layout, profiles, repo
13
+ from fdl.cli import app
14
+ from fdl.stores import LocalStore
15
+
16
+ runner = CliRunner()
17
+
18
+
19
+ def run(*args) -> str:
20
+ result = runner.invoke(app, [str(a) for a in args])
21
+ assert result.exit_code == 0, result.output
22
+ return result.output
23
+
24
+
25
+ def write(working: Path, store: LocalStore, sql: str) -> None:
26
+ con = duckdb.connect()
27
+ for statement in catalog.attach_sql(working, alias="lake", data_path=repo.data_url(store)):
28
+ if statement:
29
+ con.execute(statement)
30
+ con.execute(sql)
31
+ con.close()
32
+
33
+
34
+ @pytest.fixture
35
+ def repository(tmp_path: Path, monkeypatch):
36
+ monkeypatch.chdir(tmp_path)
37
+ remote, working = tmp_path / "remote", tmp_path / "lake.sqlite"
38
+ run("init", remote)
39
+ run("checkout", remote, working)
40
+ (tmp_path / "dataset.yml").write_text("name: towns\ntitle: Towns\n")
41
+ return LocalStore(remote), working
42
+
43
+
44
+ def profile_of(store: LocalStore, table: str = "towns") -> dict:
45
+ head = repo.head(store)
46
+ return json.loads(store.get(f"{layout.version_dir(head.id)}profiles/main/{table}.json"))
47
+
48
+
49
+ def test_a_commit_profiles_each_table(repository) -> None:
50
+ store, working = repository
51
+ write(
52
+ working,
53
+ store,
54
+ "CREATE TABLE lake.main.towns AS SELECT i AS id, 'town ' || (i % 3) AS name, "
55
+ "DATE '2020-01-01' + i::INTEGER AS day, i % 2 = 0 AS even FROM range(20) t(i)",
56
+ )
57
+ run("commit", working, "-m", "m", "--author", "t")
58
+ got = profile_of(store)
59
+
60
+ assert got["rows"] == 20
61
+ assert len(got["head"]) == profiles.HEAD_ROWS
62
+ by_name = {c["name"]: c for c in got["columns"]}
63
+ assert by_name["id"]["min"] == 0 and by_name["id"]["max"] == 19
64
+ assert by_name["name"]["distinct_count"] == 3
65
+ assert {v["value"]: v["count"] for v in by_name["name"]["values"]} == {"town 0": 7, "town 1": 7, "town 2": 6}
66
+ assert by_name["day"]["min"] == "2020-01-01"
67
+ assert by_name["even"]["true_count"] == 10
68
+
69
+
70
+ def test_many_values_list_only_the_most_common(repository) -> None:
71
+ store, working = repository
72
+ write(working, store, f"CREATE TABLE lake.main.towns AS SELECT 'v' || i AS code FROM range({profiles.ALL_VALUES + 5}) t(i)")
73
+ run("commit", working, "-m", "m", "--author", "t")
74
+ code = profile_of(store)["columns"][0]
75
+ assert len(code["values"]) == profiles.TOP_VALUES
76
+ assert code["distinct_count"] > profiles.ALL_VALUES
77
+
78
+
79
+ def test_an_unchanged_table_keeps_its_parents_profile(repository, monkeypatch) -> None:
80
+ store, working = repository
81
+ write(working, store, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
82
+ write(working, store, "CREATE TABLE lake.main.other AS SELECT 1 AS id")
83
+ run("commit", working, "-m", "m", "--author", "t")
84
+ write(working, store, "INSERT INTO lake.main.other VALUES (2)")
85
+
86
+ built: list[str] = []
87
+ original = profiles.build
88
+
89
+ def spy(working, wanted, **kwargs):
90
+ wanted = list(wanted)
91
+ built.extend(t["name"] for t in wanted)
92
+ return original(working, wanted, **kwargs)
93
+
94
+ monkeypatch.setattr(profiles, "build", spy)
95
+ run("commit", working, "-m", "again", "--author", "t")
96
+ assert built == ["other"]
97
+ assert profile_of(store, "towns")["rows"] == 1
98
+ assert profile_of(store, "other")["rows"] == 2
99
+
100
+
101
+ def test_expiry_deletes_the_profiles(repository) -> None:
102
+ store, working = repository
103
+ write(working, store, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
104
+ run("commit", working, "-m", "m", "--author", "t")
105
+ first = repo.head(store).id
106
+ write(working, store, "INSERT INTO lake.main.towns VALUES (2)")
107
+ run("commit", working, "-m", "again", "--author", "t")
108
+ repo.expire(store, first)
109
+ assert store.list(f"{layout.version_dir(first)}profiles/") == []
110
+
111
+
112
+ def test_a_geometry_column_gets_its_extent_and_a_sample(repository) -> None:
113
+ store, working = repository
114
+ try:
115
+ duckdb.connect().execute("INSTALL spatial; LOAD spatial")
116
+ except duckdb.Error:
117
+ pytest.skip("no spatial extension")
118
+ write(
119
+ working,
120
+ store,
121
+ "INSTALL spatial; LOAD spatial; CREATE TABLE lake.main.towns AS "
122
+ "SELECT i AS id, ST_Point(139 + i, 35) AS geom FROM range(3) t(i)",
123
+ )
124
+ run("commit", working, "-m", "m", "--author", "t")
125
+ got = profile_of(store)
126
+ assert got["geometry"]["column"] == "geom"
127
+ assert got["geometry"]["count"] == 3
128
+ assert got["geometry"]["bbox"] == [139, 35, 141, 35]
129
+ assert got["geometry"]["types"] == ["POINT"]
130
+ assert len(got["geometry"]["sample"]["features"]) == 3
131
+ assert [r.keys() for r in got["head"]][0] == {"id"}
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes