fdl-cli 0.19.0__tar.gz → 0.20.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/CHANGELOG.md +23 -0
  2. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/PKG-INFO +1 -1
  3. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/reference/cli.md +3 -1
  4. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/spec/data-files.md +1 -1
  5. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/spec/index.md +1 -1
  6. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/spec/layout.md +2 -1
  7. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/spec/versions.md +45 -0
  8. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/spec/writing.md +2 -2
  9. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/reference/cli.md +3 -1
  10. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/spec/data-files.md +1 -1
  11. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/spec/index.md +1 -1
  12. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/spec/layout.md +2 -1
  13. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/spec/versions.md +45 -0
  14. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/spec/writing.md +2 -2
  15. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/pyproject.toml +1 -1
  16. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/src/fdl/cli.py +20 -4
  17. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/src/fdl/layout.py +2 -0
  18. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/src/fdl/objects.py +12 -2
  19. fdl_cli-0.20.1/src/fdl/profiles.py +296 -0
  20. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/src/fdl/repo.py +67 -1
  21. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/tests/test_http.py +2 -1
  22. fdl_cli-0.20.1/tests/test_profiles.py +131 -0
  23. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/tests/test_repo.py +17 -0
  24. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/.github/workflows/docs.yml +0 -0
  25. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/.github/workflows/publish.yml +0 -0
  26. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/.github/workflows/release.yml +0 -0
  27. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/.gitignore +0 -0
  28. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/.node-version +0 -0
  29. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/.python-version +0 -0
  30. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/CLAUDE.md +0 -0
  31. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/CONTRIBUTING.md +0 -0
  32. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/LICENSE +0 -0
  33. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/README.md +0 -0
  34. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/blume.config.ts +0 -0
  35. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/concepts/meta.ts +0 -0
  36. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/concepts/repositories.md +0 -0
  37. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/concepts/versions-and-refs.md +0 -0
  38. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/concepts/working-catalogs.md +0 -0
  39. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/getting-started.md +0 -0
  40. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/guides/building-a-host.md +0 -0
  41. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/guides/meta.ts +0 -0
  42. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/guides/mirrors.md +0 -0
  43. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/guides/public-and-private.md +0 -0
  44. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/guides/reading-without-duckdb.md +0 -0
  45. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/guides/serving-over-http.md +0 -0
  46. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/guides/stores.md +0 -0
  47. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/index.md +0 -0
  48. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/internals/ducklake-formats.md +0 -0
  49. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/internals/from-write-to-read.md +0 -0
  50. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/internals/inside-a-repository.md +0 -0
  51. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/internals/meta.ts +0 -0
  52. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/concepts/meta.ts +0 -0
  53. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/concepts/repositories.md +0 -0
  54. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/concepts/versions-and-refs.md +0 -0
  55. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/concepts/working-catalogs.md +0 -0
  56. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/getting-started.md +0 -0
  57. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/guides/building-a-host.md +0 -0
  58. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/guides/meta.ts +0 -0
  59. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/guides/mirrors.md +0 -0
  60. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/guides/public-and-private.md +0 -0
  61. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/guides/reading-without-duckdb.md +0 -0
  62. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/guides/serving-over-http.md +0 -0
  63. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/guides/stores.md +0 -0
  64. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/index.md +0 -0
  65. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/internals/ducklake-formats.md +0 -0
  66. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/internals/from-write-to-read.md +0 -0
  67. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/internals/inside-a-repository.md +0 -0
  68. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/internals/meta.ts +0 -0
  69. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/reference/meta.ts +0 -0
  70. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/reference/python.md +0 -0
  71. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/spec/http.md +0 -0
  72. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/spec/meta.ts +0 -0
  73. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/spec/reading.md +0 -0
  74. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/spec/refs.md +0 -0
  75. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/reference/meta.ts +0 -0
  76. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/reference/python.md +0 -0
  77. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/spec/http.md +0 -0
  78. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/spec/meta.ts +0 -0
  79. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/spec/reading.md +0 -0
  80. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/spec/refs.md +0 -0
  81. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/package-lock.json +0 -0
  82. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/package.json +0 -0
  83. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/public/CNAME +0 -0
  84. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/src/fdl/__init__.py +0 -0
  85. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/src/fdl/catalog.py +0 -0
  86. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/src/fdl/dataset.py +0 -0
  87. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/src/fdl/py.typed +0 -0
  88. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/src/fdl/refs.py +0 -0
  89. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/src/fdl/store.py +0 -0
  90. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/src/fdl/stores.py +0 -0
  91. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/src/fdl/version.py +0 -0
  92. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/tests/http_repo.py +0 -0
  93. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/tests/test_cli.py +0 -0
  94. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/tests/test_dataset.py +0 -0
  95. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/tests/test_objects.py +0 -0
  96. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/tests/test_stores.py +0 -0
  97. {fdl_cli-0.19.0 → fdl_cli-0.20.1}/uv.lock +0 -0
@@ -1,6 +1,29 @@
1
1
  # CHANGELOG
2
2
 
3
3
 
4
+ ## v0.20.1 (2026-11-06)
5
+
6
+ ### Bug Fixes
7
+
8
+ - Withdraw a version already past retention by deleting unread files at once
9
+ ([#40](https://github.com/queria-io/fdl/pull/40),
10
+ [`d9e3788`](https://github.com/queria-io/fdl/commit/d9e3788b6ffb429a9d301491a5e2a73eca8d891f))
11
+
12
+
13
+ ## v0.20.0 (2026-11-06)
14
+
15
+ ### Documentation
16
+
17
+ - Drop the note that the draft spec changes as tools are built
18
+ ([#36](https://github.com/queria-io/fdl/pull/36),
19
+ [`de531b8`](https://github.com/queria-io/fdl/commit/de531b806a3b1d65fe4e5347ce203d22573981d6))
20
+
21
+ ### Features
22
+
23
+ - Write a profile of each table beside a version ([#39](https://github.com/queria-io/fdl/pull/39),
24
+ [`bdabd3f`](https://github.com/queria-io/fdl/commit/bdabd3fcd3f0b777609de78691b8ad64a1c40c26))
25
+
26
+
4
27
  ## v0.19.0 (2026-11-06)
5
28
 
6
29
  ### Features
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: fdl-cli
3
- Version: 0.19.0
3
+ Version: 0.20.1
4
4
  Summary: Versions of a DuckLake catalog on object storage, kept the way Git keeps commits
5
5
  Project-URL: Homepage, https://fdl.flo8s.com/
6
6
  Project-URL: Documentation, https://fdl.flo8s.com/
@@ -53,6 +53,8 @@ fdl commit <working.sqlite> -m "What changed" --author alice [--dataset dataset.
53
53
 
54
54
  コマンドは、説明の隣の `README.md` と、そこにある `docs/` より下のファイル(隠しファイルを除く)をバージョンの tree に入れます。そのフォルダからのパスで `tree.json` を書き、リポジトリに無いものを書いたままの中身で `assets/<sha256>` にアップロードします([tree.json](/ja/spec/versions#treejson))。
55
55
 
56
+ あわせてテーブルごとの profile を書きます([profiles/](/ja/spec/versions#profiles))。読むのは、データファイルか列が親から変わったテーブルだけです。読めなかったテーブルには profile を書かず、警告を出してコミットを続けます。
57
+
56
58
  `asset: true` の印を付けた列は、ファイルの sha256 を持ちます([アセット](/ja/spec/versions#アセット))。コマンドはその列の値を読み、リポジトリに無いファイルを `--assets` のフォルダから `assets/<sha256>` にアップロードします。ファイルは中身で照合するので、フォルダの中のファイル名は問いません。
57
59
 
58
60
  親と同じ中身になるとき([変更がないとき](/ja/spec/writing#変更がないとき))は `Nothing to commit` と表示し、ブランチを動かさずに終了コード 0 で終わります。同じ行を新しいファイルに書き直しただけの作り直しも、変更なしとみなします。`--allow-empty` を付けると、それでもバージョンを作ります。行を読めず判定できないときは、警告を出してバージョンを作ります。
@@ -106,7 +108,7 @@ fdl tag <remote> <name> --delete
106
108
  fdl withdraw <remote> <version>... [--yes]
107
109
  ```
108
110
 
109
- 間違って公開した個人データなど、公開してはいけないデータを持つバージョンを取り下げます([取り下げ](/ja/spec/data-files#取り下げ))。先にそのデータを除いたバージョンをコミットし、該当するバージョンを指すブランチが無い状態にします。そのうえでこのコマンドが、該当するバージョンを指すタグ、そのカタログが参照し残すバージョンが読まないデータファイル、カタログを消し、`"reason": "withdrawn"` の `expired.json` を置きます。`version.json` は残します。
111
+ 間違って公開した個人データなど、公開してはいけないデータを持つバージョンを取り下げます([取り下げ](/ja/spec/data-files#取り下げ))。先にそのデータを除いたバージョンをコミットし、該当するバージョンを指すブランチが無い状態にします。そのうえでこのコマンドが、該当するバージョンを指すタグ、そのカタログが参照し残すバージョンが読まないデータファイル、カタログを消し、`"reason": "withdrawn"` の `expired.json` を置きます。`version.json` は残します。保持期間を過ぎたバージョンを含めると、そのカタログはもう無いので、有効なバージョンのどれも読まないデータファイルを、猶予期間を待たずにすべて消します。
110
112
 
111
113
  アセットは、まだ消しません。データを持つものは `assets/` から手で消してください。
112
114
 
@@ -55,7 +55,7 @@ DuckLake 自身の片付け(`ducklake_cleanup_old_files`、`ducklake_expire_sn
55
55
 
56
56
  1. そのデータを除いた新しいバージョンを作り、該当するバージョンを指しているブランチをすべてそこへ動かす。新しいバージョンの親はいまのバージョンなので、前へ動かすことになる
57
57
  2. 該当するバージョンを指すタグを消す
58
- 3. 個人情報を持つデータファイルや、それが写ったスクリーンショットなど、そのデータを持つ共有オブジェクトをすぐに消す。該当するバージョンから到達し、残すバージョンのどれからも到達しないものが対象。そのオブジェクトに到達するバージョンは、すべて該当するバージョンになる
58
+ 3. 個人情報を持つデータファイルや、それが写ったスクリーンショットなど、そのデータを持つ共有オブジェクトをすぐに消す。該当するバージョンから到達し、残すバージョンのどれからも到達しないものが対象。そのオブジェクトに到達するバージョンは、すべて該当するバージョンになる。該当するバージョンに保持期間を過ぎたものがあると、カタログが無いので、それがどのオブジェクトに到達していたかが分からない。その場合は、有効なバージョンのどれからも到達しない共有オブジェクトもすべてすぐに消す
59
59
  4. 該当するバージョンそれぞれに `"reason": "withdrawn"` の `expired.json` を置き、カタログを消す
60
60
 
61
61
  共有オブジェクトをカタログより先に消すのは、途中で止まった取り下げをやり直せるようにするためです。カタログが残っていれば、どのファイルがそのデータを持つかが分かります。
@@ -5,7 +5,7 @@ sidebar:
5
5
  description: FDL 0.1(草案)の規範となる説明。何を決め、何をホストに任せ、どんな用語を使うか。
6
6
  ---
7
7
 
8
- FDL 0.1 の仕様です。草案で、実装を作りながら変えていきます。ツールはこの仕様に合わせて作り、ツールと仕様が食い違ったときは、先に仕様を直してからツールを合わせます。この日本語訳は参考で、英語版を正とします。
8
+ FDL 0.1 の仕様の草案です。ツールはこの仕様に合わせて作り、ツールと仕様が食い違ったときは、先に仕様を直してからツールを合わせます。この日本語訳は参考で、英語版を正とします。
9
9
 
10
10
  FDL は、[DuckLake](https://ducklake.select/) のカタログのバージョンを残すための、置き場のファイル配置です。Git がコミットを保存する形にならっています。どのファイルを置き場のどこに置き、読み手と書き手がそれをどう扱うかを決めます。サーバーは要りません。置き場に求めるのは、読む・書く・条件付きで書くの 3 つです。
11
11
 
@@ -23,9 +23,10 @@ description: リポジトリのファイル、そのパス、fdl.json の中身
23
23
  dataset.json バージョンの中身の説明
24
24
  files.json カタログから作った、表ごとのデータファイルの索引(任意)
25
25
  tree.json README など、バージョンのほかのファイルのパスの一覧(任意)
26
+ profiles/<schema>/<table>.json データから測ったテーブルごとの要約(任意)
26
27
  catalogs/<format>.duckdb 別の DuckLake 形式のカタログ(任意)
27
28
  expired.json 保持期間を過ぎた印(過ぎたときだけ)
28
- <others> ホストが足すファイル(プレビューなど)
29
+ <others> ホストが足すファイル
29
30
  <data_dir><schema>/<table>/ducklake-<uuid>.parquet データファイル(共有、変えない)
30
31
  assets/<sha256> テーブルの行やバージョンの tree.json が指すファイル(共有、変更しない)
31
32
  ```
@@ -154,6 +154,51 @@ DuckLake を使わずに Parquet ファイルを読む読み手は、`deletes`
154
154
  - 書き手は、ファイルを編集する場所(Git のリポジトリの中の YAML の隣など)に置き、書き手のツールが `tree.json` を書く。書き手が手で書くものではない
155
155
  - `tree.json` はバージョンの一部で、そのバージョンの中では変わらない。保持期間から外すときはカタログと一緒に消す
156
156
 
157
+ ## profiles/
158
+
159
+ profile は、`ducklake.snapshot_id` の時点の 1 つのテーブルの要約です。行数、列ごとの中身、先頭の行、図形の列を持つテーブルでは図形の範囲と見本を持ちます。読み手はこの小さなファイル 1 つで、カタログを開かずデータも読まずにテーブルを表示できます。ホストは検索の索引をこれから作れます。profile が持つのはデータから測った事実で、列が何を意味するかは書き手が `dataset.json` に書きます。`files.json` と同じくカタログとデータから作るもので、食い違うときはカタログとデータが正です。任意で、読み手は profile が無くても読めなければなりません。
160
+
161
+ テーブルごとの profile は `versions/<ID>/profiles/<schema>/<table>.json` に置きます。スキーマ名かテーブル名が空・`.`・`..` のとき、または `/` を含むときは置きません。ビューには置きません。
162
+
163
+ ```json profiles/main/towns.json
164
+ {
165
+ "rows": 1741,
166
+ "columns": [
167
+ { "name": "town_code", "type": "varchar", "null_count": 0, "distinct_count": 1741,
168
+ "values": [{ "value": "082201", "count": 1 }] },
169
+ { "name": "prefecture", "type": "varchar", "null_count": 0, "distinct_count": 2,
170
+ "values": [{ "value": "茨城県", "count": 1200 }, { "value": "栃木県", "count": 541 }] },
171
+ { "name": "population", "type": "int64", "null_count": 3, "distinct_count": 1650,
172
+ "min": 0, "max": 84211, "mean": 3120.5, "std": 4410.2, "q1": 420, "median": 1630, "q3": 3990,
173
+ "values": [{ "value": "0", "count": 12 }],
174
+ "histogram": [{ "start": 0, "end": 4210.55, "count": 1302 }] },
175
+ { "name": "surveyed_on", "type": "date", "null_count": 0, "distinct_count": 1,
176
+ "min": "2025-10-01", "max": "2025-10-01", "span_days": 0,
177
+ "values": [{ "value": "2025-10-01", "count": 1741 }] },
178
+ { "name": "coastal", "type": "boolean", "null_count": 0, "true_count": 210, "false_count": 1531 },
179
+ { "name": "geom", "type": "geometry" }
180
+ ],
181
+ "head": [{ "town_code": "082201", "prefecture": "茨城県", "population": 2210, "surveyed_on": "2025-10-01", "coastal": false }],
182
+ "geometry": {
183
+ "column": "geom",
184
+ "count": 1741,
185
+ "bbox": [139.69, 35.74, 140.85, 37.15],
186
+ "types": ["MULTIPOLYGON", "POLYGON"],
187
+ "sample": { "type": "FeatureCollection", "features": [] }
188
+ }
189
+ }
190
+ ```
191
+
192
+ - `rows` はテーブルの行数。`columns` にはカタログの順にすべての列を並べ、`name` と `type`(DuckLake の型名)は `dataset.json` と同じにする
193
+ - 図形の列を除くすべての列が `null_count` を持つ。数値・テキスト・日付と時刻の列は `distinct_count` と `values` を持つ。大きなテーブルの `distinct_count` は推定値のことがある
194
+ - `values` は、値(テキスト)とその値を持つ行の数を、多い順、同数なら値の順に並べる。列が持つ異なる値が 1,000 以下なら全部を並べ、そのとき `distinct_count` は正確な数になる。読み手は、`values` の長さが `distinct_count` と等しいことで全部の一覧だと判定する。それより多い列は上位 10 件だけを並べる。ホストは全部の一覧を、地域コードや名前などの値で検索できる
195
+ - 数値の列は `min`・`max`・`mean`・`std`、四分位数の `q1`・`median`・`q3`、`min` から `max` までを等幅に区切った `histogram` を持つ。日付と時刻の列は `min` と `max` をテキストで持ち、日付とタイムスタンプは `span_days` も持つ。真偽値の列は `true_count` と `false_count` を持つ
196
+ - `head` は、テーブルが返す先頭の行を JSON のオブジェクトで持つ。図形の列と、値を JSON で表せない列は入れない
197
+ - `geometry` は、書き手が読めたときに、最初の図形の列を説明する。図形を持つ行の数、全体の `bbox`(列の座標で `[xmin, ymin, xmax, ymax]`)、含まれる図形の種類、地図を描くための、無作為に選んで簡略化した図形の GeoJSON `FeatureCollection` を持つ
198
+ - 書き手は計算できないキーを省いてよい。読み手は知らないキーを無視する
199
+ - 書き手は、データファイル(`files.json`)と列が親と同じテーブルには親の profile を使い回す。コミットで読むのは変わったテーブルだけになる
200
+ - profile はバージョンの一部で、そのバージョンの中では変わらない。保持期間から外すときはカタログと一緒に消す
201
+
157
202
  ## 固めたカタログ
158
203
 
159
204
  `ducklake.duckdb` は、DuckDB のファイル形式の DuckLake カタログです。
@@ -17,7 +17,7 @@ description: チェックアウト、コミット、push。作業カタログ、
17
17
  | --- | --- |
18
18
  | チェックアウト | ブランチのバージョンのカタログから作業カタログを作り、そのバージョンを覚える。最初のバージョンの前は空のものを作る |
19
19
  | コミット | 作業カタログの最後のスナップショットを新しいバージョンに固める。親は覚えておいたバージョン。中身を `dataset.json` で説明する |
20
- | push | データファイルを確かめ、カタログ、`dataset.json`・`files.json`・`tree.json`、`version.json` の順にアップロードし、条件付きで ref を動かす |
20
+ | push | データファイルを確かめ、カタログ、`dataset.json`・`files.json`・`tree.json`・profile、`version.json` の順にアップロードし、条件付きで ref を動かす |
21
21
 
22
22
  チェックアウトでは、DuckLake の管理用の表を、固めたカタログから SQLite へそのまま写します。固めるときはその逆で、最後のスナップショットまでの行を残します。どちらもデータファイルは書き換えません。
23
23
 
@@ -40,7 +40,7 @@ description: チェックアウト、コミット、push。作業カタログ、
40
40
 
41
41
  1. データファイル・アセット
42
42
  2. `versions/<ID>/ducklake.duckdb`(と `catalogs/`)
43
- 3. `versions/<ID>/dataset.json`・`versions/<ID>/files.json`・`versions/<ID>/tree.json`
43
+ 3. `versions/<ID>/dataset.json`・`versions/<ID>/files.json`・`versions/<ID>/tree.json`・`versions/<ID>/profiles/`
44
44
  4. `versions/<ID>/version.json`
45
45
  5. ref(条件付き)
46
46
 
@@ -53,6 +53,8 @@ Every version is described by a `dataset.json` (see [Versions](/spec/versions#da
53
53
 
54
54
  The command puts `README.md` beside the description and every file below `docs/` there (hidden ones left out) in the version's tree: it writes `tree.json` with their paths from that folder and uploads those the repository lacks to `assets/<sha256>`, as written (see [tree.json](/spec/versions#treejson)).
55
55
 
56
+ It also writes a profile of each table (see [profiles/](/spec/versions#profiles)), reading only the tables whose data files or columns changed since the parent. A table it cannot read gets none, and the commit goes on with a warning.
57
+
56
58
  Columns marked `asset: true` hold the sha256 of files (see [Assets](/spec/versions#assets)). The command reads the values those columns hold and uploads the files the repository lacks from the `--assets` folder to `assets/<sha256>`, by their content: file names in the folder do not matter.
57
59
 
58
60
  When the version would hold what its parent holds (see [When nothing changed](/spec/writing#when-nothing-changed)), it prints `Nothing to commit`, leaves the branch where it is and exits 0. A rebuild that writes the same rows to new files counts as unchanged. `--allow-empty` makes the version anyway. If the rows cannot be read to tell, it warns and makes the version.
@@ -106,7 +108,7 @@ fdl tag <remote> <name> --delete
106
108
  fdl withdraw <remote> <version>... [--yes]
107
109
  ```
108
110
 
109
- Removes versions that hold data that must not be published, such as personal data published by mistake (see [Withdrawal](/spec/data-files#withdrawal)). First commit a version without the data, so that no branch points at an affected version. The command then deletes the tags that point at the versions, the data files their catalogs name that no kept version reads, and their catalogs, and puts `expired.json` with `"reason": "withdrawn"`. `version.json` stays.
111
+ Removes versions that hold data that must not be published, such as personal data published by mistake (see [Withdrawal](/spec/data-files#withdrawal)). First commit a version without the data, so that no branch points at an affected version. The command then deletes the tags that point at the versions, the data files their catalogs name that no kept version reads, and their catalogs, and puts `expired.json` with `"reason": "withdrawn"`. `version.json` stays. When one of the versions is already past retention, its catalog is gone, so every data file no live version reads is deleted at once, without waiting for the grace period.
110
112
 
111
113
  Assets are not deleted yet: delete those that hold the data from `assets/` by hand.
112
114
 
@@ -55,7 +55,7 @@ Withdrawal removes data that must not be published, such as personal data publis
55
55
 
56
56
  1. Make a new version without the data and move every branch that holds an affected version to it. This is a forward move: the new version's parent is the current one
57
57
  2. Delete the tags that point at affected versions
58
- 3. Delete the shared objects that hold the data, at once, such as a data file with personal data or a screenshot showing it: those an affected version reaches and no version kept reaches. Every version that reaches such an object is an affected version
58
+ 3. Delete the shared objects that hold the data, at once, such as a data file with personal data or a screenshot showing it: those an affected version reaches and no version kept reaches. Every version that reaches such an object is an affected version. An affected version already past retention has no catalog to say which objects it reached, so when there is one, every shared object no live version reaches is deleted at once too
59
59
  4. For every affected version, put `expired.json` with `"reason": "withdrawn"` and delete its catalogs
60
60
 
61
61
  The shared objects go before the catalogs, so a withdrawal that stops half way can be run again: the catalogs still say which files hold the data.
@@ -5,7 +5,7 @@ sidebar:
5
5
  description: The normative description of FDL 0.1 (draft) - what it decides, what it leaves to hosts, and the terms it uses.
6
6
  ---
7
7
 
8
- This is the specification of FDL 0.1, a draft. It changes as the tools that implement it are built. Tools are written against this specification, and where a tool and this text disagree, the text is changed first and the tool follows.
8
+ This is the specification of FDL 0.1, a draft. Tools are written against this specification, and where a tool and this text disagree, the text is changed first and the tool follows.
9
9
 
10
10
  FDL is a storage layout for keeping versions of a [DuckLake](https://ducklake.select/) catalog, modelled on how Git stores commits. It says which files go where in a store and how readers and writers handle them. No server is needed: the store has to read, write and write on condition.
11
11
 
@@ -23,9 +23,10 @@ Paths are relative to the root, lower case, and separated by `/`. The root is wh
23
23
  dataset.json what the version holds, described
24
24
  files.json each table's data files, indexed from the catalog (optional)
25
25
  tree.json the other files of the version, such as a README, by path (optional)
26
+ profiles/<schema>/<table>.json a summary of each table, measured from the data (optional)
26
27
  catalogs/<format>.duckdb the catalog in another DuckLake format (optional)
27
28
  expired.json the past-retention mark (only once expired)
28
- <others> files a host adds, such as previews
29
+ <others> files a host adds
29
30
  <data_dir><schema>/<table>/ducklake-<uuid>.parquet data files (shared, immutable)
30
31
  assets/<sha256> files the rows of a table or a version's tree.json point at (shared, immutable)
31
32
  ```
@@ -154,6 +154,51 @@ When a version is taken out of retention, `files.json` is deleted with its catal
154
154
  - A writer keeps the files where it edits them, such as beside its YAML in a Git repository, and its tool writes `tree.json`. The writer does not write it by hand
155
155
  - `tree.json` is part of the version and does not change with it. It is deleted with the catalogs when the version is taken out of retention
156
156
 
157
+ ## profiles/
158
+
159
+ A profile summarizes one table at `ducklake.snapshot_id`: its row count, what each column holds, its first rows and, for a table with a geometry column, the extent and a sample of its shapes. A reader can show a table, or a host can index it for search, from this one small file, without opening the catalog or reading the data. It holds what was measured from the data; what a column means is the writer's, in `dataset.json`. Like `files.json`, it is made from the catalog and the data, which win where the two differ, and it is optional: a reader must work without it.
160
+
161
+ Each table's profile is at `versions/<ID>/profiles/<schema>/<table>.json`. A table whose schema or name is empty, `.`, `..` or holds `/` has none. Views have none.
162
+
163
+ ```json profiles/main/towns.json
164
+ {
165
+ "rows": 1741,
166
+ "columns": [
167
+ { "name": "town_code", "type": "varchar", "null_count": 0, "distinct_count": 1741,
168
+ "values": [{ "value": "082201", "count": 1 }] },
169
+ { "name": "prefecture", "type": "varchar", "null_count": 0, "distinct_count": 2,
170
+ "values": [{ "value": "Ibaraki", "count": 1200 }, { "value": "Tochigi", "count": 541 }] },
171
+ { "name": "population", "type": "int64", "null_count": 3, "distinct_count": 1650,
172
+ "min": 0, "max": 84211, "mean": 3120.5, "std": 4410.2, "q1": 420, "median": 1630, "q3": 3990,
173
+ "values": [{ "value": "0", "count": 12 }],
174
+ "histogram": [{ "start": 0, "end": 4210.55, "count": 1302 }] },
175
+ { "name": "surveyed_on", "type": "date", "null_count": 0, "distinct_count": 1,
176
+ "min": "2025-10-01", "max": "2025-10-01", "span_days": 0,
177
+ "values": [{ "value": "2025-10-01", "count": 1741 }] },
178
+ { "name": "coastal", "type": "boolean", "null_count": 0, "true_count": 210, "false_count": 1531 },
179
+ { "name": "geom", "type": "geometry" }
180
+ ],
181
+ "head": [{ "town_code": "082201", "prefecture": "Ibaraki", "population": 2210, "surveyed_on": "2025-10-01", "coastal": false }],
182
+ "geometry": {
183
+ "column": "geom",
184
+ "count": 1741,
185
+ "bbox": [139.69, 35.74, 140.85, 37.15],
186
+ "types": ["MULTIPOLYGON", "POLYGON"],
187
+ "sample": { "type": "FeatureCollection", "features": [] }
188
+ }
189
+ }
190
+ ```
191
+
192
+ - `rows` is the table's row count. `columns` lists every column in the catalog's order, with `name` and `type` (DuckLake's type name) as in `dataset.json`
193
+ - Every column but a geometry one has `null_count`. Numbers, text, dates and times have `distinct_count`, which may be estimated on a large table, and `values`
194
+ - `values` lists each value, as text, with the number of rows that hold it, most common first and then by value. When the column holds at most 1,000 distinct values, it lists them all and `distinct_count` is exact: a reader tells a whole list by its length matching `distinct_count`. Otherwise it lists the 10 most common. A host can search a whole list by value, such as a town's code or name
195
+ - Numbers have `min`, `max`, `mean`, `std`, the quartiles `q1`, `median` and `q3`, and a `histogram` of equal-width buckets between `min` and `max`. Dates and times have `min` and `max` as text, and dates and timestamps `span_days`. Booleans have `true_count` and `false_count`
196
+ - `head` holds the first rows the table returns, as JSON objects, leaving out geometry and columns whose values have no JSON form
197
+ - `geometry` describes the first geometry column, when the writer could read it: how many rows have a shape, the `bbox` of all of them (`[xmin, ymin, xmax, ymax]` in the column's coordinates), the geometry types present, and a GeoJSON `FeatureCollection` of a random sample of simplified shapes for drawing a map
198
+ - A writer may leave out any key it cannot compute. A reader ignores keys it does not know
199
+ - A writer reuses its parent's profile for a table whose data files (in `files.json`) and columns are the parent's, so a commit reads only the tables that changed
200
+ - Profiles are part of the version and do not change with it. They are deleted with the catalogs when the version is taken out of retention
201
+
157
202
  ## The frozen catalog
158
203
 
159
204
  `ducklake.duckdb` is a DuckLake catalog in DuckDB's file format.
@@ -17,7 +17,7 @@ A writer writes to a working catalog: a DuckLake catalog in SQLite, outside the
17
17
  | --- | --- |
18
18
  | checkout | Make a working catalog from the catalog of the branch's version, and remember that version. Before the first version, make an empty one |
19
19
  | commit | Freeze the working catalog's last snapshot into a new version, with the remembered version as its parent, and describe it in `dataset.json` |
20
- | push | Check the data files, upload the catalog, `dataset.json`, `files.json` and `tree.json`, then `version.json`, then move the ref on condition |
20
+ | push | Check the data files, upload the catalog, `dataset.json`, `files.json`, `tree.json` and the profiles, then `version.json`, then move the ref on condition |
21
21
 
22
22
  Checking out copies DuckLake's bookkeeping tables from the frozen catalog into SQLite as they are. Freezing does the reverse, keeping the rows up to the last snapshot. Neither rewrites a data file.
23
23
 
@@ -40,7 +40,7 @@ A table rebuilt from the same input is written to new data files, so comparing f
40
40
 
41
41
  1. Data files and assets
42
42
  2. `versions/<ID>/ducklake.duckdb` (and any `catalogs/`)
43
- 3. `versions/<ID>/dataset.json`, `versions/<ID>/files.json` and `versions/<ID>/tree.json`
43
+ 3. `versions/<ID>/dataset.json`, `versions/<ID>/files.json`, `versions/<ID>/tree.json` and `versions/<ID>/profiles/`
44
44
  4. `versions/<ID>/version.json`
45
45
  5. The ref, on condition
46
46
 
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "fdl-cli"
3
- version = "0.19.0"
3
+ version = "0.20.1"
4
4
  description = "Versions of a DuckLake catalog on object storage, kept the way Git keeps commits"
5
5
  readme = "README.md"
6
6
  license = "MIT"
@@ -148,12 +148,16 @@ def commit(
148
148
  typer.echo(f"warning: could not tell whether anything changed ({exc}); committing", err=True)
149
149
  version = make(True)
150
150
  local = Path(state["data"]) if state.get("data") else None
151
+ reading = Path(state["data"]).resolve().as_posix() + "/" if local else repo.data_url(store)
152
+ setup = _read_setup(store)
151
153
  described = json.loads((out / version.id / layout.DATASET).read_text(encoding="utf-8"))
152
154
  held = objects.asset_values(
153
- working,
154
- objects.asset_columns(described),
155
- data_path=Path(state["data"]).resolve().as_posix() + "/" if local else repo.data_url(store),
155
+ working, objects.asset_columns(described), data_path=reading, setup=setup
156
156
  )
157
+ try:
158
+ repo.write_profiles(store, out / version.id, parent, working, data_path=reading, setup=setup)
159
+ except Exception as exc: # noqa: BLE001 - a profile is never worth a failed commit
160
+ typer.echo(f"warning: tables were not profiled ({exc})", err=True)
157
161
  local_objects = objects.ByContent(
158
162
  {item["sha256"]: root / item["path"] for item in listed},
159
163
  assets_dir,
@@ -180,6 +184,16 @@ def commit(
180
184
  typer.echo(f"{state['ref']} -> {version.id} (version {version.seq})")
181
185
 
182
186
 
187
+ def _read_setup(store) -> list[str]:
188
+ """What DuckDB needs to read the repository's data files: the token, over HTTP."""
189
+ token = getattr(store, "token", None)
190
+ if not token:
191
+ return []
192
+ scope = store.url.replace("'", "''")
193
+ secret = token.replace("'", "''")
194
+ return [f"CREATE SECRET fdl_remote (TYPE http, BEARER_TOKEN '{secret}', SCOPE '{scope}')"]
195
+
196
+
183
197
  @app.command()
184
198
  def assets(folder: Path = typer.Argument(..., help="The folder of files the rows will point at")) -> None:
185
199
  """List the files below a folder as CSV: path, sha256, bytes, media_type.
@@ -281,7 +295,9 @@ def withdraw(
281
295
 
282
296
  Move every branch to a version without the data first. This deletes the
283
297
  tags that point at the versions, the data files no kept version reads,
284
- and the versions' catalogs, at once. Their version.json stays.
298
+ and the versions' catalogs, at once. Their version.json stays. A version
299
+ already past retention has no catalog, so then every data file no live
300
+ version reads is deleted.
285
301
  """
286
302
  store = open_store(remote)
287
303
  try:
@@ -29,6 +29,8 @@ EXPIRED = "expired.json"
29
29
  CATALOGS_DIR = "catalogs/"
30
30
  #: The other files of a version (its README, say), by path to the asset that holds each (optional).
31
31
  TREE = "tree.json"
32
+ #: A summary of each table, made from the data, under the version (optional).
33
+ PROFILES_DIR = "profiles/"
32
34
  #: Files the rows of a table point at, by sha256 (shared).
33
35
  ASSETS_DIR = "assets/"
34
36
 
@@ -174,9 +174,17 @@ def asset_columns(described: dict[str, Any]) -> list[tuple[str, str, str]]:
174
174
 
175
175
 
176
176
  def asset_values(
177
- working: Path, columns: Iterable[tuple[str, str, str]], *, data_path: str
177
+ working: Path,
178
+ columns: Iterable[tuple[str, str, str]],
179
+ *,
180
+ data_path: str,
181
+ setup: Iterable[str] = (),
178
182
  ) -> set[str]:
179
- """The sha256 the marked columns hold in the working catalog's last snapshot."""
183
+ """The sha256 the marked columns hold in the working catalog's last snapshot.
184
+
185
+ ``setup`` runs before the catalog is attached: a ``CREATE SECRET`` for data
186
+ files in private storage, say.
187
+ """
180
188
  import duckdb
181
189
 
182
190
  columns = list(columns)
@@ -189,6 +197,8 @@ def asset_values(
189
197
  found: set[str] = set()
190
198
  con = duckdb.connect()
191
199
  try:
200
+ for statement in setup:
201
+ con.execute(statement)
192
202
  for statement in catalog.attach_sql(Path(working), alias="lake", data_path=data_path, read_only=True):
193
203
  if statement:
194
204
  con.execute(statement)