fdl-cli 0.20.0__tar.gz → 0.21.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/CHANGELOG.md +28 -0
  2. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/PKG-INFO +1 -1
  3. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/reference/cli.md +3 -3
  4. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/spec/data-files.md +1 -1
  5. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/spec/versions.md +2 -2
  6. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/reference/cli.md +3 -3
  7. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/spec/data-files.md +1 -1
  8. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/spec/versions.md +2 -2
  9. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/pyproject.toml +1 -1
  10. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/cli.py +6 -4
  11. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/profiles.py +21 -4
  12. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/repo.py +78 -15
  13. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/tests/test_http.py +2 -1
  14. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/tests/test_objects.py +34 -0
  15. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/tests/test_profiles.py +27 -0
  16. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/tests/test_repo.py +17 -0
  17. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/.github/workflows/docs.yml +0 -0
  18. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/.github/workflows/publish.yml +0 -0
  19. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/.github/workflows/release.yml +0 -0
  20. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/.gitignore +0 -0
  21. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/.node-version +0 -0
  22. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/.python-version +0 -0
  23. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/CLAUDE.md +0 -0
  24. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/CONTRIBUTING.md +0 -0
  25. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/LICENSE +0 -0
  26. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/README.md +0 -0
  27. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/blume.config.ts +0 -0
  28. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/concepts/meta.ts +0 -0
  29. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/concepts/repositories.md +0 -0
  30. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/concepts/versions-and-refs.md +0 -0
  31. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/concepts/working-catalogs.md +0 -0
  32. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/getting-started.md +0 -0
  33. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/guides/building-a-host.md +0 -0
  34. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/guides/meta.ts +0 -0
  35. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/guides/mirrors.md +0 -0
  36. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/guides/public-and-private.md +0 -0
  37. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/guides/reading-without-duckdb.md +0 -0
  38. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/guides/serving-over-http.md +0 -0
  39. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/guides/stores.md +0 -0
  40. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/index.md +0 -0
  41. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/internals/ducklake-formats.md +0 -0
  42. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/internals/from-write-to-read.md +0 -0
  43. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/internals/inside-a-repository.md +0 -0
  44. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/internals/meta.ts +0 -0
  45. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/concepts/meta.ts +0 -0
  46. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/concepts/repositories.md +0 -0
  47. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/concepts/versions-and-refs.md +0 -0
  48. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/concepts/working-catalogs.md +0 -0
  49. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/getting-started.md +0 -0
  50. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/guides/building-a-host.md +0 -0
  51. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/guides/meta.ts +0 -0
  52. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/guides/mirrors.md +0 -0
  53. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/guides/public-and-private.md +0 -0
  54. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/guides/reading-without-duckdb.md +0 -0
  55. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/guides/serving-over-http.md +0 -0
  56. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/guides/stores.md +0 -0
  57. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/index.md +0 -0
  58. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/internals/ducklake-formats.md +0 -0
  59. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/internals/from-write-to-read.md +0 -0
  60. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/internals/inside-a-repository.md +0 -0
  61. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/internals/meta.ts +0 -0
  62. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/reference/meta.ts +0 -0
  63. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/reference/python.md +0 -0
  64. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/spec/http.md +0 -0
  65. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/spec/index.md +0 -0
  66. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/spec/layout.md +0 -0
  67. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/spec/meta.ts +0 -0
  68. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/spec/reading.md +0 -0
  69. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/spec/refs.md +0 -0
  70. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/spec/writing.md +0 -0
  71. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/reference/meta.ts +0 -0
  72. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/reference/python.md +0 -0
  73. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/spec/http.md +0 -0
  74. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/spec/index.md +0 -0
  75. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/spec/layout.md +0 -0
  76. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/spec/meta.ts +0 -0
  77. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/spec/reading.md +0 -0
  78. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/spec/refs.md +0 -0
  79. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/spec/writing.md +0 -0
  80. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/package-lock.json +0 -0
  81. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/package.json +0 -0
  82. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/public/CNAME +0 -0
  83. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/__init__.py +0 -0
  84. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/catalog.py +0 -0
  85. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/dataset.py +0 -0
  86. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/layout.py +0 -0
  87. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/objects.py +0 -0
  88. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/py.typed +0 -0
  89. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/refs.py +0 -0
  90. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/store.py +0 -0
  91. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/stores.py +0 -0
  92. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/version.py +0 -0
  93. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/tests/http_repo.py +0 -0
  94. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/tests/test_cli.py +0 -0
  95. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/tests/test_dataset.py +0 -0
  96. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/tests/test_stores.py +0 -0
  97. {fdl_cli-0.20.0 → fdl_cli-0.21.0}/uv.lock +0 -0
@@ -1,6 +1,34 @@
1
1
  # CHANGELOG
2
2
 
3
3
 
4
+ ## v0.21.0 (2026-11-06)
5
+
6
+ ### Bug Fixes
7
+
8
+ - **withdraw**: Delete the assets only withdrawn versions reach
9
+ ([#41](https://github.com/queria-io/fdl/pull/41),
10
+ [`33a452e`](https://github.com/queria-io/fdl/commit/33a452e2de9ab0640a32a6550fd21851ba4dc038))
11
+
12
+ A version reaches the assets its tree.json lists and those its asset columns hold. Withdrawal now
13
+ deletes those no kept version reaches, before the data files, so a run that stops half way can
14
+ still work them out.
15
+
16
+ ### Features
17
+
18
+ - **profiles**: List every value of a column with up to 2,000 distinct values
19
+ ([#42](https://github.com/queria-io/fdl/pull/42),
20
+ [`50a9333`](https://github.com/queria-io/fdl/commit/50a9333fc7b1d2fae990d555ed1393d11d01328c))
21
+
22
+
23
+ ## v0.20.1 (2026-11-06)
24
+
25
+ ### Bug Fixes
26
+
27
+ - Withdraw a version already past retention by deleting unread files at once
28
+ ([#40](https://github.com/queria-io/fdl/pull/40),
29
+ [`d9e3788`](https://github.com/queria-io/fdl/commit/d9e3788b6ffb429a9d301491a5e2a73eca8d891f))
30
+
31
+
4
32
  ## v0.20.0 (2026-11-06)
5
33
 
6
34
  ### Documentation
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: fdl-cli
3
- Version: 0.20.0
3
+ Version: 0.21.0
4
4
  Summary: Versions of a DuckLake catalog on object storage, kept the way Git keeps commits
5
5
  Project-URL: Homepage, https://fdl.flo8s.com/
6
6
  Project-URL: Documentation, https://fdl.flo8s.com/
@@ -108,11 +108,11 @@ fdl tag <remote> <name> --delete
108
108
  fdl withdraw <remote> <version>... [--yes]
109
109
  ```
110
110
 
111
- 間違って公開した個人データなど、公開してはいけないデータを持つバージョンを取り下げます([取り下げ](/ja/spec/data-files#取り下げ))。先にそのデータを除いたバージョンをコミットし、該当するバージョンを指すブランチが無い状態にします。そのうえでこのコマンドが、該当するバージョンを指すタグ、そのカタログが参照し残すバージョンが読まないデータファイル、カタログを消し、`"reason": "withdrawn"` の `expired.json` を置きます。`version.json` は残します。
111
+ 間違って公開した個人データなど、公開してはいけないデータを持つバージョンを取り下げます([取り下げ](/ja/spec/data-files#取り下げ))。先にそのデータを除いたバージョンをコミットし、該当するバージョンを指すブランチが無い状態にします。そのうえでこのコマンドが、該当するバージョンを指すタグ、そのバージョンが参照し残すバージョンが参照しないデータファイルとアセット、カタログを消し、`"reason": "withdrawn"` の `expired.json` を置きます。`version.json` は残します。保持期間を過ぎたバージョンを含めると、そのカタログはもう無いので、有効なバージョンのどれも参照しないデータファイルとアセットを、猶予期間を待たずにすべて消します。
112
112
 
113
- アセットは、まだ消しません。データを持つものは `assets/` から手で消してください。
113
+ バージョンが参照するアセットは、`tree.json` に載っているものと、アセット列が持つ sha256 です。
114
114
 
115
- `--yes` を付けないと、消すものを表示するだけです。残すバージョンが読んでいるファイルは残すものとして表示します。それがそのデータを持つなら、そのバージョンも取り下げます。ブランチかリクエストが該当するバージョンを指しているあいだは断られます。
115
+ `--yes` を付けないと、消すものを表示するだけです。残すバージョンが参照しているファイルは残すものとして表示します。それがそのデータを持つなら、そのバージョンも取り下げます。ブランチかリクエストが該当するバージョンを指しているあいだは断られます。
116
116
 
117
117
  ## fdl expire
118
118
 
@@ -55,7 +55,7 @@ DuckLake 自身の片付け(`ducklake_cleanup_old_files`、`ducklake_expire_sn
55
55
 
56
56
  1. そのデータを除いた新しいバージョンを作り、該当するバージョンを指しているブランチをすべてそこへ動かす。新しいバージョンの親はいまのバージョンなので、前へ動かすことになる
57
57
  2. 該当するバージョンを指すタグを消す
58
- 3. 個人情報を持つデータファイルや、それが写ったスクリーンショットなど、そのデータを持つ共有オブジェクトをすぐに消す。該当するバージョンから到達し、残すバージョンのどれからも到達しないものが対象。そのオブジェクトに到達するバージョンは、すべて該当するバージョンになる
58
+ 3. 個人情報を持つデータファイルや、それが写ったスクリーンショットなど、そのデータを持つ共有オブジェクトをすぐに消す。該当するバージョンから到達し、残すバージョンのどれからも到達しないものが対象。そのオブジェクトに到達するバージョンは、すべて該当するバージョンになる。該当するバージョンに保持期間を過ぎたものがあると、カタログが無いので、それがどのオブジェクトに到達していたかが分からない。その場合は、有効なバージョンのどれからも到達しない共有オブジェクトもすべてすぐに消す
59
59
  4. 該当するバージョンそれぞれに `"reason": "withdrawn"` の `expired.json` を置き、カタログを消す
60
60
 
61
61
  共有オブジェクトをカタログより先に消すのは、途中で止まった取り下げをやり直せるようにするためです。カタログが残っていれば、どのファイルがそのデータを持つかが分かります。
@@ -191,12 +191,12 @@ profile は、`ducklake.snapshot_id` の時点の 1 つのテーブルの要約
191
191
 
192
192
  - `rows` はテーブルの行数。`columns` にはカタログの順にすべての列を並べ、`name` と `type`(DuckLake の型名)は `dataset.json` と同じにする
193
193
  - 図形の列を除くすべての列が `null_count` を持つ。数値・テキスト・日付と時刻の列は `distinct_count` と `values` を持つ。大きなテーブルの `distinct_count` は推定値のことがある
194
- - `values` は、値(テキスト)とその値を持つ行の数を、多い順、同数なら値の順に並べる。列が持つ異なる値が 1,000 以下なら全部を並べ、そのとき `distinct_count` は正確な数になる。読み手は、`values` の長さが `distinct_count` と等しいことで全部の一覧だと判定する。それより多い列は上位 10 件だけを並べる。ホストは全部の一覧を、地域コードや名前などの値で検索できる
194
+ - `values` は、値(テキスト)とその値を持つ行の数を、多い順、同数なら値の順に並べる。列が持つ異なる値が 2,000 以下なら全部を並べ、そのとき `distinct_count` は正確な数になる。読み手は、`values` の長さが `distinct_count` と等しいことで全部の一覧だと判定する。それより多い列は上位 10 件だけを並べる。ホストは全部の一覧を、地域コードや名前などの値で検索できる
195
195
  - 数値の列は `min`・`max`・`mean`・`std`、四分位数の `q1`・`median`・`q3`、`min` から `max` までを等幅に区切った `histogram` を持つ。日付と時刻の列は `min` と `max` をテキストで持ち、日付とタイムスタンプは `span_days` も持つ。真偽値の列は `true_count` と `false_count` を持つ
196
196
  - `head` は、テーブルが返す先頭の行を JSON のオブジェクトで持つ。図形の列と、値を JSON で表せない列は入れない
197
197
  - `geometry` は、書き手が読めたときに、最初の図形の列を説明する。図形を持つ行の数、全体の `bbox`(列の座標で `[xmin, ymin, xmax, ymax]`)、含まれる図形の種類、地図を描くための、無作為に選んで簡略化した図形の GeoJSON `FeatureCollection` を持つ
198
198
  - 書き手は計算できないキーを省いてよい。読み手は知らないキーを無視する
199
- - 書き手は、データファイル(`files.json`)と列が親と同じテーブルには親の profile を使い回す。コミットで読むのは変わったテーブルだけになる
199
+ - 書き手は、データファイル(`files.json`)と列が親と同じテーブルには親の profile を使い回す。コミットで読むのは変わったテーブルだけになる。親の profile の `values` が書き手と異なる上限で書かれているときは、profile を作り直す
200
200
  - profile はバージョンの一部で、そのバージョンの中では変わらない。保持期間から外すときはカタログと一緒に消す
201
201
 
202
202
  ## 固めたカタログ
@@ -108,11 +108,11 @@ fdl tag <remote> <name> --delete
108
108
  fdl withdraw <remote> <version>... [--yes]
109
109
  ```
110
110
 
111
- Removes versions that hold data that must not be published, such as personal data published by mistake (see [Withdrawal](/spec/data-files#withdrawal)). First commit a version without the data, so that no branch points at an affected version. The command then deletes the tags that point at the versions, the data files their catalogs name that no kept version reads, and their catalogs, and puts `expired.json` with `"reason": "withdrawn"`. `version.json` stays.
111
+ Removes versions that hold data that must not be published, such as personal data published by mistake (see [Withdrawal](/spec/data-files#withdrawal)). First commit a version without the data, so that no branch points at an affected version. The command then deletes the tags that point at the versions, the data files and assets the versions reach that no kept version reaches, and their catalogs, and puts `expired.json` with `"reason": "withdrawn"`. `version.json` stays. When one of the versions is already past retention, its catalog is gone, so every data file and asset no live version reaches is deleted at once, without waiting for the grace period.
112
112
 
113
- Assets are not deleted yet: delete those that hold the data from `assets/` by hand.
113
+ A version reaches the assets its `tree.json` lists and those its asset columns hold.
114
114
 
115
- Without `--yes` it only prints what it would delete. Files a kept version still reads are listed as kept; if they hold the data, withdraw that version too. Refused while a branch or a request points at one of the versions.
115
+ Without `--yes` it only prints what it would delete. Files a kept version still reaches are listed as kept; if they hold the data, withdraw that version too. Refused while a branch or a request points at one of the versions.
116
116
 
117
117
  ## fdl expire
118
118
 
@@ -55,7 +55,7 @@ Withdrawal removes data that must not be published, such as personal data publis
55
55
 
56
56
  1. Make a new version without the data and move every branch that holds an affected version to it. This is a forward move: the new version's parent is the current one
57
57
  2. Delete the tags that point at affected versions
58
- 3. Delete the shared objects that hold the data, at once, such as a data file with personal data or a screenshot showing it: those an affected version reaches and no version kept reaches. Every version that reaches such an object is an affected version
58
+ 3. Delete the shared objects that hold the data, at once, such as a data file with personal data or a screenshot showing it: those an affected version reaches and no version kept reaches. Every version that reaches such an object is an affected version. An affected version already past retention has no catalog to say which objects it reached, so when there is one, every shared object no live version reaches is deleted at once too
59
59
  4. For every affected version, put `expired.json` with `"reason": "withdrawn"` and delete its catalogs
60
60
 
61
61
  The shared objects go before the catalogs, so a withdrawal that stops half way can be run again: the catalogs still say which files hold the data.
@@ -191,12 +191,12 @@ Each table's profile is at `versions/<ID>/profiles/<schema>/<table>.json`. A tab
191
191
 
192
192
  - `rows` is the table's row count. `columns` lists every column in the catalog's order, with `name` and `type` (DuckLake's type name) as in `dataset.json`
193
193
  - Every column but a geometry one has `null_count`. Numbers, text, dates and times have `distinct_count`, which may be estimated on a large table, and `values`
194
- - `values` lists each value, as text, with the number of rows that hold it, most common first and then by value. When the column holds at most 1,000 distinct values, it lists them all and `distinct_count` is exact: a reader tells a whole list by its length matching `distinct_count`. Otherwise it lists the 10 most common. A host can search a whole list by value, such as a town's code or name
194
+ - `values` lists each value, as text, with the number of rows that hold it, most common first and then by value. When the column holds at most 2,000 distinct values, it lists them all and `distinct_count` is exact: a reader tells a whole list by its length matching `distinct_count`. Otherwise it lists the 10 most common. A host can search a whole list by value, such as a town's code or name
195
195
  - Numbers have `min`, `max`, `mean`, `std`, the quartiles `q1`, `median` and `q3`, and a `histogram` of equal-width buckets between `min` and `max`. Dates and times have `min` and `max` as text, and dates and timestamps `span_days`. Booleans have `true_count` and `false_count`
196
196
  - `head` holds the first rows the table returns, as JSON objects, leaving out geometry and columns whose values have no JSON form
197
197
  - `geometry` describes the first geometry column, when the writer could read it: how many rows have a shape, the `bbox` of all of them (`[xmin, ymin, xmax, ymax]` in the column's coordinates), the geometry types present, and a GeoJSON `FeatureCollection` of a random sample of simplified shapes for drawing a map
198
198
  - A writer may leave out any key it cannot compute. A reader ignores keys it does not know
199
- - A writer reuses its parent's profile for a table whose data files (in `files.json`) and columns are the parent's, so a commit reads only the tables that changed
199
+ - A writer reuses its parent's profile for a table whose data files (in `files.json`) and columns are the parent's, so a commit reads only the tables that changed. It makes the profile again when the parent's `values` follow another limit than its own
200
200
  - Profiles are part of the version and do not change with it. They are deleted with the catalogs when the version is taken out of retention
201
201
 
202
202
  ## The frozen catalog
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "fdl-cli"
3
- version = "0.20.0"
3
+ version = "0.21.0"
4
4
  description = "Versions of a DuckLake catalog on object storage, kept the way Git keeps commits"
5
5
  readme = "README.md"
6
6
  license = "MIT"
@@ -294,12 +294,14 @@ def withdraw(
294
294
  """Remove versions that hold data that must not be published.
295
295
 
296
296
  Move every branch to a version without the data first. This deletes the
297
- tags that point at the versions, the data files no kept version reads,
298
- and the versions' catalogs, at once. Their version.json stays.
297
+ tags that point at the versions, the data files and assets no kept version
298
+ reaches, and the versions' catalogs, at once. Their version.json stays. A
299
+ version already past retention has no catalog, so then every data file and
300
+ asset no live version reaches is deleted.
299
301
  """
300
302
  store = open_store(remote)
301
303
  try:
302
- plan = repo.plan_withdrawal(store, versions)
304
+ plan = repo.plan_withdrawal(store, versions, setup=_read_setup(store))
303
305
  for name in plan.tags:
304
306
  typer.echo(f"delete tag {name}")
305
307
  for key in plan.deleted:
@@ -307,7 +309,7 @@ def withdraw(
307
309
  for version_id in plan.versions:
308
310
  typer.echo(f"withdraw {version_id}")
309
311
  for key in plan.still_read:
310
- typer.echo(f"keep file {key} (a kept version reads it)")
312
+ typer.echo(f"keep file {key} (a kept version reaches it)")
311
313
  if not yes:
312
314
  typer.echo("Nothing was changed. Run again with --yes to do this.")
313
315
  return
@@ -25,8 +25,9 @@ GEOMETRY_SAMPLE = 500
25
25
  #: Vertex tolerance of the sampled shapes, in the column's units (about 11 m in degrees).
26
26
  SIMPLIFY_TOLERANCE = 0.0001
27
27
  HISTOGRAM_BUCKETS = 20
28
- #: A column with at most this many distinct values lists them all in `values`.
29
- ALL_VALUES = 1000
28
+ #: A column with at most this many distinct values lists them all in `values`: enough for
29
+ #: a column of codes or names that covers a whole country, such as its municipalities.
30
+ ALL_VALUES = 2000
30
31
  #: Otherwise `values` lists the most common.
31
32
  TOP_VALUES = 10
32
33
  #: Above this many rows, distinct values are estimated rather than counted.
@@ -112,14 +113,30 @@ def build(working: Path, wanted: Iterable[dict[str, Any]], *, data_path: str, se
112
113
 
113
114
 
114
115
  def reusable(previous: bytes, table: dict[str, Any]) -> bool:
115
- """Whether a parent's profile describes ``table`` as it is: the same columns and types."""
116
+ """Whether a parent's profile describes ``table`` as it is and as this writer would.
117
+
118
+ The same columns and types, and `values` listed by the current `ALL_VALUES`:
119
+ a profile written under another limit is made again rather than carried forward.
120
+ """
116
121
  try:
117
122
  held = json.loads(previous).get("columns", [])
118
123
  except ValueError:
119
124
  return False
120
- return [{"name": c.get("name"), "type": c.get("type")} for c in held] == columns_of(table)
125
+ if [{"name": c.get("name"), "type": c.get("type")} for c in held] != columns_of(table):
126
+ return False
127
+ return all(_listed_as_now(c) for c in held)
121
128
 
122
129
 
130
+ def _listed_as_now(column: dict[str, Any]) -> bool:
131
+ values, distinct = column.get("values"), column.get("distinct_count")
132
+ if values is None:
133
+ return True
134
+ if not isinstance(values, list) or not isinstance(distinct, int):
135
+ return False
136
+ if len(values) == distinct:
137
+ return distinct <= ALL_VALUES
138
+ return distinct > ALL_VALUES and len(values) <= TOP_VALUES
139
+
123
140
 
124
141
  def _load_spatial(con) -> bool:
125
142
  try:
@@ -622,18 +622,23 @@ class Withdrawal:
622
622
  versions: tuple[str, ...]
623
623
  #: Tags that point at the versions, deleted first.
624
624
  tags: tuple[str, ...]
625
- #: Data files the versions' catalogs name and no version kept reads.
625
+ #: Shared objects (data files and assets) the versions reach and no kept
626
+ #: version reaches, as store keys.
626
627
  deleted: tuple[str, ...]
627
- #: Data files the versions name that a kept version still reads. Kept: if
628
- #: they hold the data, the version that reads them has to be withdrawn too.
628
+ #: Shared objects the versions reach that a kept version still reaches. Kept:
629
+ #: if they hold the data, the version that reaches them has to be withdrawn too.
629
630
  still_read: tuple[str, ...]
630
631
 
631
632
 
632
- def plan_withdrawal(store: Store, version_ids: Iterable[str]) -> Withdrawal:
633
+ def plan_withdrawal(store: Store, version_ids: Iterable[str], *, setup: Iterable[str] = ()) -> Withdrawal:
633
634
  """Work out a withdrawal (see `withdraw`) without changing anything.
634
635
 
635
636
  Refused while a branch or a request still points at one of the versions:
636
637
  the first step, a new version without the data, is the writer's.
638
+
639
+ Shared objects are data files and assets. ``setup`` runs before a
640
+ catalog is attached to read its asset columns: a ``CREATE SECRET`` for
641
+ data files in private storage, say.
637
642
  """
638
643
  config = read_config(store)
639
644
  affected = tuple(dict.fromkeys(version_ids))
@@ -653,52 +658,110 @@ def plan_withdrawal(store: Store, version_ids: Iterable[str]) -> Withdrawal:
653
658
  )
654
659
  tags.append(name)
655
660
 
661
+ setup = list(setup)
662
+ # Store keys: data files under the data directory, assets under assets/.
656
663
  candidates: set[str] = set()
657
664
  kept: set[str] = set()
665
+ catalogless = False
658
666
  with tempfile.TemporaryDirectory(prefix="fdl-") as tmp:
659
667
  for version_id in _version_ids(store):
660
668
  if version_id in affected:
669
+ version = read_version(store, version_id)
661
670
  local = _fetch_catalog(store, version_id, Path(tmp))
662
671
  if local is not None:
663
672
  # Every file the catalog has ever named: earlier snapshots
664
673
  # in it may hold the data too.
665
- candidates.update(catalog.data_files(local))
674
+ candidates.update(config.data_dir + key for key in catalog.data_files(local))
675
+ try:
676
+ candidates.update(_reached_assets(store, version, local, setup))
677
+ except objects.ObjectError:
678
+ # A run stopped half way may have deleted the data
679
+ # files that held its asset columns: then, as for a
680
+ # version without a catalog, every asset no live
681
+ # version reaches goes.
682
+ catalogless = True
666
683
  local.unlink()
684
+ else:
685
+ catalogless = True
667
686
  continue
668
687
  if is_expired(store, version_id):
669
688
  continue
670
689
  version = read_version(store, version_id)
671
690
  # files.json lists what the version reads, without its catalog.
672
691
  indexed = _indexed_files(store, version)
692
+ local = None
693
+ if indexed is None or _asset_columns(store, version):
694
+ local = _fetch_catalog(store, version_id, Path(tmp))
673
695
  if indexed is not None:
674
- kept.update(indexed)
675
- continue
676
- local = _fetch_catalog(store, version_id, Path(tmp))
696
+ kept.update(config.data_dir + key for key in indexed)
697
+ elif local is not None:
698
+ kept.update(config.data_dir + key for key in _read_files(local, version))
699
+ kept.update(_reached_assets(store, version, local, setup))
677
700
  if local is not None:
678
- kept.update(_read_files(local, version))
679
701
  local.unlink()
702
+ if catalogless:
703
+ # A version already past retention no longer says which objects it
704
+ # reached. Those no live version reaches are waiting for the grace
705
+ # period anyway, so all of them go now.
706
+ candidates.update(store.list(config.data_dir))
707
+ candidates.update(store.list(layout.ASSETS_DIR))
680
708
  return Withdrawal(
681
709
  versions=affected,
682
710
  tags=tuple(tags),
683
- deleted=tuple(config.data_dir + key for key in sorted(candidates - kept)),
684
- still_read=tuple(config.data_dir + key for key in sorted(candidates & kept)),
711
+ deleted=tuple(sorted(candidates - kept)),
712
+ still_read=tuple(sorted(candidates & kept)),
685
713
  )
686
714
 
687
715
 
716
+ def _asset_columns(store: Store, version: Version) -> list[tuple[str, str, str]]:
717
+ """The asset columns the version's `dataset.json` marks."""
718
+ if version.dataset is None:
719
+ return []
720
+ try:
721
+ described = json.loads(store.get(layout.version_dir(version.id) + version.dataset.path))
722
+ except Exception as exc: # noqa: BLE001 - stores raise their own types
723
+ if is_missing(exc):
724
+ return []
725
+ raise
726
+ return objects.asset_columns(described)
727
+
728
+
729
+ def _reached_assets(store: Store, version: Version, local: Path | None, setup: list[str]) -> set[str]:
730
+ """The asset keys the version reaches: its `tree.json` and its asset columns."""
731
+ reached: set[str] = set()
732
+ if version.tree is not None:
733
+ try:
734
+ listed = json.loads(store.get(layout.version_dir(version.id) + version.tree.path))
735
+ except Exception as exc: # noqa: BLE001 - stores raise their own types
736
+ if not is_missing(exc):
737
+ raise
738
+ else:
739
+ reached.update(objects.asset_key(item["sha256"]) for item in listed["entries"])
740
+ columns = _asset_columns(store, version)
741
+ if columns and local is not None:
742
+ values = objects.asset_values(local, columns, data_path=data_url(store), setup=setup)
743
+ reached.update(objects.asset_key(sha256) for sha256 in values)
744
+ return reached
745
+
746
+
688
747
  def withdraw(store: Store, version_ids: Iterable[str], *, plan: Withdrawal | None = None) -> Withdrawal:
689
748
  """Remove versions that hold data that must not be published.
690
749
 
691
- Unlike retention it takes live versions and deletes their data files at
692
- once. In the spec's order, except that the data files go before the
750
+ Unlike retention it takes live versions and deletes their data files and
751
+ assets at once. In the spec's order: the shared objects go before the
693
752
  catalogs, so a run that stops half way can be run again while the
694
- catalogs still say which files they are. `version.json` stays.
753
+ catalogs still say which objects they are. `version.json` stays.
695
754
  """
696
755
  plan = plan or plan_withdrawal(store, version_ids)
697
756
  for name in plan.tags:
698
757
  refs.delete(store, name)
699
758
  if plan.tags:
700
759
  update_refs_index(store)
701
- store.delete(list(plan.deleted))
760
+ # Assets first: the data files are what says which assets the versions'
761
+ # rows reach, so a run that stops half way can still work them out.
762
+ assets = [key for key in plan.deleted if key.startswith(layout.ASSETS_DIR)]
763
+ store.delete(assets)
764
+ store.delete([key for key in plan.deleted if not key.startswith(layout.ASSETS_DIR)])
702
765
  for version_id in plan.versions:
703
766
  expire(store, version_id, reason="withdrawn")
704
767
  return plan
@@ -156,7 +156,8 @@ def test_a_file_the_host_will_not_sign_goes_through_the_host(tmp_path: Path) ->
156
156
 
157
157
  assert local.get("data/small.parquet") == b"ab" and local.get("data/large.parquet") == b"abcdef"
158
158
  puts = [key for method, key in server.requests if method == "PUT"]
159
- assert puts == ["storage/data/small.parquet", "data/large.parquet"]
159
+ # Uploads run at once, so they may arrive in any order.
160
+ assert sorted(puts) == ["data/large.parquet", "storage/data/small.parquet"]
160
161
 
161
162
 
162
163
  def test_a_rebuild_of_the_same_rows_over_http_is_nothing_to_commit(served, tmp_path: Path) -> None:
@@ -175,3 +175,37 @@ def test_media_type_does_not_depend_on_the_machine(path: str, expected: str) ->
175
175
  def test_asset_columns_that_cannot_be_read_are_an_object_error(tmp_path: Path) -> None:
176
176
  with pytest.raises(objects.ObjectError, match="could not read the asset columns"):
177
177
  objects.asset_values(tmp_path / "missing.ducklake", [("main", "t", "c")], data_path=str(tmp_path) + "/")
178
+
179
+
180
+ def test_withdrawal_deletes_the_assets_only_the_withdrawn_versions_reach(tmp_path: Path, repository) -> None:
181
+ store, working = repository
182
+ folder = tmp_path / "files"
183
+ folder.mkdir()
184
+ (folder / "kept.pdf").write_bytes(b"%PDF-kept")
185
+ (folder / "leak.pdf").write_bytes(b"%PDF-leak")
186
+ (tmp_path / "files.csv").write_text(run("assets", folder))
187
+ (tmp_path / "README.md").write_text("# Reports\n")
188
+ (tmp_path / "dataset.yml").write_text(
189
+ "name: reports\ntitle: Reports\ntables:\n"
190
+ " - schema: main\n name: reports\n fields:\n - name: sha256\n asset: true\n"
191
+ )
192
+ write(
193
+ working,
194
+ store,
195
+ f"CREATE TABLE lake.main.reports AS SELECT path, sha256 FROM read_csv('{tmp_path / 'files.csv'}', all_varchar = true)",
196
+ )
197
+ run("commit", working, "-m", "leak", "--author", "t", "--assets", folder)
198
+ leaked = repo.head(store, "refs/heads/main")
199
+ write(working, store, "DELETE FROM lake.main.reports WHERE path = 'leak.pdf'")
200
+ run("commit", working, "-m", "clean", "--author", "t")
201
+ rows = {row["path"]: row["sha256"] for row in csv.DictReader(io.StringIO(run("assets", folder)))}
202
+ readme = objects.asset_key(objects.sha256_of(tmp_path / "README.md"))
203
+
204
+ plan = repo.plan_withdrawal(store, [leaked.id])
205
+ assert objects.asset_key(rows["leak.pdf"]) in plan.deleted
206
+ assert {objects.asset_key(rows["kept.pdf"]), readme} <= set(plan.still_read)
207
+
208
+ run("withdraw", store.root, leaked.id, "--yes")
209
+ assert store.head(objects.asset_key(rows["leak.pdf"])) is None
210
+ assert store.head(objects.asset_key(rows["kept.pdf"])) is not None
211
+ assert store.head(readme) is not None
@@ -76,6 +76,33 @@ def test_many_values_list_only_the_most_common(repository) -> None:
76
76
  assert code["distinct_count"] > profiles.ALL_VALUES
77
77
 
78
78
 
79
+ def test_a_column_at_the_limit_lists_every_value(repository) -> None:
80
+ store, working = repository
81
+ write(working, store, f"CREATE TABLE lake.main.towns AS SELECT 'v' || i AS code FROM range({profiles.ALL_VALUES}) t(i)")
82
+ run("commit", working, "-m", "m", "--author", "t")
83
+ code = profile_of(store)["columns"][0]
84
+ assert len(code["values"]) == code["distinct_count"] == profiles.ALL_VALUES
85
+
86
+
87
+ @pytest.mark.parametrize(
88
+ ("values", "distinct", "kept"),
89
+ [
90
+ (3, 3, True),
91
+ (profiles.TOP_VALUES, profiles.ALL_VALUES + 1, True),
92
+ # Written under a lower limit: a short list of a column now listed whole.
93
+ (profiles.TOP_VALUES, profiles.ALL_VALUES - 1, False),
94
+ # Written under a higher limit: a whole list longer than now allowed.
95
+ (profiles.ALL_VALUES + 1, profiles.ALL_VALUES + 1, False),
96
+ ],
97
+ )
98
+ def test_a_profile_written_under_another_limit_is_made_again(values: int, distinct: int, kept: bool) -> None:
99
+ table = {"fields": [{"name": "code", "type": "VARCHAR"}]}
100
+ column = {"name": "code", "type": "VARCHAR", "distinct_count": distinct}
101
+ column["values"] = [{"value": f"v{i}", "count": 1} for i in range(values)]
102
+ previous = json.dumps({"rows": distinct, "columns": [column]}).encode()
103
+ assert profiles.reusable(previous, table) is kept
104
+
105
+
79
106
  def test_an_unchanged_table_keeps_its_parents_profile(repository, monkeypatch) -> None:
80
107
  store, working = repository
81
108
  write(working, store, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
@@ -387,6 +387,23 @@ def test_withdraw_removes_the_data_from_live_versions(tmp_path: Path, remote) ->
387
387
  assert repo.head(remote) == clean
388
388
 
389
389
 
390
+ def test_withdrawing_a_version_past_retention_deletes_unread_files_at_once(tmp_path: Path, remote) -> None:
391
+ first = pushed(tmp_path, remote)
392
+ leaked = set(remote.list("data/"))
393
+ build(tmp_path / "work.sqlite", remote, "DROP TABLE lake.main.towns; CREATE TABLE lake.main.kept AS SELECT 1 AS id")
394
+ second = commit(tmp_path, remote, parent=first)
395
+ repo.push(remote, tmp_path / "local" / second.id, second)
396
+ kept = set(remote.list("data/")) - leaked
397
+ repo.expire(remote, first.id)
398
+ assert leaked <= set(remote.list("data/")) # waiting for the grace period
399
+
400
+ plan = repo.withdraw(remote, [first.id])
401
+ assert set(plan.deleted) == leaked
402
+ assert set(remote.list("data/")) == kept
403
+ assert json.loads(remote.get(layout.version_expired(first.id)))["reason"] == "withdrawn"
404
+ assert repo.head(remote) == second
405
+
406
+
390
407
  def test_a_file_a_kept_version_still_reads_is_not_deleted(tmp_path: Path, remote) -> None:
391
408
  first = pushed(tmp_path, remote)
392
409
  second = pushed(tmp_path, remote, parent=first, sql="INSERT INTO lake.main.towns VALUES (2)")
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes