fdl-cli 0.28.0__tar.gz → 0.29.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/CHANGELOG.md +22 -0
  2. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/PKG-INFO +1 -1
  3. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/getting-started.md +2 -0
  4. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/getting-started.md +2 -0
  5. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/reference/cli.md +6 -4
  6. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/spec/data-files.md +3 -2
  7. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/reference/cli.md +6 -4
  8. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/spec/data-files.md +3 -2
  9. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/pyproject.toml +1 -1
  10. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/cli.py +48 -2
  11. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/repo.py +8 -5
  12. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/tests/test_cli.py +53 -0
  13. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/tests/test_repo.py +6 -0
  14. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/.github/workflows/docs.yml +0 -0
  15. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/.github/workflows/publish.yml +0 -0
  16. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/.github/workflows/release.yml +0 -0
  17. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/.github/workflows/test.yml +0 -0
  18. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/.gitignore +0 -0
  19. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/.node-version +0 -0
  20. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/.python-version +0 -0
  21. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/CLAUDE.md +0 -0
  22. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/CONTRIBUTING.md +0 -0
  23. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/LICENSE +0 -0
  24. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/README.md +0 -0
  25. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/SECURITY.md +0 -0
  26. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/blume.config.ts +0 -0
  27. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/cases/collection.json +0 -0
  28. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/cases/data-dir.json +0 -0
  29. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/concepts/meta.ts +0 -0
  30. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/concepts/repositories.md +0 -0
  31. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/concepts/versions-and-refs.md +0 -0
  32. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/concepts/working-metadata.md +0 -0
  33. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/guides/building-a-host.md +0 -0
  34. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/guides/meta.ts +0 -0
  35. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/guides/mirrors.md +0 -0
  36. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/guides/public-and-private.md +0 -0
  37. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/guides/reading-without-duckdb.md +0 -0
  38. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/guides/serving-over-http.md +0 -0
  39. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/guides/stores.md +0 -0
  40. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/index.md +0 -0
  41. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/internals/ducklake-formats.md +0 -0
  42. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/internals/from-write-to-read.md +0 -0
  43. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/internals/inside-a-repository.md +0 -0
  44. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/internals/meta.ts +0 -0
  45. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/concepts/meta.ts +0 -0
  46. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/concepts/repositories.md +0 -0
  47. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/concepts/versions-and-refs.md +0 -0
  48. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/concepts/working-metadata.md +0 -0
  49. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/guides/building-a-host.md +0 -0
  50. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/guides/meta.ts +0 -0
  51. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/guides/mirrors.md +0 -0
  52. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/guides/public-and-private.md +0 -0
  53. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/guides/reading-without-duckdb.md +0 -0
  54. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/guides/serving-over-http.md +0 -0
  55. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/guides/stores.md +0 -0
  56. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/index.md +0 -0
  57. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/internals/ducklake-formats.md +0 -0
  58. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/internals/from-write-to-read.md +0 -0
  59. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/internals/inside-a-repository.md +0 -0
  60. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/internals/meta.ts +0 -0
  61. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/reference/meta.ts +0 -0
  62. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/reference/python.md +0 -0
  63. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/spec/collection-cases.md +0 -0
  64. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/spec/http.md +0 -0
  65. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/spec/index.md +0 -0
  66. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/spec/layout.md +0 -0
  67. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/spec/meta.ts +0 -0
  68. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/spec/reading.md +0 -0
  69. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/spec/refs.md +0 -0
  70. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/spec/versions.md +0 -0
  71. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/spec/writing.md +0 -0
  72. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/stability.md +0 -0
  73. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/reference/meta.ts +0 -0
  74. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/reference/python.md +0 -0
  75. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/spec/collection-cases.md +0 -0
  76. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/spec/http.md +0 -0
  77. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/spec/index.md +0 -0
  78. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/spec/layout.md +0 -0
  79. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/spec/meta.ts +0 -0
  80. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/spec/reading.md +0 -0
  81. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/spec/refs.md +0 -0
  82. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/spec/versions.md +0 -0
  83. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/spec/writing.md +0 -0
  84. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/stability.md +0 -0
  85. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/package-lock.json +0 -0
  86. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/package.json +0 -0
  87. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/public/CNAME +0 -0
  88. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/__init__.py +0 -0
  89. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/catalog.py +0 -0
  90. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/dataset.py +0 -0
  91. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/diff.py +0 -0
  92. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/layout.py +0 -0
  93. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/lineage.py +0 -0
  94. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/merge.py +0 -0
  95. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/objects.py +0 -0
  96. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/profiles.py +0 -0
  97. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/py.typed +0 -0
  98. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/refs.py +0 -0
  99. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/store.py +0 -0
  100. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/stores.py +0 -0
  101. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/version.py +0 -0
  102. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/tests/http_repo.py +0 -0
  103. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/tests/test_collection_cases.py +0 -0
  104. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/tests/test_data_dir_cases.py +0 -0
  105. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/tests/test_dataset.py +0 -0
  106. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/tests/test_http.py +0 -0
  107. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/tests/test_merge.py +0 -0
  108. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/tests/test_objects.py +0 -0
  109. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/tests/test_profiles.py +0 -0
  110. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/tests/test_stores.py +0 -0
  111. {fdl_cli-0.28.0 → fdl_cli-0.29.0}/uv.lock +0 -0
@@ -1,6 +1,28 @@
1
1
  # CHANGELOG
2
2
 
3
3
 
4
+ ## v0.29.0 (2026-11-15)
5
+
6
+ ### Features
7
+
8
+ - Record who takes a version out in expired.json, and keep version.json on withdrawal
9
+ ([#65](https://github.com/queria-io/fdl/pull/65),
10
+ [`1d3159b`](https://github.com/queria-io/fdl/commit/1d3159b4a50d309424b4b25a756477f7c225934b))
11
+
12
+ A withdrawal no longer deletes version.json: it rewrites the fields that hold the data, so every
13
+ repository holds its history back to the first version. expire and withdraw take --by, recorded in
14
+ expired.json.
15
+
16
+ - Write the version's dataset.json, README and docs/ on clone
17
+ ([#64](https://github.com/queria-io/fdl/pull/64),
18
+ [`1f37b04`](https://github.com/queria-io/fdl/commit/1f37b047866e662ff967e4adda561ed5ba5018e1))
19
+
20
+ A clone made only the working metadata, so whoever cloned a dataset had to write its description
21
+ again before they could commit. Clone now writes the version's dataset.json (without what commit
22
+ fills in) and the files of its tree.json into the current folder, unless a description is already
23
+ there.
24
+
25
+
4
26
  ## v0.28.0 (2026-11-15)
5
27
 
6
28
  ### Bug Fixes
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: fdl-cli
3
- Version: 0.28.0
3
+ Version: 0.29.0
4
4
  Summary: Versions of a DuckLake catalog on object storage, kept the way Git keeps commits
5
5
  Project-URL: Homepage, https://fdl.queria.io/
6
6
  Project-URL: Documentation, https://fdl.queria.io/
@@ -140,6 +140,8 @@ fdl commit work/lake.sqlite -m "Add 2026" --author alice
140
140
  fdl push work/lake.sqlite
141
141
  ```
142
142
 
143
+ The current folder already holds `dataset.json`, so the clone leaves it as it is. In an empty folder, the clone also writes the version's `dataset.json`, `README.md` and `docs/` there, so whoever clones the repository can change the description as well as the data.
144
+
143
145
  The new version lists the first as its parent. Both metadata files name the first Parquet file; only the second names the new one.
144
146
 
145
147
  If someone else pushed in between, the push is refused, because the new version would not be built on what `main` holds now. Clone again and redo the change. [Versions and refs](/concepts/versions-and-refs) explains why.
@@ -140,6 +140,8 @@ fdl commit work/lake.sqlite -m "Add 2026" --author alice
140
140
  fdl push work/lake.sqlite
141
141
  ```
142
142
 
143
+ 今のフォルダには `dataset.json` があるので、クローンはそれをそのままにします。空のフォルダでは、バージョンの `dataset.json`・`README.md`・`docs/` もそこに書き出すので、リポジトリをクローンした人はデータだけでなく説明も変えられます。
144
+
143
145
  新しいバージョンは、最初のバージョンを親として記録します。最初の Parquet ファイルは両方のメタデータが参照し、新しいファイルは 2 つめのメタデータだけが参照します。
144
146
 
145
147
  途中でほかの人が push していたら、この push は断られます。新しいバージョンが、いまの `main` の上に積まれていないためです。クローンし直して、変更をやり直してください。理由は[バージョンと ref](/ja/concepts/versions-and-refs) で説明しています。
@@ -43,7 +43,9 @@ HTTP のリポジトリでは、新しいデータファイルは `<working.sqli
43
43
 
44
44
  `--storage` を付けると、作業メタデータはリポジトリのストレージで直接データファイルを読み書きします。`--storage` にはそのストレージでのリポジトリの根を渡します。新しいファイルはその `data_dir` の下に直接書かれるので、push でアップロードするものはありません。鍵は `AWS_*` の環境変数(R2 などの S3 互換ストレージは `AWS_ENDPOINT_URL`)か、自分で作った DuckDB の secret から取ります。作業メタデータに書くビルドにも同じ鍵が要ります。
45
45
 
46
- バージョンの保持期間が過ぎているとき、メタデータの sha256 が `version.json` の記録と違うとき、メタデータがデータフォルダの外のデータファイルを指しているときは断られます。
46
+ 今のフォルダに `dataset.json` も `dataset.yml` も無いときは、`fdl commit` が読むファイルもそこに書き出します。バージョンの `dataset.json` からコミットがカタログから埋める値(行数、列の型、NULL の可否)を除いたものと、`tree.json` が挙げるファイル(`README.md` や `docs/` など)です。これらを編集してコミットすると説明を変えられます。フォルダに既に説明があるときはそのままにし、何も書きません。
47
+
48
+ バージョンの保持期間が過ぎているとき、メタデータの sha256 が `version.json` の記録と違うとき、メタデータがデータフォルダの外のデータファイルを指しているとき、書き出すファイルが今のフォルダに既にあるときは断られます。
47
49
 
48
50
  ## fdl commit
49
51
 
@@ -211,7 +213,7 @@ fdl push <working.sqlite>
211
213
  ## fdl withdraw
212
214
 
213
215
  ```bash
214
- fdl withdraw <remote> <version>... [--yes]
216
+ fdl withdraw <remote> <version>... [--yes] [--by <name>]
215
217
  ```
216
218
 
217
219
  間違って公開した個人データなど、公開してはいけないデータを持つバージョンを取り下げます([取り下げ](/ja/spec/data-files#取り下げ))。先にそのデータを除いたバージョンをコミットし、該当するバージョンを指すブランチが無い状態にします。そのうえでこのコマンドが、該当するバージョンを指すタグ、そのバージョンが参照し残すバージョンが参照しないデータファイルとアセット、メタデータを消し、`"reason": "withdrawn"` の `expired.json` を置きます。`version.json` は残します。保持期間を過ぎたバージョンを含めると、そのメタデータはもう無いので、有効なバージョンのどれも参照しないデータファイルとアセットを、猶予期間を待たずにすべて消します。
@@ -231,10 +233,10 @@ fdl profile <remote> [--ref refs/heads/main]
231
233
  ## fdl expire
232
234
 
233
235
  ```bash
234
- fdl expire <remote> [--keep 1] [--since 2026-10-01T00:00:00Z] [--ref refs/heads/main]
236
+ fdl expire <remote> [--keep 1] [--since 2026-10-01T00:00:00Z] [--ref refs/heads/main] [--by <name>]
235
237
  ```
236
238
 
237
- ブランチの最初の親のつながりで、新しい `--keep` 個より古いバージョンの保持期間を終わらせます。`expired.json` を置き、メタデータを消します。`--since` 以降に作ったバージョンと、ref が指すバージョンは残します。データファイルは消しません。消すのは `fdl gc` です。
239
+ ブランチの最初の親のつながりで、新しい `--keep` 個より古いバージョンの保持期間を終わらせます。`expired.json` を置き、メタデータを消します。`--since` 以降に作ったバージョンと、ref が指すバージョンは残します。データファイルは消しません。消すのは `fdl gc` です。`--by` は保持期間を終わらせた人を `expired.json` に記録します。`fdl withdraw` の `--by` も同じです。
238
240
 
239
241
  ## fdl gc
240
242
 
@@ -61,9 +61,10 @@ DuckLake 自身の片付け(`ducklake_cleanup_old_files`、`ducklake_expire_sn
61
61
  3. 猶予期間のあと、上の条件に合う共有オブジェクトを消す
62
62
 
63
63
  ```json expired.json
64
- { "expired_at": "2026-12-01T00:00:00Z", "reason": "retention" }
64
+ { "expired_at": "2026-12-01T00:00:00Z", "reason": "retention", "by": "nightly retention" }
65
65
  ```
66
66
 
67
+ - `by` には、保持期間を終わらせた人や仕組み(定期実行のジョブなど)を書いてよい。これで履歴には、バージョンを作った人と同じく、そのデータを消した人も残る
67
68
  - `version.json` は残す。履歴にはそのバージョンがあったことと中身が残り、親のつながりも切れない
68
69
  - `expired.json` を見た読み手は、`reason` が何であれ、保持期間を過ぎたものとして扱い、メタデータを開かない
69
70
  - ref が指すバージョンの保持期間を終わらせてはいけない
@@ -83,5 +84,5 @@ DuckLake 自身の片付け(`ducklake_cleanup_old_files`、`ducklake_expire_sn
83
84
  { "expired_at": "2026-12-01T00:00:00Z", "reason": "withdrawn" }
84
85
  ```
85
86
 
86
- - `version.json` は、保持期間のときと同じく残す。記録そのもの(`message` や `inputs`)にそのデータがあれば記録も消し、履歴にはその位置で親が欠けて見える
87
+ - `version.json` は、保持期間のときと同じく残す。これで、どのリポジトリも最初のバージョンまでの履歴を持つ。記録そのもの(`author`・`message`・`inputs`)にそのデータがあれば、`expired.json` を置く前に記録を書き換えてデータを除く。`author` や `message` の該当する値は `withdrawn` に置き換え、`inputs` は該当する要素を除く。`id`・`seq`・`parents` などほかの項目はそのまま残す
87
88
  - ミラーは自分の写しを持っている。`"reason": "withdrawn"` の `expired.json` を写したミラーは、元が消したメタデータと共有オブジェクトを消す。元のホストはこれを強制できないので、知っているミラーに伝える
@@ -43,7 +43,9 @@ For a repository over HTTP, new data files are written to `<working.sqlite>.data
43
43
 
44
44
  With `--storage`, the working metadata reads and writes the data files in the repository's own storage instead: `--storage` is the repository's root there, and new files go straight under its `data_dir`, so the push has none to upload. The keys come from the `AWS_*` variables (`AWS_ENDPOINT_URL` for R2 and other S3-compatible stores), or from a DuckDB secret of your own. The build that writes to the working metadata needs them too.
45
45
 
46
- Refused when the version is past retention, when its metadata's sha256 is not the one `version.json` records, or when the metadata names a data file outside the data folder.
46
+ When the current folder holds no `dataset.json` or `dataset.yml`, it also writes there what `fdl commit` reads: the version's `dataset.json`, without what the commit fills in from the catalog (row counts, column types, nulls), and the files its `tree.json` lists, such as `README.md` and `docs/`. Edit them and commit to change the description. When the folder already holds a description, it is left as it is and nothing is written.
47
+
48
+ Refused when the version is past retention, when its metadata's sha256 is not the one `version.json` records, when the metadata names a data file outside the data folder, or when a file it would write is already in the current folder.
47
49
 
48
50
  ## fdl commit
49
51
 
@@ -211,7 +213,7 @@ Refused when the working metadata has changes not committed, when a version is c
211
213
  ## fdl withdraw
212
214
 
213
215
  ```bash
214
- fdl withdraw <remote> <version>... [--yes]
216
+ fdl withdraw <remote> <version>... [--yes] [--by <name>]
215
217
  ```
216
218
 
217
219
  Removes versions that hold data that must not be published, such as personal data published by mistake (see [Withdrawal](/spec/data-files#withdrawal)). First commit a version without the data, so that no branch points at an affected version. The command then deletes the tags that point at the versions, the data files and assets the versions reach that no kept version reaches, and their metadata files, and puts `expired.json` with `"reason": "withdrawn"`. `version.json` stays. When one of the versions is already past retention, its metadata is gone, so every data file and asset no live version reaches is deleted at once, without waiting for the grace period.
@@ -231,10 +233,10 @@ Profiles the tables and views of the version the branch or tag holds that have n
231
233
  ## fdl expire
232
234
 
233
235
  ```bash
234
- fdl expire <remote> [--keep 1] [--since 2026-10-01T00:00:00Z] [--ref refs/heads/main]
236
+ fdl expire <remote> [--keep 1] [--since 2026-10-01T00:00:00Z] [--ref refs/heads/main] [--by <name>]
235
237
  ```
236
238
 
237
- Takes versions behind the newest `--keep` on the branch's first-parent line out of retention: puts `expired.json` and deletes their metadata files. Versions made at or after `--since`, and any a ref points at, are kept. Data files are not deleted; `fdl gc` deletes them.
239
+ Takes versions behind the newest `--keep` on the branch's first-parent line out of retention: puts `expired.json` and deletes their metadata files. Versions made at or after `--since`, and any a ref points at, are kept. Data files are not deleted; `fdl gc` deletes them. `--by` records who takes the versions out in `expired.json`, as does `--by` of `fdl withdraw`.
238
240
 
239
241
  ## fdl gc
240
242
 
@@ -61,9 +61,10 @@ A version is taken out of retention in this order:
61
61
  3. After the grace period, delete the shared objects that meet the conditions above
62
62
 
63
63
  ```json expired.json
64
- { "expired_at": "2026-12-01T00:00:00Z", "reason": "retention" }
64
+ { "expired_at": "2026-12-01T00:00:00Z", "reason": "retention", "by": "nightly retention" }
65
65
  ```
66
66
 
67
+ - `by` may say who or what took the version out: a person, or a process such as a scheduled job. With it, the history shows who removed each version's data, as it shows who made it
67
68
  - `version.json` stays, so the history still says the version was there and what it was, and the graph of parents stays whole
68
69
  - A reader that sees `expired.json` treats the version as past retention and does not open its metadata, whatever the `reason`
69
70
  - A version a ref points at must not be taken out
@@ -83,5 +84,5 @@ The shared objects go before the metadata files, so a withdrawal that stops half
83
84
  { "expired_at": "2026-12-01T00:00:00Z", "reason": "withdrawn" }
84
85
  ```
85
86
 
86
- - `version.json` stays, as with retention. If the record itself holds the data (in `message` or `inputs`), it is deleted too, and the history shows a missing parent at that point
87
+ - `version.json` stays, as with retention, so every repository holds its history back to the first version. If the record itself holds the data (in `author`, `message` or `inputs`), it is rewritten without it before `expired.json` is put: the value in `author` or `message` is replaced with `withdrawn`, and the entries of `inputs` that hold it are dropped. `id`, `seq`, `parents` and the other fields stay as they were
87
88
  - Mirrors keep their own copies. A mirror that copies `expired.json` with `"reason": "withdrawn"` deletes the metadata files and the shared objects the origin deleted. The origin cannot force this; the host tells the mirrors it knows of
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "fdl-cli"
3
- version = "0.28.0"
3
+ version = "0.29.0"
4
4
  description = "Versions of a DuckLake catalog on object storage, kept the way Git keeps commits"
5
5
  readme = "README.md"
6
6
  license = "MIT"
@@ -10,6 +10,7 @@ pushed.
10
10
  from __future__ import annotations
11
11
 
12
12
  import datetime as dt
13
+ import hashlib
13
14
  import json
14
15
  import os
15
16
  import re
@@ -122,7 +123,11 @@ def clone(
122
123
  try:
123
124
  # Read before the clone: a data_dir moved meanwhile then fails the commit.
124
125
  config = repo.read_config(store)
126
+ files = _described_files(store, ref)
125
127
  version = repo.clone(store, working, ref=ref, local_data=local, storage=storage)
128
+ for path, body in files.items():
129
+ path.parent.mkdir(parents=True, exist_ok=True)
130
+ path.write_bytes(body)
126
131
  if storage:
127
132
  data = repo.storage_data_path(storage, config)
128
133
  else:
@@ -136,10 +141,49 @@ def clone(
136
141
  state["storage"] = storage
137
142
  _write_state(working, state)
138
143
  typer.echo(f"{working} at {version.id if version else 'empty metadata'}")
144
+ if files:
145
+ typer.echo("Wrote its description here: " + ", ".join(sorted(path.as_posix() for path in files)))
139
146
  typer.echo("Write to it with:")
140
147
  typer.echo(" " + catalog.attach_sql(working, alias="lake", data_path=data)[-1] + ";")
141
148
 
142
149
 
150
+ def _described_files(store, ref: str) -> dict[Path, bytes]:
151
+ """What `fdl commit` reads from here, as the version ``ref`` holds it: its
152
+ `dataset.json` (without what the commit fills in) and the files of its `tree.json`.
153
+
154
+ Nothing when the branch has no version yet, or when a description is
155
+ already here: that one is the writer's, and the commit reads it. A file in
156
+ the way is refused rather than overwritten.
157
+ """
158
+ version = repo.head(store, ref)
159
+ if version is None or any(Path(name).exists() for name in dataset.DEFAULT_FILES):
160
+ return {}
161
+ described = diffs.described(store, version)
162
+ if described is None:
163
+ return {}
164
+ entries = (diffs.listed(store, version) or {}).get("entries", [])
165
+ objects.check_tree(entries)
166
+ files = {
167
+ Path(layout.DATASET): dataset.to_json(merging.declared(described)),
168
+ **{
169
+ Path(item["path"]): _tree_file(store, item)
170
+ for item in entries
171
+ },
172
+ }
173
+ present = sorted(path.as_posix() for path in files if path.exists())
174
+ if present:
175
+ raise repo.RepoError(f"{', '.join(present)} already here: clone into a folder without them")
176
+ return files
177
+
178
+
179
+ def _tree_file(store, item: dict) -> bytes:
180
+ """A file of `tree.json`, checked against its sha256."""
181
+ body = store.get(objects.asset_key(item["sha256"]))
182
+ if hashlib.sha256(body).hexdigest() != item["sha256"]:
183
+ raise repo.RepoError(f"{item['path']}: the file's sha256 is not the one tree.json records")
184
+ return body
185
+
186
+
143
187
  @app.command()
144
188
  def commit(
145
189
  working: Path = Working,
@@ -912,6 +956,7 @@ def expire(
912
956
  keep: int = typer.Option(1, help="How many of the newest versions to keep"),
913
957
  since: str | None = typer.Option(None, help="Also keep those made at or after this time (ISO 8601)"),
914
958
  ref: str = typer.Option(layout.DEFAULT_BRANCH),
959
+ by: str | None = typer.Option(None, help="Who or what takes them out, recorded in expired.json"),
915
960
  ) -> None:
916
961
  """Take older versions out of retention: mark them and drop their catalogs.
917
962
 
@@ -920,7 +965,7 @@ def expire(
920
965
  store = _store(remote)
921
966
  try:
922
967
  newest = repo.head(store, ref)
923
- expired = repo.prune(store, newest, keep=keep, since=since) if newest else []
968
+ expired = repo.prune(store, newest, keep=keep, since=since, by=by) if newest else []
924
969
  except ERRORS as exc:
925
970
  raise _fail(exc) from exc
926
971
  typer.echo(f"Expired {len(expired)} version(s).")
@@ -965,6 +1010,7 @@ def withdraw(
965
1010
  remote: str = Remote,
966
1011
  versions: list[str] = typer.Argument(..., help="The versions that hold the data"),
967
1012
  yes: bool = typer.Option(False, "--yes", help="Do it. Without this, only say what would be done"),
1013
+ by: str | None = typer.Option(None, help="Who withdraws them, recorded in expired.json"),
968
1014
  ) -> None:
969
1015
  """Remove versions that hold data that must not be published.
970
1016
 
@@ -988,7 +1034,7 @@ def withdraw(
988
1034
  if not yes:
989
1035
  typer.echo("Nothing was changed. Run again with --yes to do this.")
990
1036
  return
991
- repo.withdraw(store, versions, plan=plan)
1037
+ repo.withdraw(store, versions, plan=plan, by=by)
992
1038
  except ERRORS as exc:
993
1039
  raise _fail(exc) from exc
994
1040
  typer.echo(f"Withdrew {len(plan.versions)} version(s) and deleted {len(plan.deleted)} file(s).")
@@ -769,7 +769,7 @@ def is_expired(store: Store, version_id: str) -> bool:
769
769
  return store.head(layout.version_expired(version_id)) is not None
770
770
 
771
771
 
772
- def expire(store: Store, version_id: str, *, reason: str = "retention") -> None:
772
+ def expire(store: Store, version_id: str, *, reason: str = "retention", by: str | None = None) -> None:
773
773
  """Take a version out of retention: the mark first, then its catalogs.
774
774
 
775
775
  `version.json` stays, so the history still says the version was there and
@@ -781,7 +781,7 @@ def expire(store: Store, version_id: str, *, reason: str = "retention") -> None:
781
781
  held = _held(store)
782
782
  if version_id in held:
783
783
  raise RepoError(f"{held[version_id]} points at {version_id}, so it is kept")
784
- record = {"expired_at": utc_now(), "reason": reason}
784
+ record = {"expired_at": utc_now(), "reason": reason, **({"by": by} if by else {})}
785
785
  try:
786
786
  store.put(
787
787
  layout.version_expired(version_id),
@@ -1014,7 +1014,9 @@ def _reached_assets(store: Store, version: Version, local: Path | None, setup: l
1014
1014
  return reached
1015
1015
 
1016
1016
 
1017
- def withdraw(store: Store, version_ids: Iterable[str], *, plan: Withdrawal | None = None) -> Withdrawal:
1017
+ def withdraw(
1018
+ store: Store, version_ids: Iterable[str], *, plan: Withdrawal | None = None, by: str | None = None
1019
+ ) -> Withdrawal:
1018
1020
  """Remove versions that hold data that must not be published.
1019
1021
 
1020
1022
  Unlike retention it takes live versions and deletes their data files and
@@ -1033,7 +1035,7 @@ def withdraw(store: Store, version_ids: Iterable[str], *, plan: Withdrawal | Non
1033
1035
  store.delete(assets)
1034
1036
  store.delete([key for key in plan.deleted if not key.startswith(layout.ASSETS_DIR)])
1035
1037
  for version_id in plan.versions:
1036
- expire(store, version_id, reason="withdrawn")
1038
+ expire(store, version_id, reason="withdrawn", by=by)
1037
1039
  return plan
1038
1040
 
1039
1041
 
@@ -1080,6 +1082,7 @@ def prune(
1080
1082
  keep: int = 1,
1081
1083
  since: str | None = None,
1082
1084
  reason: str = "retention",
1085
+ by: str | None = None,
1083
1086
  ) -> list[str]:
1084
1087
  """Expire the versions behind ``newest`` that fall out of retention.
1085
1088
 
@@ -1099,7 +1102,7 @@ def prune(
1099
1102
  break
1100
1103
  recent = since is not None and _instant(current.created_at) >= _instant(since)
1101
1104
  if position >= keep and not recent and current.id not in held:
1102
- expire(store, current.id, reason=reason)
1105
+ expire(store, current.id, reason=reason, by=by)
1103
1106
  expired.append(current.id)
1104
1107
  current = read_version(store, current.parents[0]) if current.parents else None
1105
1108
  return expired
@@ -535,3 +535,56 @@ def test_clone_names_a_folder_given_as_dot_by_where_it_is(tmp_path: Path, monkey
535
535
  monkeypatch.chdir(tmp_path)
536
536
  assert _default_working(".") == Path(f"{tmp_path.name}.sqlite")
537
537
  assert _default_working("https://data.example.invalid/datasets/alice/towns/") == Path("towns.sqlite")
538
+
539
+
540
+ def _published_with_a_readme(tmp_path: Path) -> tuple[Path, LocalStore]:
541
+ remote, working = tmp_path / "remote", tmp_path / "work" / "lake.sqlite"
542
+ store = LocalStore(remote)
543
+ (tmp_path / "README.md").write_text("# Towns\n")
544
+ (tmp_path / "docs").mkdir()
545
+ (tmp_path / "docs" / "codes.md").write_text("Codes\n")
546
+ run("init", remote)
547
+ run("clone", remote, working)
548
+ write(working, store, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
549
+ commit(working, "-m", "first", "--author", "tester")
550
+ return remote, store
551
+
552
+
553
+ def test_clone_writes_the_description_to_edit_and_commit(tmp_path: Path, monkeypatch) -> None:
554
+ remote, store = _published_with_a_readme(tmp_path)
555
+ elsewhere = tmp_path / "elsewhere"
556
+ elsewhere.mkdir()
557
+ monkeypatch.chdir(elsewhere)
558
+ run("clone", remote, "lake.sqlite")
559
+
560
+ written = json.loads((elsewhere / "dataset.json").read_text())
561
+ assert written["title"] == "Towns"
562
+ assert "rows" not in written["tables"][0]
563
+ assert (elsewhere / "README.md").read_text() == "# Towns\n"
564
+ assert (elsewhere / "docs" / "codes.md").read_text() == "Codes\n"
565
+
566
+ (elsewhere / "dataset.json").write_text(json.dumps({**written, "title": "Towns of Japan"}))
567
+ commit(elsewhere / "lake.sqlite", "-m", "retitle", "--author", "tester")
568
+ head = repo.head(store)
569
+ assert json.loads(store.get(layout.version_dir(head.id) + layout.DATASET))["title"] == "Towns of Japan"
570
+ assert head.tree is not None
571
+
572
+
573
+ def test_clone_keeps_a_description_already_here(tmp_path: Path, described: Path) -> None:
574
+ remote, _ = _published_with_a_readme(tmp_path)
575
+ described.write_text('{"name": "towns", "title": "Mine"}')
576
+ run("clone", remote, tmp_path / "again.sqlite")
577
+ assert json.loads(described.read_text())["title"] == "Mine"
578
+
579
+
580
+ def test_clone_refuses_to_overwrite_a_file_in_the_way(tmp_path: Path, monkeypatch) -> None:
581
+ remote, _ = _published_with_a_readme(tmp_path)
582
+ elsewhere = tmp_path / "elsewhere"
583
+ elsewhere.mkdir()
584
+ (elsewhere / "README.md").write_text("mine\n")
585
+ monkeypatch.chdir(elsewhere)
586
+ result = runner.invoke(app, ["clone", str(remote), "lake.sqlite"])
587
+ assert result.exit_code != 0
588
+ assert "README.md already here" in result.output
589
+ assert (elsewhere / "README.md").read_text() == "mine\n"
590
+ assert not (elsewhere / "lake.sqlite").exists()
@@ -272,6 +272,12 @@ def test_expire_marks_and_drops_the_metadata_but_keeps_the_record(tmp_path: Path
272
272
  repo.expire(remote, first.id)
273
273
 
274
274
 
275
+ def test_expire_records_who_took_the_version_out(tmp_path: Path, remote) -> None:
276
+ first, _ = pushed_chain(tmp_path, remote, 2)
277
+ repo.expire(remote, first.id, by="nightly retention")
278
+ assert json.loads(remote.get(layout.version_expired(first.id)))["by"] == "nightly retention"
279
+
280
+
275
281
  def test_a_version_a_ref_holds_is_not_expired(tmp_path: Path, remote) -> None:
276
282
  (first,) = pushed_chain(tmp_path, remote, 1)
277
283
  with pytest.raises(repo.RepoError, match="refs/heads/main"):
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes