fdl-cli 0.28.0__tar.gz → 0.29.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/CHANGELOG.md +22 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/PKG-INFO +1 -1
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/getting-started.md +2 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/getting-started.md +2 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/reference/cli.md +6 -4
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/spec/data-files.md +3 -2
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/reference/cli.md +6 -4
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/spec/data-files.md +3 -2
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/pyproject.toml +1 -1
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/cli.py +48 -2
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/repo.py +8 -5
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/tests/test_cli.py +53 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/tests/test_repo.py +6 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/.github/workflows/docs.yml +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/.github/workflows/publish.yml +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/.github/workflows/release.yml +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/.github/workflows/test.yml +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/.gitignore +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/.node-version +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/.python-version +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/CLAUDE.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/CONTRIBUTING.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/LICENSE +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/README.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/SECURITY.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/blume.config.ts +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/cases/collection.json +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/cases/data-dir.json +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/concepts/meta.ts +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/concepts/repositories.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/concepts/versions-and-refs.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/concepts/working-metadata.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/guides/building-a-host.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/guides/meta.ts +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/guides/mirrors.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/guides/public-and-private.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/guides/reading-without-duckdb.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/guides/serving-over-http.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/guides/stores.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/index.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/internals/from-write-to-read.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/internals/inside-a-repository.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/internals/meta.ts +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/concepts/meta.ts +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/concepts/repositories.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/concepts/versions-and-refs.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/concepts/working-metadata.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/guides/building-a-host.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/guides/meta.ts +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/guides/mirrors.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/guides/public-and-private.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/guides/reading-without-duckdb.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/guides/serving-over-http.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/guides/stores.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/index.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/internals/from-write-to-read.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/internals/inside-a-repository.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/internals/meta.ts +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/reference/meta.ts +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/reference/python.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/spec/collection-cases.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/spec/http.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/spec/index.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/spec/layout.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/spec/meta.ts +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/spec/reading.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/spec/refs.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/spec/versions.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/spec/writing.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/ja/stability.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/reference/meta.ts +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/reference/python.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/spec/collection-cases.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/spec/http.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/spec/index.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/spec/layout.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/spec/meta.ts +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/spec/reading.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/spec/refs.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/spec/versions.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/spec/writing.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/docs/stability.md +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/package-lock.json +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/package.json +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/public/CNAME +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/__init__.py +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/catalog.py +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/dataset.py +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/diff.py +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/layout.py +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/lineage.py +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/merge.py +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/objects.py +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/profiles.py +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/py.typed +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/refs.py +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/store.py +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/stores.py +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/src/fdl/version.py +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/tests/http_repo.py +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/tests/test_collection_cases.py +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/tests/test_data_dir_cases.py +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/tests/test_dataset.py +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/tests/test_http.py +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/tests/test_merge.py +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/tests/test_objects.py +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/tests/test_profiles.py +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/tests/test_stores.py +0 -0
- {fdl_cli-0.28.0 → fdl_cli-0.29.0}/uv.lock +0 -0
|
@@ -1,6 +1,28 @@
|
|
|
1
1
|
# CHANGELOG
|
|
2
2
|
|
|
3
3
|
|
|
4
|
+
## v0.29.0 (2026-11-15)
|
|
5
|
+
|
|
6
|
+
### Features
|
|
7
|
+
|
|
8
|
+
- Record who takes a version out in expired.json, and keep version.json on withdrawal
|
|
9
|
+
([#65](https://github.com/queria-io/fdl/pull/65),
|
|
10
|
+
[`1d3159b`](https://github.com/queria-io/fdl/commit/1d3159b4a50d309424b4b25a756477f7c225934b))
|
|
11
|
+
|
|
12
|
+
A withdrawal no longer deletes version.json: it rewrites the fields that hold the data, so every
|
|
13
|
+
repository holds its history back to the first version. expire and withdraw take --by, recorded in
|
|
14
|
+
expired.json.
|
|
15
|
+
|
|
16
|
+
- Write the version's dataset.json, README and docs/ on clone
|
|
17
|
+
([#64](https://github.com/queria-io/fdl/pull/64),
|
|
18
|
+
[`1f37b04`](https://github.com/queria-io/fdl/commit/1f37b047866e662ff967e4adda561ed5ba5018e1))
|
|
19
|
+
|
|
20
|
+
A clone made only the working metadata, so whoever cloned a dataset had to write its description
|
|
21
|
+
again before they could commit. Clone now writes the version's dataset.json (without what commit
|
|
22
|
+
fills in) and the files of its tree.json into the current folder, unless a description is already
|
|
23
|
+
there.
|
|
24
|
+
|
|
25
|
+
|
|
4
26
|
## v0.28.0 (2026-11-15)
|
|
5
27
|
|
|
6
28
|
### Bug Fixes
|
|
@@ -140,6 +140,8 @@ fdl commit work/lake.sqlite -m "Add 2026" --author alice
|
|
|
140
140
|
fdl push work/lake.sqlite
|
|
141
141
|
```
|
|
142
142
|
|
|
143
|
+
The current folder already holds `dataset.json`, so the clone leaves it as it is. In an empty folder, the clone also writes the version's `dataset.json`, `README.md` and `docs/` there, so whoever clones the repository can change the description as well as the data.
|
|
144
|
+
|
|
143
145
|
The new version lists the first as its parent. Both metadata files name the first Parquet file; only the second names the new one.
|
|
144
146
|
|
|
145
147
|
If someone else pushed in between, the push is refused, because the new version would not be built on what `main` holds now. Clone again and redo the change. [Versions and refs](/concepts/versions-and-refs) explains why.
|
|
@@ -140,6 +140,8 @@ fdl commit work/lake.sqlite -m "Add 2026" --author alice
|
|
|
140
140
|
fdl push work/lake.sqlite
|
|
141
141
|
```
|
|
142
142
|
|
|
143
|
+
今のフォルダには `dataset.json` があるので、クローンはそれをそのままにします。空のフォルダでは、バージョンの `dataset.json`・`README.md`・`docs/` もそこに書き出すので、リポジトリをクローンした人はデータだけでなく説明も変えられます。
|
|
144
|
+
|
|
143
145
|
新しいバージョンは、最初のバージョンを親として記録します。最初の Parquet ファイルは両方のメタデータが参照し、新しいファイルは 2 つめのメタデータだけが参照します。
|
|
144
146
|
|
|
145
147
|
途中でほかの人が push していたら、この push は断られます。新しいバージョンが、いまの `main` の上に積まれていないためです。クローンし直して、変更をやり直してください。理由は[バージョンと ref](/ja/concepts/versions-and-refs) で説明しています。
|
|
@@ -43,7 +43,9 @@ HTTP のリポジトリでは、新しいデータファイルは `<working.sqli
|
|
|
43
43
|
|
|
44
44
|
`--storage` を付けると、作業メタデータはリポジトリのストレージで直接データファイルを読み書きします。`--storage` にはそのストレージでのリポジトリの根を渡します。新しいファイルはその `data_dir` の下に直接書かれるので、push でアップロードするものはありません。鍵は `AWS_*` の環境変数(R2 などの S3 互換ストレージは `AWS_ENDPOINT_URL`)か、自分で作った DuckDB の secret から取ります。作業メタデータに書くビルドにも同じ鍵が要ります。
|
|
45
45
|
|
|
46
|
-
|
|
46
|
+
今のフォルダに `dataset.json` も `dataset.yml` も無いときは、`fdl commit` が読むファイルもそこに書き出します。バージョンの `dataset.json` からコミットがカタログから埋める値(行数、列の型、NULL の可否)を除いたものと、`tree.json` が挙げるファイル(`README.md` や `docs/` など)です。これらを編集してコミットすると説明を変えられます。フォルダに既に説明があるときはそのままにし、何も書きません。
|
|
47
|
+
|
|
48
|
+
バージョンの保持期間が過ぎているとき、メタデータの sha256 が `version.json` の記録と違うとき、メタデータがデータフォルダの外のデータファイルを指しているとき、書き出すファイルが今のフォルダに既にあるときは断られます。
|
|
47
49
|
|
|
48
50
|
## fdl commit
|
|
49
51
|
|
|
@@ -211,7 +213,7 @@ fdl push <working.sqlite>
|
|
|
211
213
|
## fdl withdraw
|
|
212
214
|
|
|
213
215
|
```bash
|
|
214
|
-
fdl withdraw <remote> <version>... [--yes]
|
|
216
|
+
fdl withdraw <remote> <version>... [--yes] [--by <name>]
|
|
215
217
|
```
|
|
216
218
|
|
|
217
219
|
間違って公開した個人データなど、公開してはいけないデータを持つバージョンを取り下げます([取り下げ](/ja/spec/data-files#取り下げ))。先にそのデータを除いたバージョンをコミットし、該当するバージョンを指すブランチが無い状態にします。そのうえでこのコマンドが、該当するバージョンを指すタグ、そのバージョンが参照し残すバージョンが参照しないデータファイルとアセット、メタデータを消し、`"reason": "withdrawn"` の `expired.json` を置きます。`version.json` は残します。保持期間を過ぎたバージョンを含めると、そのメタデータはもう無いので、有効なバージョンのどれも参照しないデータファイルとアセットを、猶予期間を待たずにすべて消します。
|
|
@@ -231,10 +233,10 @@ fdl profile <remote> [--ref refs/heads/main]
|
|
|
231
233
|
## fdl expire
|
|
232
234
|
|
|
233
235
|
```bash
|
|
234
|
-
fdl expire <remote> [--keep 1] [--since 2026-10-01T00:00:00Z] [--ref refs/heads/main]
|
|
236
|
+
fdl expire <remote> [--keep 1] [--since 2026-10-01T00:00:00Z] [--ref refs/heads/main] [--by <name>]
|
|
235
237
|
```
|
|
236
238
|
|
|
237
|
-
ブランチの最初の親のつながりで、新しい `--keep` 個より古いバージョンの保持期間を終わらせます。`expired.json` を置き、メタデータを消します。`--since` 以降に作ったバージョンと、ref が指すバージョンは残します。データファイルは消しません。消すのは `fdl gc`
|
|
239
|
+
ブランチの最初の親のつながりで、新しい `--keep` 個より古いバージョンの保持期間を終わらせます。`expired.json` を置き、メタデータを消します。`--since` 以降に作ったバージョンと、ref が指すバージョンは残します。データファイルは消しません。消すのは `fdl gc` です。`--by` は保持期間を終わらせた人を `expired.json` に記録します。`fdl withdraw` の `--by` も同じです。
|
|
238
240
|
|
|
239
241
|
## fdl gc
|
|
240
242
|
|
|
@@ -61,9 +61,10 @@ DuckLake 自身の片付け(`ducklake_cleanup_old_files`、`ducklake_expire_sn
|
|
|
61
61
|
3. 猶予期間のあと、上の条件に合う共有オブジェクトを消す
|
|
62
62
|
|
|
63
63
|
```json expired.json
|
|
64
|
-
{ "expired_at": "2026-12-01T00:00:00Z", "reason": "retention" }
|
|
64
|
+
{ "expired_at": "2026-12-01T00:00:00Z", "reason": "retention", "by": "nightly retention" }
|
|
65
65
|
```
|
|
66
66
|
|
|
67
|
+
- `by` には、保持期間を終わらせた人や仕組み(定期実行のジョブなど)を書いてよい。これで履歴には、バージョンを作った人と同じく、そのデータを消した人も残る
|
|
67
68
|
- `version.json` は残す。履歴にはそのバージョンがあったことと中身が残り、親のつながりも切れない
|
|
68
69
|
- `expired.json` を見た読み手は、`reason` が何であれ、保持期間を過ぎたものとして扱い、メタデータを開かない
|
|
69
70
|
- ref が指すバージョンの保持期間を終わらせてはいけない
|
|
@@ -83,5 +84,5 @@ DuckLake 自身の片付け(`ducklake_cleanup_old_files`、`ducklake_expire_sn
|
|
|
83
84
|
{ "expired_at": "2026-12-01T00:00:00Z", "reason": "withdrawn" }
|
|
84
85
|
```
|
|
85
86
|
|
|
86
|
-
- `version.json`
|
|
87
|
+
- `version.json` は、保持期間のときと同じく残す。これで、どのリポジトリも最初のバージョンまでの履歴を持つ。記録そのもの(`author`・`message`・`inputs`)にそのデータがあれば、`expired.json` を置く前に記録を書き換えてデータを除く。`author` や `message` の該当する値は `withdrawn` に置き換え、`inputs` は該当する要素を除く。`id`・`seq`・`parents` などほかの項目はそのまま残す
|
|
87
88
|
- ミラーは自分の写しを持っている。`"reason": "withdrawn"` の `expired.json` を写したミラーは、元が消したメタデータと共有オブジェクトを消す。元のホストはこれを強制できないので、知っているミラーに伝える
|
|
@@ -43,7 +43,9 @@ For a repository over HTTP, new data files are written to `<working.sqlite>.data
|
|
|
43
43
|
|
|
44
44
|
With `--storage`, the working metadata reads and writes the data files in the repository's own storage instead: `--storage` is the repository's root there, and new files go straight under its `data_dir`, so the push has none to upload. The keys come from the `AWS_*` variables (`AWS_ENDPOINT_URL` for R2 and other S3-compatible stores), or from a DuckDB secret of your own. The build that writes to the working metadata needs them too.
|
|
45
45
|
|
|
46
|
-
|
|
46
|
+
When the current folder holds no `dataset.json` or `dataset.yml`, it also writes there what `fdl commit` reads: the version's `dataset.json`, without what the commit fills in from the catalog (row counts, column types, nulls), and the files its `tree.json` lists, such as `README.md` and `docs/`. Edit them and commit to change the description. When the folder already holds a description, it is left as it is and nothing is written.
|
|
47
|
+
|
|
48
|
+
Refused when the version is past retention, when its metadata's sha256 is not the one `version.json` records, when the metadata names a data file outside the data folder, or when a file it would write is already in the current folder.
|
|
47
49
|
|
|
48
50
|
## fdl commit
|
|
49
51
|
|
|
@@ -211,7 +213,7 @@ Refused when the working metadata has changes not committed, when a version is c
|
|
|
211
213
|
## fdl withdraw
|
|
212
214
|
|
|
213
215
|
```bash
|
|
214
|
-
fdl withdraw <remote> <version>... [--yes]
|
|
216
|
+
fdl withdraw <remote> <version>... [--yes] [--by <name>]
|
|
215
217
|
```
|
|
216
218
|
|
|
217
219
|
Removes versions that hold data that must not be published, such as personal data published by mistake (see [Withdrawal](/spec/data-files#withdrawal)). First commit a version without the data, so that no branch points at an affected version. The command then deletes the tags that point at the versions, the data files and assets the versions reach that no kept version reaches, and their metadata files, and puts `expired.json` with `"reason": "withdrawn"`. `version.json` stays. When one of the versions is already past retention, its metadata is gone, so every data file and asset no live version reaches is deleted at once, without waiting for the grace period.
|
|
@@ -231,10 +233,10 @@ Profiles the tables and views of the version the branch or tag holds that have n
|
|
|
231
233
|
## fdl expire
|
|
232
234
|
|
|
233
235
|
```bash
|
|
234
|
-
fdl expire <remote> [--keep 1] [--since 2026-10-01T00:00:00Z] [--ref refs/heads/main]
|
|
236
|
+
fdl expire <remote> [--keep 1] [--since 2026-10-01T00:00:00Z] [--ref refs/heads/main] [--by <name>]
|
|
235
237
|
```
|
|
236
238
|
|
|
237
|
-
Takes versions behind the newest `--keep` on the branch's first-parent line out of retention: puts `expired.json` and deletes their metadata files. Versions made at or after `--since`, and any a ref points at, are kept. Data files are not deleted; `fdl gc` deletes them.
|
|
239
|
+
Takes versions behind the newest `--keep` on the branch's first-parent line out of retention: puts `expired.json` and deletes their metadata files. Versions made at or after `--since`, and any a ref points at, are kept. Data files are not deleted; `fdl gc` deletes them. `--by` records who takes the versions out in `expired.json`, as does `--by` of `fdl withdraw`.
|
|
238
240
|
|
|
239
241
|
## fdl gc
|
|
240
242
|
|
|
@@ -61,9 +61,10 @@ A version is taken out of retention in this order:
|
|
|
61
61
|
3. After the grace period, delete the shared objects that meet the conditions above
|
|
62
62
|
|
|
63
63
|
```json expired.json
|
|
64
|
-
{ "expired_at": "2026-12-01T00:00:00Z", "reason": "retention" }
|
|
64
|
+
{ "expired_at": "2026-12-01T00:00:00Z", "reason": "retention", "by": "nightly retention" }
|
|
65
65
|
```
|
|
66
66
|
|
|
67
|
+
- `by` may say who or what took the version out: a person, or a process such as a scheduled job. With it, the history shows who removed each version's data, as it shows who made it
|
|
67
68
|
- `version.json` stays, so the history still says the version was there and what it was, and the graph of parents stays whole
|
|
68
69
|
- A reader that sees `expired.json` treats the version as past retention and does not open its metadata, whatever the `reason`
|
|
69
70
|
- A version a ref points at must not be taken out
|
|
@@ -83,5 +84,5 @@ The shared objects go before the metadata files, so a withdrawal that stops half
|
|
|
83
84
|
{ "expired_at": "2026-12-01T00:00:00Z", "reason": "withdrawn" }
|
|
84
85
|
```
|
|
85
86
|
|
|
86
|
-
- `version.json` stays, as with retention. If the record itself holds the data (in `message` or `inputs`), it is
|
|
87
|
+
- `version.json` stays, as with retention, so every repository holds its history back to the first version. If the record itself holds the data (in `author`, `message` or `inputs`), it is rewritten without it before `expired.json` is put: the value in `author` or `message` is replaced with `withdrawn`, and the entries of `inputs` that hold it are dropped. `id`, `seq`, `parents` and the other fields stay as they were
|
|
87
88
|
- Mirrors keep their own copies. A mirror that copies `expired.json` with `"reason": "withdrawn"` deletes the metadata files and the shared objects the origin deleted. The origin cannot force this; the host tells the mirrors it knows of
|
|
@@ -10,6 +10,7 @@ pushed.
|
|
|
10
10
|
from __future__ import annotations
|
|
11
11
|
|
|
12
12
|
import datetime as dt
|
|
13
|
+
import hashlib
|
|
13
14
|
import json
|
|
14
15
|
import os
|
|
15
16
|
import re
|
|
@@ -122,7 +123,11 @@ def clone(
|
|
|
122
123
|
try:
|
|
123
124
|
# Read before the clone: a data_dir moved meanwhile then fails the commit.
|
|
124
125
|
config = repo.read_config(store)
|
|
126
|
+
files = _described_files(store, ref)
|
|
125
127
|
version = repo.clone(store, working, ref=ref, local_data=local, storage=storage)
|
|
128
|
+
for path, body in files.items():
|
|
129
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
130
|
+
path.write_bytes(body)
|
|
126
131
|
if storage:
|
|
127
132
|
data = repo.storage_data_path(storage, config)
|
|
128
133
|
else:
|
|
@@ -136,10 +141,49 @@ def clone(
|
|
|
136
141
|
state["storage"] = storage
|
|
137
142
|
_write_state(working, state)
|
|
138
143
|
typer.echo(f"{working} at {version.id if version else 'empty metadata'}")
|
|
144
|
+
if files:
|
|
145
|
+
typer.echo("Wrote its description here: " + ", ".join(sorted(path.as_posix() for path in files)))
|
|
139
146
|
typer.echo("Write to it with:")
|
|
140
147
|
typer.echo(" " + catalog.attach_sql(working, alias="lake", data_path=data)[-1] + ";")
|
|
141
148
|
|
|
142
149
|
|
|
150
|
+
def _described_files(store, ref: str) -> dict[Path, bytes]:
|
|
151
|
+
"""What `fdl commit` reads from here, as the version ``ref`` holds it: its
|
|
152
|
+
`dataset.json` (without what the commit fills in) and the files of its `tree.json`.
|
|
153
|
+
|
|
154
|
+
Nothing when the branch has no version yet, or when a description is
|
|
155
|
+
already here: that one is the writer's, and the commit reads it. A file in
|
|
156
|
+
the way is refused rather than overwritten.
|
|
157
|
+
"""
|
|
158
|
+
version = repo.head(store, ref)
|
|
159
|
+
if version is None or any(Path(name).exists() for name in dataset.DEFAULT_FILES):
|
|
160
|
+
return {}
|
|
161
|
+
described = diffs.described(store, version)
|
|
162
|
+
if described is None:
|
|
163
|
+
return {}
|
|
164
|
+
entries = (diffs.listed(store, version) or {}).get("entries", [])
|
|
165
|
+
objects.check_tree(entries)
|
|
166
|
+
files = {
|
|
167
|
+
Path(layout.DATASET): dataset.to_json(merging.declared(described)),
|
|
168
|
+
**{
|
|
169
|
+
Path(item["path"]): _tree_file(store, item)
|
|
170
|
+
for item in entries
|
|
171
|
+
},
|
|
172
|
+
}
|
|
173
|
+
present = sorted(path.as_posix() for path in files if path.exists())
|
|
174
|
+
if present:
|
|
175
|
+
raise repo.RepoError(f"{', '.join(present)} already here: clone into a folder without them")
|
|
176
|
+
return files
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _tree_file(store, item: dict) -> bytes:
|
|
180
|
+
"""A file of `tree.json`, checked against its sha256."""
|
|
181
|
+
body = store.get(objects.asset_key(item["sha256"]))
|
|
182
|
+
if hashlib.sha256(body).hexdigest() != item["sha256"]:
|
|
183
|
+
raise repo.RepoError(f"{item['path']}: the file's sha256 is not the one tree.json records")
|
|
184
|
+
return body
|
|
185
|
+
|
|
186
|
+
|
|
143
187
|
@app.command()
|
|
144
188
|
def commit(
|
|
145
189
|
working: Path = Working,
|
|
@@ -912,6 +956,7 @@ def expire(
|
|
|
912
956
|
keep: int = typer.Option(1, help="How many of the newest versions to keep"),
|
|
913
957
|
since: str | None = typer.Option(None, help="Also keep those made at or after this time (ISO 8601)"),
|
|
914
958
|
ref: str = typer.Option(layout.DEFAULT_BRANCH),
|
|
959
|
+
by: str | None = typer.Option(None, help="Who or what takes them out, recorded in expired.json"),
|
|
915
960
|
) -> None:
|
|
916
961
|
"""Take older versions out of retention: mark them and drop their catalogs.
|
|
917
962
|
|
|
@@ -920,7 +965,7 @@ def expire(
|
|
|
920
965
|
store = _store(remote)
|
|
921
966
|
try:
|
|
922
967
|
newest = repo.head(store, ref)
|
|
923
|
-
expired = repo.prune(store, newest, keep=keep, since=since) if newest else []
|
|
968
|
+
expired = repo.prune(store, newest, keep=keep, since=since, by=by) if newest else []
|
|
924
969
|
except ERRORS as exc:
|
|
925
970
|
raise _fail(exc) from exc
|
|
926
971
|
typer.echo(f"Expired {len(expired)} version(s).")
|
|
@@ -965,6 +1010,7 @@ def withdraw(
|
|
|
965
1010
|
remote: str = Remote,
|
|
966
1011
|
versions: list[str] = typer.Argument(..., help="The versions that hold the data"),
|
|
967
1012
|
yes: bool = typer.Option(False, "--yes", help="Do it. Without this, only say what would be done"),
|
|
1013
|
+
by: str | None = typer.Option(None, help="Who withdraws them, recorded in expired.json"),
|
|
968
1014
|
) -> None:
|
|
969
1015
|
"""Remove versions that hold data that must not be published.
|
|
970
1016
|
|
|
@@ -988,7 +1034,7 @@ def withdraw(
|
|
|
988
1034
|
if not yes:
|
|
989
1035
|
typer.echo("Nothing was changed. Run again with --yes to do this.")
|
|
990
1036
|
return
|
|
991
|
-
repo.withdraw(store, versions, plan=plan)
|
|
1037
|
+
repo.withdraw(store, versions, plan=plan, by=by)
|
|
992
1038
|
except ERRORS as exc:
|
|
993
1039
|
raise _fail(exc) from exc
|
|
994
1040
|
typer.echo(f"Withdrew {len(plan.versions)} version(s) and deleted {len(plan.deleted)} file(s).")
|
|
@@ -769,7 +769,7 @@ def is_expired(store: Store, version_id: str) -> bool:
|
|
|
769
769
|
return store.head(layout.version_expired(version_id)) is not None
|
|
770
770
|
|
|
771
771
|
|
|
772
|
-
def expire(store: Store, version_id: str, *, reason: str = "retention") -> None:
|
|
772
|
+
def expire(store: Store, version_id: str, *, reason: str = "retention", by: str | None = None) -> None:
|
|
773
773
|
"""Take a version out of retention: the mark first, then its catalogs.
|
|
774
774
|
|
|
775
775
|
`version.json` stays, so the history still says the version was there and
|
|
@@ -781,7 +781,7 @@ def expire(store: Store, version_id: str, *, reason: str = "retention") -> None:
|
|
|
781
781
|
held = _held(store)
|
|
782
782
|
if version_id in held:
|
|
783
783
|
raise RepoError(f"{held[version_id]} points at {version_id}, so it is kept")
|
|
784
|
-
record = {"expired_at": utc_now(), "reason": reason}
|
|
784
|
+
record = {"expired_at": utc_now(), "reason": reason, **({"by": by} if by else {})}
|
|
785
785
|
try:
|
|
786
786
|
store.put(
|
|
787
787
|
layout.version_expired(version_id),
|
|
@@ -1014,7 +1014,9 @@ def _reached_assets(store: Store, version: Version, local: Path | None, setup: l
|
|
|
1014
1014
|
return reached
|
|
1015
1015
|
|
|
1016
1016
|
|
|
1017
|
-
def withdraw(
|
|
1017
|
+
def withdraw(
|
|
1018
|
+
store: Store, version_ids: Iterable[str], *, plan: Withdrawal | None = None, by: str | None = None
|
|
1019
|
+
) -> Withdrawal:
|
|
1018
1020
|
"""Remove versions that hold data that must not be published.
|
|
1019
1021
|
|
|
1020
1022
|
Unlike retention it takes live versions and deletes their data files and
|
|
@@ -1033,7 +1035,7 @@ def withdraw(store: Store, version_ids: Iterable[str], *, plan: Withdrawal | Non
|
|
|
1033
1035
|
store.delete(assets)
|
|
1034
1036
|
store.delete([key for key in plan.deleted if not key.startswith(layout.ASSETS_DIR)])
|
|
1035
1037
|
for version_id in plan.versions:
|
|
1036
|
-
expire(store, version_id, reason="withdrawn")
|
|
1038
|
+
expire(store, version_id, reason="withdrawn", by=by)
|
|
1037
1039
|
return plan
|
|
1038
1040
|
|
|
1039
1041
|
|
|
@@ -1080,6 +1082,7 @@ def prune(
|
|
|
1080
1082
|
keep: int = 1,
|
|
1081
1083
|
since: str | None = None,
|
|
1082
1084
|
reason: str = "retention",
|
|
1085
|
+
by: str | None = None,
|
|
1083
1086
|
) -> list[str]:
|
|
1084
1087
|
"""Expire the versions behind ``newest`` that fall out of retention.
|
|
1085
1088
|
|
|
@@ -1099,7 +1102,7 @@ def prune(
|
|
|
1099
1102
|
break
|
|
1100
1103
|
recent = since is not None and _instant(current.created_at) >= _instant(since)
|
|
1101
1104
|
if position >= keep and not recent and current.id not in held:
|
|
1102
|
-
expire(store, current.id, reason=reason)
|
|
1105
|
+
expire(store, current.id, reason=reason, by=by)
|
|
1103
1106
|
expired.append(current.id)
|
|
1104
1107
|
current = read_version(store, current.parents[0]) if current.parents else None
|
|
1105
1108
|
return expired
|
|
@@ -535,3 +535,56 @@ def test_clone_names_a_folder_given_as_dot_by_where_it_is(tmp_path: Path, monkey
|
|
|
535
535
|
monkeypatch.chdir(tmp_path)
|
|
536
536
|
assert _default_working(".") == Path(f"{tmp_path.name}.sqlite")
|
|
537
537
|
assert _default_working("https://data.example.invalid/datasets/alice/towns/") == Path("towns.sqlite")
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def _published_with_a_readme(tmp_path: Path) -> tuple[Path, LocalStore]:
|
|
541
|
+
remote, working = tmp_path / "remote", tmp_path / "work" / "lake.sqlite"
|
|
542
|
+
store = LocalStore(remote)
|
|
543
|
+
(tmp_path / "README.md").write_text("# Towns\n")
|
|
544
|
+
(tmp_path / "docs").mkdir()
|
|
545
|
+
(tmp_path / "docs" / "codes.md").write_text("Codes\n")
|
|
546
|
+
run("init", remote)
|
|
547
|
+
run("clone", remote, working)
|
|
548
|
+
write(working, store, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
|
|
549
|
+
commit(working, "-m", "first", "--author", "tester")
|
|
550
|
+
return remote, store
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
def test_clone_writes_the_description_to_edit_and_commit(tmp_path: Path, monkeypatch) -> None:
|
|
554
|
+
remote, store = _published_with_a_readme(tmp_path)
|
|
555
|
+
elsewhere = tmp_path / "elsewhere"
|
|
556
|
+
elsewhere.mkdir()
|
|
557
|
+
monkeypatch.chdir(elsewhere)
|
|
558
|
+
run("clone", remote, "lake.sqlite")
|
|
559
|
+
|
|
560
|
+
written = json.loads((elsewhere / "dataset.json").read_text())
|
|
561
|
+
assert written["title"] == "Towns"
|
|
562
|
+
assert "rows" not in written["tables"][0]
|
|
563
|
+
assert (elsewhere / "README.md").read_text() == "# Towns\n"
|
|
564
|
+
assert (elsewhere / "docs" / "codes.md").read_text() == "Codes\n"
|
|
565
|
+
|
|
566
|
+
(elsewhere / "dataset.json").write_text(json.dumps({**written, "title": "Towns of Japan"}))
|
|
567
|
+
commit(elsewhere / "lake.sqlite", "-m", "retitle", "--author", "tester")
|
|
568
|
+
head = repo.head(store)
|
|
569
|
+
assert json.loads(store.get(layout.version_dir(head.id) + layout.DATASET))["title"] == "Towns of Japan"
|
|
570
|
+
assert head.tree is not None
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
def test_clone_keeps_a_description_already_here(tmp_path: Path, described: Path) -> None:
|
|
574
|
+
remote, _ = _published_with_a_readme(tmp_path)
|
|
575
|
+
described.write_text('{"name": "towns", "title": "Mine"}')
|
|
576
|
+
run("clone", remote, tmp_path / "again.sqlite")
|
|
577
|
+
assert json.loads(described.read_text())["title"] == "Mine"
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
def test_clone_refuses_to_overwrite_a_file_in_the_way(tmp_path: Path, monkeypatch) -> None:
|
|
581
|
+
remote, _ = _published_with_a_readme(tmp_path)
|
|
582
|
+
elsewhere = tmp_path / "elsewhere"
|
|
583
|
+
elsewhere.mkdir()
|
|
584
|
+
(elsewhere / "README.md").write_text("mine\n")
|
|
585
|
+
monkeypatch.chdir(elsewhere)
|
|
586
|
+
result = runner.invoke(app, ["clone", str(remote), "lake.sqlite"])
|
|
587
|
+
assert result.exit_code != 0
|
|
588
|
+
assert "README.md already here" in result.output
|
|
589
|
+
assert (elsewhere / "README.md").read_text() == "mine\n"
|
|
590
|
+
assert not (elsewhere / "lake.sqlite").exists()
|
|
@@ -272,6 +272,12 @@ def test_expire_marks_and_drops_the_metadata_but_keeps_the_record(tmp_path: Path
|
|
|
272
272
|
repo.expire(remote, first.id)
|
|
273
273
|
|
|
274
274
|
|
|
275
|
+
def test_expire_records_who_took_the_version_out(tmp_path: Path, remote) -> None:
|
|
276
|
+
first, _ = pushed_chain(tmp_path, remote, 2)
|
|
277
|
+
repo.expire(remote, first.id, by="nightly retention")
|
|
278
|
+
assert json.loads(remote.get(layout.version_expired(first.id)))["by"] == "nightly retention"
|
|
279
|
+
|
|
280
|
+
|
|
275
281
|
def test_a_version_a_ref_holds_is_not_expired(tmp_path: Path, remote) -> None:
|
|
276
282
|
(first,) = pushed_chain(tmp_path, remote, 1)
|
|
277
283
|
with pytest.raises(repo.RepoError, match="refs/heads/main"):
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|