fdl-cli 0.20.0__tar.gz → 0.20.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/CHANGELOG.md +9 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/PKG-INFO +1 -1
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/reference/cli.md +1 -1
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/spec/data-files.md +1 -1
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/reference/cli.md +1 -1
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/spec/data-files.md +1 -1
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/pyproject.toml +1 -1
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/cli.py +3 -1
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/repo.py +8 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/tests/test_http.py +2 -1
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/tests/test_repo.py +17 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/.github/workflows/docs.yml +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/.github/workflows/publish.yml +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/.github/workflows/release.yml +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/.gitignore +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/.node-version +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/.python-version +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/CLAUDE.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/CONTRIBUTING.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/LICENSE +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/README.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/blume.config.ts +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/concepts/meta.ts +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/concepts/repositories.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/concepts/versions-and-refs.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/concepts/working-catalogs.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/getting-started.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/guides/building-a-host.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/guides/meta.ts +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/guides/mirrors.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/guides/public-and-private.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/guides/reading-without-duckdb.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/guides/serving-over-http.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/guides/stores.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/index.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/internals/from-write-to-read.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/internals/inside-a-repository.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/internals/meta.ts +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/concepts/meta.ts +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/concepts/repositories.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/concepts/versions-and-refs.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/concepts/working-catalogs.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/getting-started.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/guides/building-a-host.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/guides/meta.ts +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/guides/mirrors.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/guides/public-and-private.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/guides/reading-without-duckdb.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/guides/serving-over-http.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/guides/stores.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/index.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/internals/from-write-to-read.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/internals/inside-a-repository.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/internals/meta.ts +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/reference/meta.ts +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/reference/python.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/spec/http.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/spec/index.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/spec/layout.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/spec/meta.ts +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/spec/reading.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/spec/refs.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/spec/versions.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/spec/writing.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/reference/meta.ts +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/reference/python.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/spec/http.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/spec/index.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/spec/layout.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/spec/meta.ts +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/spec/reading.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/spec/refs.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/spec/versions.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/spec/writing.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/package-lock.json +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/package.json +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/public/CNAME +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/__init__.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/catalog.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/dataset.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/layout.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/objects.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/profiles.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/py.typed +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/refs.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/store.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/stores.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/version.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/tests/http_repo.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/tests/test_cli.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/tests/test_dataset.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/tests/test_objects.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/tests/test_profiles.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/tests/test_stores.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.20.1}/uv.lock +0 -0
|
@@ -1,6 +1,15 @@
|
|
|
1
1
|
# CHANGELOG
|
|
2
2
|
|
|
3
3
|
|
|
4
|
+
## v0.20.1 (2026-11-06)
|
|
5
|
+
|
|
6
|
+
### Bug Fixes
|
|
7
|
+
|
|
8
|
+
- Withdraw a version already past retention by deleting unread files at once
|
|
9
|
+
([#40](https://github.com/queria-io/fdl/pull/40),
|
|
10
|
+
[`d9e3788`](https://github.com/queria-io/fdl/commit/d9e3788b6ffb429a9d301491a5e2a73eca8d891f))
|
|
11
|
+
|
|
12
|
+
|
|
4
13
|
## v0.20.0 (2026-11-06)
|
|
5
14
|
|
|
6
15
|
### Documentation
|
|
@@ -108,7 +108,7 @@ fdl tag <remote> <name> --delete
|
|
|
108
108
|
fdl withdraw <remote> <version>... [--yes]
|
|
109
109
|
```
|
|
110
110
|
|
|
111
|
-
間違って公開した個人データなど、公開してはいけないデータを持つバージョンを取り下げます([取り下げ](/ja/spec/data-files#取り下げ))。先にそのデータを除いたバージョンをコミットし、該当するバージョンを指すブランチが無い状態にします。そのうえでこのコマンドが、該当するバージョンを指すタグ、そのカタログが参照し残すバージョンが読まないデータファイル、カタログを消し、`"reason": "withdrawn"` の `expired.json` を置きます。`version.json`
|
|
111
|
+
間違って公開した個人データなど、公開してはいけないデータを持つバージョンを取り下げます([取り下げ](/ja/spec/data-files#取り下げ))。先にそのデータを除いたバージョンをコミットし、該当するバージョンを指すブランチが無い状態にします。そのうえでこのコマンドが、該当するバージョンを指すタグ、そのカタログが参照し残すバージョンが読まないデータファイル、カタログを消し、`"reason": "withdrawn"` の `expired.json` を置きます。`version.json` は残します。保持期間を過ぎたバージョンを含めると、そのカタログはもう無いので、有効なバージョンのどれも読まないデータファイルを、猶予期間を待たずにすべて消します。
|
|
112
112
|
|
|
113
113
|
アセットは、まだ消しません。データを持つものは `assets/` から手で消してください。
|
|
114
114
|
|
|
@@ -55,7 +55,7 @@ DuckLake 自身の片付け(`ducklake_cleanup_old_files`、`ducklake_expire_sn
|
|
|
55
55
|
|
|
56
56
|
1. そのデータを除いた新しいバージョンを作り、該当するバージョンを指しているブランチをすべてそこへ動かす。新しいバージョンの親はいまのバージョンなので、前へ動かすことになる
|
|
57
57
|
2. 該当するバージョンを指すタグを消す
|
|
58
|
-
3.
|
|
58
|
+
3. 個人情報を持つデータファイルや、それが写ったスクリーンショットなど、そのデータを持つ共有オブジェクトをすぐに消す。該当するバージョンから到達し、残すバージョンのどれからも到達しないものが対象。そのオブジェクトに到達するバージョンは、すべて該当するバージョンになる。該当するバージョンに保持期間を過ぎたものがあると、カタログが無いので、それがどのオブジェクトに到達していたかが分からない。その場合は、有効なバージョンのどれからも到達しない共有オブジェクトもすべてすぐに消す
|
|
59
59
|
4. 該当するバージョンそれぞれに `"reason": "withdrawn"` の `expired.json` を置き、カタログを消す
|
|
60
60
|
|
|
61
61
|
共有オブジェクトをカタログより先に消すのは、途中で止まった取り下げをやり直せるようにするためです。カタログが残っていれば、どのファイルがそのデータを持つかが分かります。
|
|
@@ -108,7 +108,7 @@ fdl tag <remote> <name> --delete
|
|
|
108
108
|
fdl withdraw <remote> <version>... [--yes]
|
|
109
109
|
```
|
|
110
110
|
|
|
111
|
-
Removes versions that hold data that must not be published, such as personal data published by mistake (see [Withdrawal](/spec/data-files#withdrawal)). First commit a version without the data, so that no branch points at an affected version. The command then deletes the tags that point at the versions, the data files their catalogs name that no kept version reads, and their catalogs, and puts `expired.json` with `"reason": "withdrawn"`. `version.json` stays.
|
|
111
|
+
Removes versions that hold data that must not be published, such as personal data published by mistake (see [Withdrawal](/spec/data-files#withdrawal)). First commit a version without the data, so that no branch points at an affected version. The command then deletes the tags that point at the versions, the data files their catalogs name that no kept version reads, and their catalogs, and puts `expired.json` with `"reason": "withdrawn"`. `version.json` stays. When one of the versions is already past retention, its catalog is gone, so every data file no live version reads is deleted at once, without waiting for the grace period.
|
|
112
112
|
|
|
113
113
|
Assets are not deleted yet: delete those that hold the data from `assets/` by hand.
|
|
114
114
|
|
|
@@ -55,7 +55,7 @@ Withdrawal removes data that must not be published, such as personal data publis
|
|
|
55
55
|
|
|
56
56
|
1. Make a new version without the data and move every branch that holds an affected version to it. This is a forward move: the new version's parent is the current one
|
|
57
57
|
2. Delete the tags that point at affected versions
|
|
58
|
-
3. Delete the shared objects that hold the data, at once, such as a data file with personal data or a screenshot showing it: those an affected version reaches and no version kept reaches. Every version that reaches such an object is an affected version
|
|
58
|
+
3. Delete the shared objects that hold the data, at once, such as a data file with personal data or a screenshot showing it: those an affected version reaches and no version kept reaches. Every version that reaches such an object is an affected version. An affected version already past retention has no catalog to say which objects it reached, so when there is one, every shared object no live version reaches is deleted at once too
|
|
59
59
|
4. For every affected version, put `expired.json` with `"reason": "withdrawn"` and delete its catalogs
|
|
60
60
|
|
|
61
61
|
The shared objects go before the catalogs, so a withdrawal that stops half way can be run again: the catalogs still say which files hold the data.
|
|
@@ -295,7 +295,9 @@ def withdraw(
|
|
|
295
295
|
|
|
296
296
|
Move every branch to a version without the data first. This deletes the
|
|
297
297
|
tags that point at the versions, the data files no kept version reads,
|
|
298
|
-
and the versions' catalogs, at once. Their version.json stays.
|
|
298
|
+
and the versions' catalogs, at once. Their version.json stays. A version
|
|
299
|
+
already past retention has no catalog, so then every data file no live
|
|
300
|
+
version reads is deleted.
|
|
299
301
|
"""
|
|
300
302
|
store = open_store(remote)
|
|
301
303
|
try:
|
|
@@ -655,6 +655,7 @@ def plan_withdrawal(store: Store, version_ids: Iterable[str]) -> Withdrawal:
|
|
|
655
655
|
|
|
656
656
|
candidates: set[str] = set()
|
|
657
657
|
kept: set[str] = set()
|
|
658
|
+
catalogless = False
|
|
658
659
|
with tempfile.TemporaryDirectory(prefix="fdl-") as tmp:
|
|
659
660
|
for version_id in _version_ids(store):
|
|
660
661
|
if version_id in affected:
|
|
@@ -664,6 +665,8 @@ def plan_withdrawal(store: Store, version_ids: Iterable[str]) -> Withdrawal:
|
|
|
664
665
|
# in it may hold the data too.
|
|
665
666
|
candidates.update(catalog.data_files(local))
|
|
666
667
|
local.unlink()
|
|
668
|
+
else:
|
|
669
|
+
catalogless = True
|
|
667
670
|
continue
|
|
668
671
|
if is_expired(store, version_id):
|
|
669
672
|
continue
|
|
@@ -677,6 +680,11 @@ def plan_withdrawal(store: Store, version_ids: Iterable[str]) -> Withdrawal:
|
|
|
677
680
|
if local is not None:
|
|
678
681
|
kept.update(_read_files(local, version))
|
|
679
682
|
local.unlink()
|
|
683
|
+
if catalogless:
|
|
684
|
+
# A version already past retention no longer says which files it
|
|
685
|
+
# named. Those no live version reads are waiting for the grace period
|
|
686
|
+
# anyway, so all of them go now.
|
|
687
|
+
candidates.update(key[len(config.data_dir) :] for key in store.list(config.data_dir))
|
|
680
688
|
return Withdrawal(
|
|
681
689
|
versions=affected,
|
|
682
690
|
tags=tuple(tags),
|
|
@@ -156,7 +156,8 @@ def test_a_file_the_host_will_not_sign_goes_through_the_host(tmp_path: Path) ->
|
|
|
156
156
|
|
|
157
157
|
assert local.get("data/small.parquet") == b"ab" and local.get("data/large.parquet") == b"abcdef"
|
|
158
158
|
puts = [key for method, key in server.requests if method == "PUT"]
|
|
159
|
-
|
|
159
|
+
# Uploads run at once, so they may arrive in any order.
|
|
160
|
+
assert sorted(puts) == ["data/large.parquet", "storage/data/small.parquet"]
|
|
160
161
|
|
|
161
162
|
|
|
162
163
|
def test_a_rebuild_of_the_same_rows_over_http_is_nothing_to_commit(served, tmp_path: Path) -> None:
|
|
@@ -387,6 +387,23 @@ def test_withdraw_removes_the_data_from_live_versions(tmp_path: Path, remote) ->
|
|
|
387
387
|
assert repo.head(remote) == clean
|
|
388
388
|
|
|
389
389
|
|
|
390
|
+
def test_withdrawing_a_version_past_retention_deletes_unread_files_at_once(tmp_path: Path, remote) -> None:
|
|
391
|
+
first = pushed(tmp_path, remote)
|
|
392
|
+
leaked = set(remote.list("data/"))
|
|
393
|
+
build(tmp_path / "work.sqlite", remote, "DROP TABLE lake.main.towns; CREATE TABLE lake.main.kept AS SELECT 1 AS id")
|
|
394
|
+
second = commit(tmp_path, remote, parent=first)
|
|
395
|
+
repo.push(remote, tmp_path / "local" / second.id, second)
|
|
396
|
+
kept = set(remote.list("data/")) - leaked
|
|
397
|
+
repo.expire(remote, first.id)
|
|
398
|
+
assert leaked <= set(remote.list("data/")) # waiting for the grace period
|
|
399
|
+
|
|
400
|
+
plan = repo.withdraw(remote, [first.id])
|
|
401
|
+
assert set(plan.deleted) == leaked
|
|
402
|
+
assert set(remote.list("data/")) == kept
|
|
403
|
+
assert json.loads(remote.get(layout.version_expired(first.id)))["reason"] == "withdrawn"
|
|
404
|
+
assert repo.head(remote) == second
|
|
405
|
+
|
|
406
|
+
|
|
390
407
|
def test_a_file_a_kept_version_still_reads_is_not_deleted(tmp_path: Path, remote) -> None:
|
|
391
408
|
first = pushed(tmp_path, remote)
|
|
392
409
|
second = pushed(tmp_path, remote, parent=first, sql="INSERT INTO lake.main.towns VALUES (2)")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|