fdl-cli 0.20.0__tar.gz → 0.20.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/CHANGELOG.md +9 -0
  2. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/PKG-INFO +1 -1
  3. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/reference/cli.md +1 -1
  4. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/spec/data-files.md +1 -1
  5. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/reference/cli.md +1 -1
  6. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/spec/data-files.md +1 -1
  7. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/pyproject.toml +1 -1
  8. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/cli.py +3 -1
  9. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/repo.py +8 -0
  10. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/tests/test_http.py +2 -1
  11. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/tests/test_repo.py +17 -0
  12. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/.github/workflows/docs.yml +0 -0
  13. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/.github/workflows/publish.yml +0 -0
  14. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/.github/workflows/release.yml +0 -0
  15. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/.gitignore +0 -0
  16. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/.node-version +0 -0
  17. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/.python-version +0 -0
  18. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/CLAUDE.md +0 -0
  19. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/CONTRIBUTING.md +0 -0
  20. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/LICENSE +0 -0
  21. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/README.md +0 -0
  22. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/blume.config.ts +0 -0
  23. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/concepts/meta.ts +0 -0
  24. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/concepts/repositories.md +0 -0
  25. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/concepts/versions-and-refs.md +0 -0
  26. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/concepts/working-catalogs.md +0 -0
  27. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/getting-started.md +0 -0
  28. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/guides/building-a-host.md +0 -0
  29. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/guides/meta.ts +0 -0
  30. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/guides/mirrors.md +0 -0
  31. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/guides/public-and-private.md +0 -0
  32. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/guides/reading-without-duckdb.md +0 -0
  33. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/guides/serving-over-http.md +0 -0
  34. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/guides/stores.md +0 -0
  35. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/index.md +0 -0
  36. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/internals/ducklake-formats.md +0 -0
  37. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/internals/from-write-to-read.md +0 -0
  38. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/internals/inside-a-repository.md +0 -0
  39. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/internals/meta.ts +0 -0
  40. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/concepts/meta.ts +0 -0
  41. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/concepts/repositories.md +0 -0
  42. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/concepts/versions-and-refs.md +0 -0
  43. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/concepts/working-catalogs.md +0 -0
  44. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/getting-started.md +0 -0
  45. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/guides/building-a-host.md +0 -0
  46. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/guides/meta.ts +0 -0
  47. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/guides/mirrors.md +0 -0
  48. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/guides/public-and-private.md +0 -0
  49. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/guides/reading-without-duckdb.md +0 -0
  50. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/guides/serving-over-http.md +0 -0
  51. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/guides/stores.md +0 -0
  52. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/index.md +0 -0
  53. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/internals/ducklake-formats.md +0 -0
  54. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/internals/from-write-to-read.md +0 -0
  55. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/internals/inside-a-repository.md +0 -0
  56. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/internals/meta.ts +0 -0
  57. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/reference/meta.ts +0 -0
  58. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/reference/python.md +0 -0
  59. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/spec/http.md +0 -0
  60. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/spec/index.md +0 -0
  61. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/spec/layout.md +0 -0
  62. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/spec/meta.ts +0 -0
  63. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/spec/reading.md +0 -0
  64. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/spec/refs.md +0 -0
  65. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/spec/versions.md +0 -0
  66. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/ja/spec/writing.md +0 -0
  67. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/reference/meta.ts +0 -0
  68. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/reference/python.md +0 -0
  69. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/spec/http.md +0 -0
  70. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/spec/index.md +0 -0
  71. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/spec/layout.md +0 -0
  72. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/spec/meta.ts +0 -0
  73. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/spec/reading.md +0 -0
  74. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/spec/refs.md +0 -0
  75. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/spec/versions.md +0 -0
  76. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/docs/spec/writing.md +0 -0
  77. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/package-lock.json +0 -0
  78. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/package.json +0 -0
  79. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/public/CNAME +0 -0
  80. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/__init__.py +0 -0
  81. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/catalog.py +0 -0
  82. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/dataset.py +0 -0
  83. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/layout.py +0 -0
  84. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/objects.py +0 -0
  85. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/profiles.py +0 -0
  86. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/py.typed +0 -0
  87. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/refs.py +0 -0
  88. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/store.py +0 -0
  89. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/stores.py +0 -0
  90. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/src/fdl/version.py +0 -0
  91. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/tests/http_repo.py +0 -0
  92. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/tests/test_cli.py +0 -0
  93. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/tests/test_dataset.py +0 -0
  94. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/tests/test_objects.py +0 -0
  95. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/tests/test_profiles.py +0 -0
  96. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/tests/test_stores.py +0 -0
  97. {fdl_cli-0.20.0 → fdl_cli-0.20.1}/uv.lock +0 -0
@@ -1,6 +1,15 @@
1
1
  # CHANGELOG
2
2
 
3
3
 
4
+ ## v0.20.1 (2026-11-06)
5
+
6
+ ### Bug Fixes
7
+
8
+ - Withdraw a version already past retention by deleting unread files at once
9
+ ([#40](https://github.com/queria-io/fdl/pull/40),
10
+ [`d9e3788`](https://github.com/queria-io/fdl/commit/d9e3788b6ffb429a9d301491a5e2a73eca8d891f))
11
+
12
+
4
13
  ## v0.20.0 (2026-11-06)
5
14
 
6
15
  ### Documentation
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: fdl-cli
3
- Version: 0.20.0
3
+ Version: 0.20.1
4
4
  Summary: Versions of a DuckLake catalog on object storage, kept the way Git keeps commits
5
5
  Project-URL: Homepage, https://fdl.flo8s.com/
6
6
  Project-URL: Documentation, https://fdl.flo8s.com/
@@ -108,7 +108,7 @@ fdl tag <remote> <name> --delete
108
108
  fdl withdraw <remote> <version>... [--yes]
109
109
  ```
110
110
 
111
- 間違って公開した個人データなど、公開してはいけないデータを持つバージョンを取り下げます([取り下げ](/ja/spec/data-files#取り下げ))。先にそのデータを除いたバージョンをコミットし、該当するバージョンを指すブランチが無い状態にします。そのうえでこのコマンドが、該当するバージョンを指すタグ、そのカタログが参照し残すバージョンが読まないデータファイル、カタログを消し、`"reason": "withdrawn"` の `expired.json` を置きます。`version.json` は残します。
111
+ 間違って公開した個人データなど、公開してはいけないデータを持つバージョンを取り下げます([取り下げ](/ja/spec/data-files#取り下げ))。先にそのデータを除いたバージョンをコミットし、該当するバージョンを指すブランチが無い状態にします。そのうえでこのコマンドが、該当するバージョンを指すタグ、そのカタログが参照し残すバージョンが読まないデータファイル、カタログを消し、`"reason": "withdrawn"` の `expired.json` を置きます。`version.json` は残します。保持期間を過ぎたバージョンを含めると、そのカタログはもう無いので、有効なバージョンのどれも読まないデータファイルを、猶予期間を待たずにすべて消します。
112
112
 
113
113
  アセットは、まだ消しません。データを持つものは `assets/` から手で消してください。
114
114
 
@@ -55,7 +55,7 @@ DuckLake 自身の片付け(`ducklake_cleanup_old_files`、`ducklake_expire_sn
55
55
 
56
56
  1. そのデータを除いた新しいバージョンを作り、該当するバージョンを指しているブランチをすべてそこへ動かす。新しいバージョンの親はいまのバージョンなので、前へ動かすことになる
57
57
  2. 該当するバージョンを指すタグを消す
58
- 3. 個人情報を持つデータファイルや、それが写ったスクリーンショットなど、そのデータを持つ共有オブジェクトをすぐに消す。該当するバージョンから到達し、残すバージョンのどれからも到達しないものが対象。そのオブジェクトに到達するバージョンは、すべて該当するバージョンになる
58
+ 3. 個人情報を持つデータファイルや、それが写ったスクリーンショットなど、そのデータを持つ共有オブジェクトをすぐに消す。該当するバージョンから到達し、残すバージョンのどれからも到達しないものが対象。そのオブジェクトに到達するバージョンは、すべて該当するバージョンになる。該当するバージョンに保持期間を過ぎたものがあると、カタログが無いので、それがどのオブジェクトに到達していたかが分からない。その場合は、有効なバージョンのどれからも到達しない共有オブジェクトもすべてすぐに消す
59
59
  4. 該当するバージョンそれぞれに `"reason": "withdrawn"` の `expired.json` を置き、カタログを消す
60
60
 
61
61
  共有オブジェクトをカタログより先に消すのは、途中で止まった取り下げをやり直せるようにするためです。カタログが残っていれば、どのファイルがそのデータを持つかが分かります。
@@ -108,7 +108,7 @@ fdl tag <remote> <name> --delete
108
108
  fdl withdraw <remote> <version>... [--yes]
109
109
  ```
110
110
 
111
- Removes versions that hold data that must not be published, such as personal data published by mistake (see [Withdrawal](/spec/data-files#withdrawal)). First commit a version without the data, so that no branch points at an affected version. The command then deletes the tags that point at the versions, the data files their catalogs name that no kept version reads, and their catalogs, and puts `expired.json` with `"reason": "withdrawn"`. `version.json` stays.
111
+ Removes versions that hold data that must not be published, such as personal data published by mistake (see [Withdrawal](/spec/data-files#withdrawal)). First commit a version without the data, so that no branch points at an affected version. The command then deletes the tags that point at the versions, the data files their catalogs name that no kept version reads, and their catalogs, and puts `expired.json` with `"reason": "withdrawn"`. `version.json` stays. When one of the versions is already past retention, its catalog is gone, so every data file no live version reads is deleted at once, without waiting for the grace period.
112
112
 
113
113
  Assets are not deleted yet: delete those that hold the data from `assets/` by hand.
114
114
 
@@ -55,7 +55,7 @@ Withdrawal removes data that must not be published, such as personal data publis
55
55
 
56
56
  1. Make a new version without the data and move every branch that holds an affected version to it. This is a forward move: the new version's parent is the current one
57
57
  2. Delete the tags that point at affected versions
58
- 3. Delete the shared objects that hold the data, at once, such as a data file with personal data or a screenshot showing it: those an affected version reaches and no version kept reaches. Every version that reaches such an object is an affected version
58
+ 3. Delete the shared objects that hold the data, at once, such as a data file with personal data or a screenshot showing it: those an affected version reaches and no version kept reaches. Every version that reaches such an object is an affected version. An affected version already past retention has no catalog to say which objects it reached, so when there is one, every shared object no live version reaches is deleted at once too
59
59
  4. For every affected version, put `expired.json` with `"reason": "withdrawn"` and delete its catalogs
60
60
 
61
61
  The shared objects go before the catalogs, so a withdrawal that stops half way can be run again: the catalogs still say which files hold the data.
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "fdl-cli"
3
- version = "0.20.0"
3
+ version = "0.20.1"
4
4
  description = "Versions of a DuckLake catalog on object storage, kept the way Git keeps commits"
5
5
  readme = "README.md"
6
6
  license = "MIT"
@@ -295,7 +295,9 @@ def withdraw(
295
295
 
296
296
  Move every branch to a version without the data first. This deletes the
297
297
  tags that point at the versions, the data files no kept version reads,
298
- and the versions' catalogs, at once. Their version.json stays.
298
+ and the versions' catalogs, at once. Their version.json stays. A version
299
+ already past retention has no catalog, so then every data file no live
300
+ version reads is deleted.
299
301
  """
300
302
  store = open_store(remote)
301
303
  try:
@@ -655,6 +655,7 @@ def plan_withdrawal(store: Store, version_ids: Iterable[str]) -> Withdrawal:
655
655
 
656
656
  candidates: set[str] = set()
657
657
  kept: set[str] = set()
658
+ catalogless = False
658
659
  with tempfile.TemporaryDirectory(prefix="fdl-") as tmp:
659
660
  for version_id in _version_ids(store):
660
661
  if version_id in affected:
@@ -664,6 +665,8 @@ def plan_withdrawal(store: Store, version_ids: Iterable[str]) -> Withdrawal:
664
665
  # in it may hold the data too.
665
666
  candidates.update(catalog.data_files(local))
666
667
  local.unlink()
668
+ else:
669
+ catalogless = True
667
670
  continue
668
671
  if is_expired(store, version_id):
669
672
  continue
@@ -677,6 +680,11 @@ def plan_withdrawal(store: Store, version_ids: Iterable[str]) -> Withdrawal:
677
680
  if local is not None:
678
681
  kept.update(_read_files(local, version))
679
682
  local.unlink()
683
+ if catalogless:
684
+ # A version already past retention no longer says which files it
685
+ # named. Those no live version reads are waiting for the grace period
686
+ # anyway, so all of them go now.
687
+ candidates.update(key[len(config.data_dir) :] for key in store.list(config.data_dir))
680
688
  return Withdrawal(
681
689
  versions=affected,
682
690
  tags=tuple(tags),
@@ -156,7 +156,8 @@ def test_a_file_the_host_will_not_sign_goes_through_the_host(tmp_path: Path) ->
156
156
 
157
157
  assert local.get("data/small.parquet") == b"ab" and local.get("data/large.parquet") == b"abcdef"
158
158
  puts = [key for method, key in server.requests if method == "PUT"]
159
- assert puts == ["storage/data/small.parquet", "data/large.parquet"]
159
+ # Uploads run at once, so they may arrive in any order.
160
+ assert sorted(puts) == ["data/large.parquet", "storage/data/small.parquet"]
160
161
 
161
162
 
162
163
  def test_a_rebuild_of_the_same_rows_over_http_is_nothing_to_commit(served, tmp_path: Path) -> None:
@@ -387,6 +387,23 @@ def test_withdraw_removes_the_data_from_live_versions(tmp_path: Path, remote) ->
387
387
  assert repo.head(remote) == clean
388
388
 
389
389
 
390
+ def test_withdrawing_a_version_past_retention_deletes_unread_files_at_once(tmp_path: Path, remote) -> None:
391
+ first = pushed(tmp_path, remote)
392
+ leaked = set(remote.list("data/"))
393
+ build(tmp_path / "work.sqlite", remote, "DROP TABLE lake.main.towns; CREATE TABLE lake.main.kept AS SELECT 1 AS id")
394
+ second = commit(tmp_path, remote, parent=first)
395
+ repo.push(remote, tmp_path / "local" / second.id, second)
396
+ kept = set(remote.list("data/")) - leaked
397
+ repo.expire(remote, first.id)
398
+ assert leaked <= set(remote.list("data/")) # waiting for the grace period
399
+
400
+ plan = repo.withdraw(remote, [first.id])
401
+ assert set(plan.deleted) == leaked
402
+ assert set(remote.list("data/")) == kept
403
+ assert json.loads(remote.get(layout.version_expired(first.id)))["reason"] == "withdrawn"
404
+ assert repo.head(remote) == second
405
+
406
+
390
407
  def test_a_file_a_kept_version_still_reads_is_not_deleted(tmp_path: Path, remote) -> None:
391
408
  first = pushed(tmp_path, remote)
392
409
  second = pushed(tmp_path, remote, parent=first, sql="INSERT INTO lake.main.towns VALUES (2)")
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes