fdl-cli 0.17.0__tar.gz → 0.18.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/CHANGELOG.md +33 -0
  2. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/PKG-INFO +1 -1
  3. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/reference/cli.md +20 -0
  4. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/reference/python.md +29 -0
  5. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/spec/data-files.md +4 -2
  6. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/spec/http.md +38 -0
  7. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/reference/cli.md +20 -0
  8. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/reference/python.md +29 -0
  9. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/spec/data-files.md +4 -2
  10. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/spec/http.md +38 -0
  11. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/pyproject.toml +1 -1
  12. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/src/fdl/cli.py +60 -1
  13. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/src/fdl/layout.py +4 -0
  14. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/src/fdl/refs.py +37 -0
  15. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/src/fdl/repo.py +192 -7
  16. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/src/fdl/stores.py +100 -0
  17. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/tests/http_repo.py +29 -1
  18. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/tests/test_cli.py +33 -0
  19. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/tests/test_http.py +34 -0
  20. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/tests/test_repo.py +100 -0
  21. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/uv.lock +1 -1
  22. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/.github/workflows/docs.yml +0 -0
  23. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/.github/workflows/publish.yml +0 -0
  24. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/.github/workflows/release.yml +0 -0
  25. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/.gitignore +0 -0
  26. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/.node-version +0 -0
  27. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/.python-version +0 -0
  28. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/CLAUDE.md +0 -0
  29. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/CONTRIBUTING.md +0 -0
  30. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/LICENSE +0 -0
  31. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/README.md +0 -0
  32. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/blume.config.ts +0 -0
  33. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/concepts/meta.ts +0 -0
  34. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/concepts/repositories.md +0 -0
  35. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/concepts/versions-and-refs.md +0 -0
  36. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/concepts/working-catalogs.md +0 -0
  37. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/getting-started.md +0 -0
  38. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/guides/building-a-host.md +0 -0
  39. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/guides/meta.ts +0 -0
  40. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/guides/mirrors.md +0 -0
  41. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/guides/public-and-private.md +0 -0
  42. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/guides/reading-without-duckdb.md +0 -0
  43. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/guides/serving-over-http.md +0 -0
  44. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/guides/stores.md +0 -0
  45. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/index.md +0 -0
  46. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/internals/ducklake-formats.md +0 -0
  47. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/internals/inside-a-repository.md +0 -0
  48. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/internals/meta.ts +0 -0
  49. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/concepts/meta.ts +0 -0
  50. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/concepts/repositories.md +0 -0
  51. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/concepts/versions-and-refs.md +0 -0
  52. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/concepts/working-catalogs.md +0 -0
  53. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/getting-started.md +0 -0
  54. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/guides/building-a-host.md +0 -0
  55. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/guides/meta.ts +0 -0
  56. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/guides/mirrors.md +0 -0
  57. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/guides/public-and-private.md +0 -0
  58. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/guides/reading-without-duckdb.md +0 -0
  59. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/guides/serving-over-http.md +0 -0
  60. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/guides/stores.md +0 -0
  61. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/index.md +0 -0
  62. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/internals/ducklake-formats.md +0 -0
  63. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/internals/inside-a-repository.md +0 -0
  64. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/internals/meta.ts +0 -0
  65. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/reference/meta.ts +0 -0
  66. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/spec/index.md +0 -0
  67. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/spec/layout.md +0 -0
  68. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/spec/meta.ts +0 -0
  69. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/spec/reading.md +0 -0
  70. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/spec/refs.md +0 -0
  71. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/spec/versions.md +0 -0
  72. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/ja/spec/writing.md +0 -0
  73. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/reference/meta.ts +0 -0
  74. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/spec/index.md +0 -0
  75. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/spec/layout.md +0 -0
  76. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/spec/meta.ts +0 -0
  77. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/spec/reading.md +0 -0
  78. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/spec/refs.md +0 -0
  79. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/spec/versions.md +0 -0
  80. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/docs/spec/writing.md +0 -0
  81. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/package-lock.json +0 -0
  82. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/package.json +0 -0
  83. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/public/CNAME +0 -0
  84. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/src/fdl/__init__.py +0 -0
  85. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/src/fdl/catalog.py +0 -0
  86. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/src/fdl/dataset.py +0 -0
  87. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/src/fdl/py.typed +0 -0
  88. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/src/fdl/store.py +0 -0
  89. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/src/fdl/version.py +0 -0
  90. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/tests/test_dataset.py +0 -0
  91. {fdl_cli-0.17.0 → fdl_cli-0.18.0}/tests/test_stores.py +0 -0
@@ -1,6 +1,39 @@
1
1
  # CHANGELOG
2
2
 
3
3
 
4
+ ## v0.18.0 (2026-11-03)
5
+
6
+ ### Bug Fixes
7
+
8
+ - Send the file's Content-Type to signed URLs, ask again for expiring ones, plan withdrawals from
9
+ files.json
10
+ ([`be180a3`](https://github.com/queria-io/fdl/commit/be180a3e23364cd28ec975bac7bcc59c2ae7b6bd))
11
+
12
+ ### Features
13
+
14
+ - Fdl withdraw
15
+ ([`2f3faf3`](https://github.com/queria-io/fdl/commit/2f3faf3cf21587f38e6322604cb0c7c83640d564))
16
+
17
+ Withdraw versions that hold data that must not be published: delete the tags that point at them, the
18
+ data files no kept version reads, then their catalogs, and mark them withdrawn. Without --yes it
19
+ only prints the plan. The spec now deletes data files before catalogs, so a stopped run can be run
20
+ again.
21
+
22
+ - Refs.json and fdl tag
23
+ ([`200a613`](https://github.com/queria-io/fdl/commit/200a6137e4451af6d5789d9a741fb54863dc7486))
24
+
25
+ Rewrite refs.json after a push or a tag moves a ref, as the spec asks. A host that keeps refs.json
26
+ itself may refuse the write with 403. Add fdl tag to list, create and delete tags, which never
27
+ move.
28
+
29
+ - Upload straight to storage through a batch of signed URLs
30
+ ([`a2b59cd`](https://github.com/queria-io/fdl/commit/a2b59cdfc2d1dd5c1f52dc013b4d8553624aeadd))
31
+
32
+ Add POST ?batch to the HTTP protocol, after Git LFS's batch API: a host hands out upload URLs for
33
+ files written once, and the writer sends them straight to storage. A host without batches, or a
34
+ file it will not sign for (413), goes through the host as before.
35
+
36
+
4
37
  ## v0.17.0 (2026-11-03)
5
38
 
6
39
  ### Bug Fixes
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: fdl-cli
3
- Version: 0.17.0
3
+ Version: 0.18.0
4
4
  Summary: Versions of a DuckLake catalog on object storage, kept the way Git keeps commits
5
5
  Project-URL: Homepage, https://fdl.flo8s.com/
6
6
  Project-URL: Documentation, https://fdl.flo8s.com/
@@ -77,6 +77,26 @@ fdl url <remote> [--ref HEAD]
77
77
 
78
78
  ref が指すバージョンのカタログの URL を表示します。読み取り専用で `ATTACH` するのに使います。
79
79
 
80
+ ## fdl tag
81
+
82
+ ```bash
83
+ fdl tag <remote> # タグを並べる
84
+ fdl tag <remote> <name> [<version>] # バージョンにタグを付ける(既定は HEAD が指すバージョン)
85
+ fdl tag <remote> <name> --delete
86
+ ```
87
+
88
+ `<version>` にはバージョン ID か、`refs/heads/main` のような ref を渡します。タグは一度作ると動かしません。別のバージョンを指す同じ名前のタグがあるときは断られます。保持期間を過ぎたバージョンにはタグを付けられません。タグを作るか消すと `refs.json` を書き直します。`fdl commit` もブランチを動かしたあとに書き直します。
89
+
90
+ ## fdl withdraw
91
+
92
+ ```bash
93
+ fdl withdraw <remote> <version>... [--yes]
94
+ ```
95
+
96
+ 間違って公開した個人データなど、公開してはいけないデータを持つバージョンを取り下げます([取り下げ](/ja/spec/data-files#取り下げ))。先にそのデータを除いたバージョンをコミットし、該当するバージョンを指すブランチが無い状態にします。そのうえでこのコマンドが、該当するバージョンを指すタグ、そのカタログが参照し残すバージョンが読まないデータファイル、カタログを消し、`"reason": "withdrawn"` の `expired.json` を置きます。`version.json` は残します。
97
+
98
+ `--yes` を付けないと、消すものを表示するだけです。残すバージョンが読んでいるファイルは残すものとして表示します。それがそのデータを持つなら、そのバージョンも取り下げます。ブランチかリクエストが該当するバージョンを指しているあいだは断られます。
99
+
80
100
  ## fdl expire
81
101
 
82
102
  ```bash
@@ -33,6 +33,7 @@ for version in repo.log(store):
33
33
  | `head(key)` | オブジェクトのヘッダー(`ETag` を含む)か `None` |
34
34
  | `list(prefix)` | プレフィックスの下のキー |
35
35
  | `delete(keys)` | 消す。既に無いキーはエラーにしない |
36
+ | `put_files(files, *, content_type)` | 任意。一度だけ書く `(key, path)` の組を書く。HTTP の置き場はホストにアップロード用の URL を求め([batch](/ja/spec/http#ストレージへ直接アップロードする))、ストレージへ直接送る |
36
37
 
37
38
  ## fdl.repo
38
39
 
@@ -81,6 +82,27 @@ repo.prune(store, newest, *, keep=1, since=None, reason="retention") -> list[str
81
82
 
82
83
  バージョン 1 つの保持期間を終わらせるか、`newest` より古く保持期間から外れるバージョンをまとめて終わらせます。ref が指すバージョンは残します。
83
84
 
85
+ ```python
86
+ repo.tag(store, name, version_id) -> None
87
+ repo.delete_tag(store, name) -> None
88
+ repo.tags(store) -> dict[str, str]
89
+ ```
90
+
91
+ バージョンに `refs/tags/<name>` を作る、消す、タグと指すバージョンを並べる、の 3 つです。タグは動かしません。同じ名前のタグが別のバージョンにあれば、`tag` は `RepoError` を出します。
92
+
93
+ ```python
94
+ repo.update_refs_index(store) -> None
95
+ ```
96
+
97
+ ref を読み直して `refs.json` を書き直します。`push`・`tag`・`delete_tag` は ref を動かしたあとにこれを呼びます。`refs.json` を自分で書くホストが 403 で断ったときは、済んだものとして扱います。
98
+
99
+ ```python
100
+ repo.plan_withdrawal(store, version_ids) -> Withdrawal
101
+ repo.withdraw(store, version_ids, *, plan=None) -> Withdrawal
102
+ ```
103
+
104
+ 公開してはいけないデータを持つバージョンを取り下げます。`plan_withdrawal` は何も変えずに、消すタグ、消すデータファイル、残すバージョンが読むので残すファイルを返します。`withdraw` はタグ、データファイル、カタログの順に消します。ブランチかリクエストが該当するバージョンを指しているあいだは、どちらも `RepoError` を出します。
105
+
84
106
  ## fdl.refs
85
107
 
86
108
  ```python
@@ -89,3 +111,10 @@ refs.write(store, name, version_id, *, expected) -> str
89
111
  ```
90
112
 
91
113
  `read` は、動かすときに渡す ETag を付けて ref を返します。`write` は、`expected` が `None` なら ref を作り、そうでなければ動かします。条件が合わなければ `RefConflict` を出します。
114
+
115
+ ```python
116
+ refs.listed(store) -> dict[str, str]
117
+ refs.write_index(store) -> dict
118
+ ```
119
+
120
+ `listed` はブランチとタグをすべて読み直します。`write_index` はそれと `HEAD` から `refs.json` を書きます。
@@ -41,8 +41,10 @@ DuckLake 自身の片付け(`ducklake_cleanup_old_files`、`ducklake_expire_sn
41
41
 
42
42
  1. そのデータを除いた新しいバージョンを作り、該当するバージョンを指しているブランチをすべてそこへ動かす。新しいバージョンの親はいまのバージョンなので、前へ動かすことになる
43
43
  2. 該当するバージョンを指すタグを消す
44
- 3. 該当するバージョンそれぞれに `"reason": "withdrawn"` の `expired.json` を置き、カタログを消す
45
- 4. そのデータを持つデータファイルを、すぐに消す
44
+ 3. そのデータを持つデータファイルを、すぐに消す。該当するバージョンのカタログが参照し、残すバージョンがどれもその時点のスナップショットで読んでいないファイルが対象
45
+ 4. 該当するバージョンそれぞれに `"reason": "withdrawn"` の `expired.json` を置き、カタログを消す
46
+
47
+ データファイルをカタログより先に消すのは、途中で止まった取り下げをやり直せるようにするためです。カタログが残っていれば、どのファイルがそのデータを持つかが分かります。
46
48
 
47
49
  ```json expired.json
48
50
  { "expired_at": "2026-12-01T00:00:00Z", "reason": "withdrawn" }
@@ -12,6 +12,7 @@ description: 書き手が置き場の鍵を持たずに push できるよう、
12
12
  | `GET`、`HEAD` `<root>/<key>` | オブジェクトを読む | `ETag` 付きの 200、または 404 |
13
13
  | `PUT` `<root>/<key>` | `If-None-Match: *` か `If-Match: <ETag>` を付けてオブジェクトを書く | 新しい `ETag` 付きの 200 か 201。条件が合わなければ 412 |
14
14
  | `GET` `<root>/?list&prefix=<p>[&after=<key>]` | プレフィックスの下のキーの一覧 | `{"keys": [...], "next": <key か null>}`。`next` がある間は `after` を付けて続きを求める |
15
+ | `POST` `<root>/?batch` | アップロード用の URL を求める。任意([下](#ストレージへ直接アップロードする)を参照) | `{"objects": [...]}` |
15
16
  | `POST` `<root>/<key>?uploads` | 分割アップロードを始める。条件は `PUT` と同じ | `{"upload": "<id>"}` |
16
17
  | `PUT` `<root>/<key>?upload=<id>&part=<n>` | `n` 番目(1 から)の部分を送る | `{"etag": "<部分のタグ>"}` |
17
18
  | `POST` `<root>/<key>?upload=<id>` | `{"parts": [{"part": n, "etag": "…"}]}` を順に並べて終える | `PUT` と同じ |
@@ -19,9 +20,46 @@ description: 書き手が置き場の鍵を持たずに push できるよう、
19
20
  - 書き込みには `Authorization: Bearer <token>` を付ける。使えるトークンが無ければ 401、ここに書けないトークンなら 403
20
21
  - 最後以外の部分は同じ大きさにする。1 回のリクエストで送れる大きさ(既定で 64 MB)を超えるファイルは分割して送る
21
22
  - どのキーを受け付けるかはホストが決める。`HEAD`、`fdl.json`、配置の外のものへの書き込みは断ってよく(ただし HTTP の `HEAD` リクエストには答える。書き手が ref の ETag を読むのに使う)、`versions/` とデータファイルは `If-None-Match: *` 付きのときだけ受け付けてよい
23
+ - 確かめたあとに自分で ref を動かすホストは、`refs.json` も自分で書き直す。書き手からの `refs.json` の書き込みは 403 で断ってよく、書き手はそれを済んだものとして扱う
22
24
  - ホストが push を確かめるのは ref を動かすとき。バージョンがあること、fast-forward であること、カタログが `version.json` と合うこと、参照するデータファイルがすべてリポジトリにあることを確かめる。断るときは 409 で、理由を本文に書く
23
25
  - `DELETE` は無い。保持期間と削除はホストの仕事
24
26
 
27
+ ## ストレージへ直接アップロードする
28
+
29
+ ホストはアップロード用の URL を渡し、ファイルをホストを通さずストレージへ直接送らせてよい。Git LFS の batch API と同じ形にする。この経路を使うのは、一度だけ書くファイル(データファイルとバージョンのファイル)だけ。ref は常にホストを通して書く。
30
+
31
+ 書き手は、アップロードするファイルを `POST <root>/?batch` で送る。
32
+
33
+ ```json
34
+ { "operation": "upload", "objects": [{ "key": "data/main/towns/a.parquet", "size": 1048576 }] }
35
+ ```
36
+
37
+ 答えにはオブジェクトごとに 1 つずつ項目が入る。順番は問わない。
38
+
39
+ ```json
40
+ {
41
+ "objects": [
42
+ {
43
+ "key": "data/main/towns/a.parquet",
44
+ "size": 1048576,
45
+ "actions": {
46
+ "upload": { "href": "https://storage.example.org/…", "header": { "If-None-Match": "*" }, "expires_in": 3600 }
47
+ }
48
+ },
49
+ { "key": "data/main/towns/b.parquet", "size": 2048 },
50
+ { "key": "data/main/towns/c.parquet", "size": 9, "error": { "code": 413, "message": "Too large for one request" } }
51
+ ]
52
+ }
53
+ ```
54
+
55
+ - `actions.upload`: 書き手はファイルを本文にして `PUT <href>` を送り、ファイルの `Content-Type` と `header` のヘッダーを付ける。リポジトリの `Authorization` は付けない。そこで 412 が返れば、オブジェクトは既にある。`expires_in` を過ぎた URL を使う前に、書き手はそのオブジェクトをもう一度尋ねる
56
+ - `actions` も `error` も無い: オブジェクトは既にある
57
+ - コード 413 の `error`: ホストはこのファイルに署名しない。書き手はホストを通して(`PUT` か分割で)アップロードする
58
+ - それ以外の `error`: アップロードは断られた。コードと理由が入る
59
+ - 書き手が一度に尋ねるのは 100 個まで
60
+ - batch を持たないホストは 404・405・501 を返し、書き手はホストを通してアップロードする
61
+ - アップロードしただけではバージョンにならない。ホストが push を確かめるのは、これまでどおり ref を動かすとき
62
+
25
63
  ## 書き方
26
64
 
27
65
  DuckDB は HTTP の URL にデータファイルを書けないので、こうしたリポジトリの作業カタログは 2 つの場所を持ちます。
@@ -77,6 +77,26 @@ fdl url <remote> [--ref HEAD]
77
77
 
78
78
  Prints the URL of the catalog of the version the ref holds, to `ATTACH` read-only.
79
79
 
80
+ ## fdl tag
81
+
82
+ ```bash
83
+ fdl tag <remote> # list the tags
84
+ fdl tag <remote> <name> [<version>] # tag a version (default: the one HEAD holds)
85
+ fdl tag <remote> <name> --delete
86
+ ```
87
+
88
+ `<version>` is a version ID or a ref such as `refs/heads/main`. A tag is created once and never moved: tagging a name that is already there at another version is refused. A version past retention cannot be tagged. Creating or deleting a tag rewrites `refs.json`, as `fdl commit` does after moving a branch.
89
+
90
+ ## fdl withdraw
91
+
92
+ ```bash
93
+ fdl withdraw <remote> <version>... [--yes]
94
+ ```
95
+
96
+ Removes versions that hold data that must not be published, such as personal data published by mistake (see [Withdrawal](/spec/data-files#withdrawal)). First commit a version without the data, so that no branch points at an affected version. The command then deletes the tags that point at the versions, the data files their catalogs name that no kept version reads, and their catalogs, and puts `expired.json` with `"reason": "withdrawn"`. `version.json` stays.
97
+
98
+ Without `--yes` it only prints what it would delete. Files a kept version still reads are listed as kept; if they hold the data, withdraw that version too. Refused while a branch or a request points at one of the versions.
99
+
80
100
  ## fdl expire
81
101
 
82
102
  ```bash
@@ -33,6 +33,7 @@ for version in repo.log(store):
33
33
  | `head(key)` | The object's headers (with `ETag`), or `None` |
34
34
  | `list(prefix)` | Keys under a prefix |
35
35
  | `delete(keys)` | Remove. A key already gone is not an error |
36
+ | `put_files(files, *, content_type)` | Optional. Write `(key, path)` pairs that are written once. The HTTP store asks the host for upload URLs ([batch](/spec/http#uploading-straight-to-storage)) and sends the files straight to storage |
36
37
 
37
38
  ## fdl.repo
38
39
 
@@ -81,6 +82,27 @@ repo.prune(store, newest, *, keep=1, since=None, reason="retention") -> list[str
81
82
 
82
83
  Take one version out of retention, or every version behind `newest` that falls out of it. Versions a ref points at are kept.
83
84
 
85
+ ```python
86
+ repo.tag(store, name, version_id) -> None
87
+ repo.delete_tag(store, name) -> None
88
+ repo.tags(store) -> dict[str, str]
89
+ ```
90
+
91
+ Create `refs/tags/<name>` at a version, delete it, or list the tags with the versions they hold. A tag is never moved: `tag` raises `RepoError` when the name is already there at another version.
92
+
93
+ ```python
94
+ repo.update_refs_index(store) -> None
95
+ ```
96
+
97
+ Rewrite `refs.json` from the refs. `push`, `tag` and `delete_tag` call it after moving a ref. A 403 from a host that keeps `refs.json` itself is taken as done.
98
+
99
+ ```python
100
+ repo.plan_withdrawal(store, version_ids) -> Withdrawal
101
+ repo.withdraw(store, version_ids, *, plan=None) -> Withdrawal
102
+ ```
103
+
104
+ Withdraw versions that hold data that must not be published. `plan_withdrawal` changes nothing and returns the tags, the data files to delete, and the files kept because a kept version reads them. `withdraw` deletes the tags, the data files and the catalogs, in that order. Both raise `RepoError` while a branch or a request points at one of the versions.
105
+
84
106
  ## fdl.refs
85
107
 
86
108
  ```python
@@ -89,3 +111,10 @@ refs.write(store, name, version_id, *, expected) -> str
89
111
  ```
90
112
 
91
113
  `read` returns the ref with the ETag a move has to name. `write` creates the ref when `expected` is `None` and moves it otherwise, and raises `RefConflict` when the condition fails.
114
+
115
+ ```python
116
+ refs.listed(store) -> dict[str, str]
117
+ refs.write_index(store) -> dict
118
+ ```
119
+
120
+ `listed` reads every branch and tag afresh. `write_index` writes `refs.json` from them and `HEAD`.
@@ -41,8 +41,10 @@ Withdrawal removes data that must not be published, such as personal data publis
41
41
 
42
42
  1. Make a new version without the data and move every branch that holds an affected version to it. This is a forward move: the new version's parent is the current one
43
43
  2. Delete the tags that point at affected versions
44
- 3. For every affected version, put `expired.json` with `"reason": "withdrawn"` and delete its catalogs
45
- 4. Delete the data files that hold the data, at once
44
+ 3. Delete the data files that hold the data, at once: those the affected versions' catalogs name and no version kept reads at its snapshot
45
+ 4. For every affected version, put `expired.json` with `"reason": "withdrawn"` and delete its catalogs
46
+
47
+ The data files go before the catalogs, so a withdrawal that stops half way can be run again: the catalogs still say which files hold the data.
46
48
 
47
49
  ```json expired.json
48
50
  { "expired_at": "2026-12-01T00:00:00Z", "reason": "withdrawn" }
@@ -12,6 +12,7 @@ A host can take pushes over HTTP, so a writer needs no keys to its storage. The
12
12
  | `GET`, `HEAD` `<root>/<key>` | Read an object | 200 with an `ETag`, or 404 |
13
13
  | `PUT` `<root>/<key>` | Write an object, with `If-None-Match: *` or `If-Match: <ETag>` | 200 or 201 with the new `ETag`; 412 when the condition fails |
14
14
  | `GET` `<root>/?list&prefix=<p>[&after=<key>]` | List keys under a prefix | `{"keys": [...], "next": <key or null>}`; ask again with `after` while `next` is set |
15
+ | `POST` `<root>/?batch` | Ask for upload URLs, optionally (see [below](#uploading-straight-to-storage)) | `{"objects": [...]}` |
15
16
  | `POST` `<root>/<key>?uploads` | Start an upload in parts, with the same conditions as `PUT` | `{"upload": "<id>"}` |
16
17
  | `PUT` `<root>/<key>?upload=<id>&part=<n>` | Send part `n` (from 1) | `{"etag": "<part tag>"}` |
17
18
  | `POST` `<root>/<key>?upload=<id>` | Finish it with `{"parts": [{"part": n, "etag": "…"}]}` in order | As `PUT` |
@@ -19,9 +20,46 @@ A host can take pushes over HTTP, so a writer needs no keys to its storage. The
19
20
  - Writes carry `Authorization: Bearer <token>`. 401 without a usable token, 403 for one that may not write here
20
21
  - Parts other than the last are the same size. A writer sends a file in parts when it is larger than one request may be (64 MB by default)
21
22
  - A host decides which keys it takes. It may refuse writes to `HEAD`, `fdl.json` and anything outside the layout (but must answer `HEAD` requests, which a writer needs for a ref's ETag), and take `versions/` and data files only with `If-None-Match: *`
23
+ - A host that moves refs itself after its checks keeps `refs.json` too, and may refuse a writer's write to it with 403. A writer treats that as done
22
24
  - Moving a ref is where a host checks a push: the version is there, it is a fast-forward, the catalog matches `version.json`, and every data file it names is in the repository. A refusal is 409 with the reason in the body
23
25
  - There is no `DELETE`. Retention and deletion are the host's
24
26
 
27
+ ## Uploading straight to storage
28
+
29
+ A host may hand out upload URLs so that files go straight to its storage rather than through it, as Git LFS's batch API does. Only files that are written once (data files and the files of a version) go this way. Refs are always written through the host.
30
+
31
+ `POST <root>/?batch` with the files a writer is about to upload:
32
+
33
+ ```json
34
+ { "operation": "upload", "objects": [{ "key": "data/main/towns/a.parquet", "size": 1048576 }] }
35
+ ```
36
+
37
+ The answer has one entry per object, in any order:
38
+
39
+ ```json
40
+ {
41
+ "objects": [
42
+ {
43
+ "key": "data/main/towns/a.parquet",
44
+ "size": 1048576,
45
+ "actions": {
46
+ "upload": { "href": "https://storage.example.org/…", "header": { "If-None-Match": "*" }, "expires_in": 3600 }
47
+ }
48
+ },
49
+ { "key": "data/main/towns/b.parquet", "size": 2048 },
50
+ { "key": "data/main/towns/c.parquet", "size": 9, "error": { "code": 413, "message": "Too large for one request" } }
51
+ ]
52
+ }
53
+ ```
54
+
55
+ - `actions.upload`: the writer sends `PUT <href>` with the file as the body, its `Content-Type` and the headers in `header`, and nothing from the repository's own `Authorization`. 412 there means the object is already there. A writer that is about to use a URL past `expires_in` asks for that object again
56
+ - No `actions` and no `error`: the object is already there
57
+ - `error` with code 413: the host will not sign this one. The writer uploads it through the host (`PUT`, or in parts)
58
+ - Any other `error`: the upload is refused, with the code and the reason
59
+ - A writer asks about at most 100 objects at a time
60
+ - A host without batches answers 404, 405 or 501, and writers upload through the host
61
+ - Uploading does not make a version. Moving a ref is still where the host checks the push
62
+
25
63
  ## Writing to it
26
64
 
27
65
  DuckDB cannot write data files to an HTTP URL, so a working catalog of such a repository keeps two places:
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "fdl-cli"
3
- version = "0.17.0"
3
+ version = "0.18.0"
4
4
  description = "Versions of a DuckLake catalog on object storage, kept the way Git keeps commits"
5
5
  readme = "README.md"
6
6
  license = "MIT"
@@ -17,7 +17,7 @@ import typer
17
17
 
18
18
  from . import catalog, dataset, layout, refs, repo
19
19
  from .stores import StoreError, open_store
20
- from .version import VersionError
20
+ from .version import VersionError, is_id
21
21
 
22
22
  app = typer.Typer(no_args_is_help=True, pretty_exceptions_short=True)
23
23
 
@@ -175,6 +175,33 @@ def url(remote: str = Remote, ref: str = typer.Option(layout.HEAD)) -> None:
175
175
  typer.echo(store.url.rstrip("/") + "/" + layout.version_catalog(version.id))
176
176
 
177
177
 
178
+ @app.command()
179
+ def tag(
180
+ remote: str = Remote,
181
+ name: str | None = typer.Argument(None, help="The tag, without refs/tags/. Omit to list the tags"),
182
+ target: str = typer.Argument(layout.HEAD, help="A version ID, or the ref whose version to tag"),
183
+ delete: bool = typer.Option(False, "-d", "--delete", help="Delete the tag"),
184
+ ) -> None:
185
+ """List tags, tag a version, or delete a tag. Tags never move."""
186
+ store = open_store(remote)
187
+ try:
188
+ if name is None:
189
+ for tag_name, version_id in repo.tags(store).items():
190
+ typer.echo(f"{tag_name} {version_id}")
191
+ return
192
+ if delete:
193
+ repo.delete_tag(store, name)
194
+ typer.echo(f"Deleted refs/tags/{name}")
195
+ return
196
+ version_id = target if is_id(target) else refs.resolve(store, target)[1]
197
+ if version_id is None:
198
+ raise repo.RepoError(f"{target} holds no version yet")
199
+ repo.tag(store, name, version_id)
200
+ except ERRORS as exc:
201
+ raise _fail(exc) from exc
202
+ typer.echo(f"refs/tags/{name} -> {version_id}")
203
+
204
+
178
205
  @app.command()
179
206
  def expire(
180
207
  remote: str = Remote,
@@ -195,5 +222,37 @@ def expire(
195
222
  typer.echo(f"Expired {len(expired)} version(s).")
196
223
 
197
224
 
225
+ @app.command()
226
+ def withdraw(
227
+ remote: str = Remote,
228
+ versions: list[str] = typer.Argument(..., help="The versions that hold the data"),
229
+ yes: bool = typer.Option(False, "--yes", help="Do it. Without this, only say what would be done"),
230
+ ) -> None:
231
+ """Remove versions that hold data that must not be published.
232
+
233
+ Move every branch to a version without the data first. This deletes the
234
+ tags that point at the versions, the data files no kept version reads,
235
+ and the versions' catalogs, at once. Their version.json stays.
236
+ """
237
+ store = open_store(remote)
238
+ try:
239
+ plan = repo.plan_withdrawal(store, versions)
240
+ for name in plan.tags:
241
+ typer.echo(f"delete tag {name}")
242
+ for key in plan.deleted:
243
+ typer.echo(f"delete file {key}")
244
+ for version_id in plan.versions:
245
+ typer.echo(f"withdraw {version_id}")
246
+ for key in plan.still_read:
247
+ typer.echo(f"keep file {key} (a kept version reads it)")
248
+ if not yes:
249
+ typer.echo("Nothing was changed. Run again with --yes to do this.")
250
+ return
251
+ repo.withdraw(store, versions, plan=plan)
252
+ except ERRORS as exc:
253
+ raise _fail(exc) from exc
254
+ typer.echo(f"Withdrew {len(plan.versions)} version(s) and deleted {len(plan.deleted)} file(s).")
255
+
256
+
198
257
  def main() -> None:
199
258
  app()
@@ -13,6 +13,8 @@ FDL_VERSION = "0.1"
13
13
 
14
14
  CONFIG = "fdl.json"
15
15
  HEAD = "HEAD"
16
+ #: The list of refs, for readers that cannot list the store.
17
+ REFS_INDEX = "refs.json"
16
18
  DEFAULT_BRANCH = "refs/heads/main"
17
19
  DEFAULT_DATA_DIR = "data/"
18
20
 
@@ -31,6 +33,8 @@ SYMBOLIC = "ref: "
31
33
 
32
34
  _REF_PART = re.compile(r"[a-z0-9][a-z0-9._-]*")
33
35
  _REF_KINDS = ("heads", "tags", "requests")
36
+ #: The kinds `refs.json` lists. Requests are the host's own.
37
+ LISTED_REFS = ("refs/heads/", "refs/tags/")
34
38
  _REF_MAX = 128
35
39
 
36
40
 
@@ -9,6 +9,7 @@ other without a server in between.
9
9
 
10
10
  from __future__ import annotations
11
11
 
12
+ import json
12
13
  from dataclasses import dataclass
13
14
 
14
15
  from . import layout
@@ -113,3 +114,39 @@ def write_head(store: Store, branch: str = layout.DEFAULT_BRANCH) -> None:
113
114
  except Exception as exc: # noqa: BLE001 - stores raise their own types
114
115
  if not is_conflict(exc):
115
116
  raise
117
+
118
+
119
+ def delete(store: Store, name: str) -> None:
120
+ """Remove a ref. Stores have no conditional delete, so this is for tags,
121
+ which never move."""
122
+ store.delete([layout.check_ref(name)])
123
+
124
+
125
+ def listed(store: Store) -> dict[str, str]:
126
+ """Every branch and tag with the version it holds, read afresh."""
127
+ found: dict[str, str] = {}
128
+ for name in sorted(n for prefix in layout.LISTED_REFS for n in store.list(prefix)):
129
+ try:
130
+ ref = read(store, name)
131
+ except (RefError, layout.LayoutError):
132
+ continue
133
+ if ref is not None:
134
+ found[name] = ref.target
135
+ return found
136
+
137
+
138
+ def write_index(store: Store) -> dict:
139
+ """Rewrite `refs.json` from a fresh read of HEAD and every ref.
140
+
141
+ Whoever moves, creates or deletes a ref does this afterwards. It is a list
142
+ for finding names, so it is written without a condition: the last writer
143
+ read the refs last.
144
+ """
145
+ head = read(store, layout.HEAD)
146
+ index = {"head": head.target if head else None, "refs": listed(store)}
147
+ store.put(
148
+ layout.REFS_INDEX,
149
+ (json.dumps(index, indent=2) + "\n").encode(),
150
+ content_type="application/json",
151
+ )
152
+ return index