fdl-cli 0.18.0__tar.gz → 0.18.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/CHANGELOG.md +21 -0
  2. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/PKG-INFO +1 -1
  3. fdl_cli-0.18.2/docs/internals/from-write-to-read.md +109 -0
  4. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/internals/meta.ts +1 -1
  5. fdl_cli-0.18.2/docs/ja/internals/from-write-to-read.md +109 -0
  6. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/internals/meta.ts +1 -1
  7. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/reference/cli.md +1 -1
  8. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/reference/python.md +6 -0
  9. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/reference/cli.md +1 -1
  10. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/reference/python.md +6 -0
  11. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/pyproject.toml +1 -1
  12. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/src/fdl/catalog.py +9 -2
  13. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/src/fdl/cli.py +2 -3
  14. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/src/fdl/repo.py +25 -2
  15. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/tests/test_http.py +23 -0
  16. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/.github/workflows/docs.yml +0 -0
  17. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/.github/workflows/publish.yml +0 -0
  18. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/.github/workflows/release.yml +0 -0
  19. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/.gitignore +0 -0
  20. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/.node-version +0 -0
  21. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/.python-version +0 -0
  22. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/CLAUDE.md +0 -0
  23. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/CONTRIBUTING.md +0 -0
  24. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/LICENSE +0 -0
  25. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/README.md +0 -0
  26. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/blume.config.ts +0 -0
  27. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/concepts/meta.ts +0 -0
  28. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/concepts/repositories.md +0 -0
  29. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/concepts/versions-and-refs.md +0 -0
  30. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/concepts/working-catalogs.md +0 -0
  31. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/getting-started.md +0 -0
  32. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/guides/building-a-host.md +0 -0
  33. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/guides/meta.ts +0 -0
  34. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/guides/mirrors.md +0 -0
  35. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/guides/public-and-private.md +0 -0
  36. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/guides/reading-without-duckdb.md +0 -0
  37. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/guides/serving-over-http.md +0 -0
  38. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/guides/stores.md +0 -0
  39. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/index.md +0 -0
  40. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/internals/ducklake-formats.md +0 -0
  41. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/internals/inside-a-repository.md +0 -0
  42. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/concepts/meta.ts +0 -0
  43. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/concepts/repositories.md +0 -0
  44. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/concepts/versions-and-refs.md +0 -0
  45. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/concepts/working-catalogs.md +0 -0
  46. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/getting-started.md +0 -0
  47. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/guides/building-a-host.md +0 -0
  48. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/guides/meta.ts +0 -0
  49. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/guides/mirrors.md +0 -0
  50. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/guides/public-and-private.md +0 -0
  51. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/guides/reading-without-duckdb.md +0 -0
  52. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/guides/serving-over-http.md +0 -0
  53. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/guides/stores.md +0 -0
  54. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/index.md +0 -0
  55. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/internals/ducklake-formats.md +0 -0
  56. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/internals/inside-a-repository.md +0 -0
  57. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/reference/meta.ts +0 -0
  58. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/spec/data-files.md +0 -0
  59. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/spec/http.md +0 -0
  60. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/spec/index.md +0 -0
  61. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/spec/layout.md +0 -0
  62. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/spec/meta.ts +0 -0
  63. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/spec/reading.md +0 -0
  64. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/spec/refs.md +0 -0
  65. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/spec/versions.md +0 -0
  66. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/spec/writing.md +0 -0
  67. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/reference/meta.ts +0 -0
  68. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/spec/data-files.md +0 -0
  69. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/spec/http.md +0 -0
  70. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/spec/index.md +0 -0
  71. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/spec/layout.md +0 -0
  72. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/spec/meta.ts +0 -0
  73. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/spec/reading.md +0 -0
  74. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/spec/refs.md +0 -0
  75. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/spec/versions.md +0 -0
  76. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/spec/writing.md +0 -0
  77. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/package-lock.json +0 -0
  78. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/package.json +0 -0
  79. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/public/CNAME +0 -0
  80. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/src/fdl/__init__.py +0 -0
  81. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/src/fdl/dataset.py +0 -0
  82. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/src/fdl/layout.py +0 -0
  83. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/src/fdl/py.typed +0 -0
  84. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/src/fdl/refs.py +0 -0
  85. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/src/fdl/store.py +0 -0
  86. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/src/fdl/stores.py +0 -0
  87. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/src/fdl/version.py +0 -0
  88. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/tests/http_repo.py +0 -0
  89. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/tests/test_cli.py +0 -0
  90. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/tests/test_dataset.py +0 -0
  91. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/tests/test_repo.py +0 -0
  92. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/tests/test_stores.py +0 -0
  93. {fdl_cli-0.18.0 → fdl_cli-0.18.2}/uv.lock +0 -0
@@ -1,6 +1,27 @@
1
1
  # CHANGELOG
2
2
 
3
3
 
4
+ ## v0.18.2 (2026-11-03)
5
+
6
+ ### Bug Fixes
7
+
8
+ - Refuse a version past retention in fdl url, and explain how FDL works
9
+ ([#35](https://github.com/queria-io/fdl/pull/35),
10
+ [`1130813`](https://github.com/queria-io/fdl/commit/1130813857d6dd5596c3b7fbb6ef3717b94c16f6))
11
+
12
+
13
+ ## v0.18.1 (2026-11-03)
14
+
15
+ ### Bug Fixes
16
+
17
+ - Compare a rebuild over HTTP with the new files where they were written
18
+ ([`c0f091a`](https://github.com/queria-io/fdl/commit/c0f091a76831b58afa91be16643e3b08ae9ae2c1))
19
+
20
+ A working catalog of a repository over HTTP writes new data files to a local folder, but comparing
21
+ with the parent read them under the repository's URL. The read failed (404), the commit fell back
22
+ to making a version, and every rebuild made one even when nothing changed.
23
+
24
+
4
25
  ## v0.18.0 (2026-11-03)
5
26
 
6
27
  ### Bug Fixes
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: fdl-cli
3
- Version: 0.18.0
3
+ Version: 0.18.2
4
4
  Summary: Versions of a DuckLake catalog on object storage, kept the way Git keeps commits
5
5
  Project-URL: Homepage, https://fdl.flo8s.com/
6
6
  Project-URL: Documentation, https://fdl.flo8s.com/
@@ -0,0 +1,109 @@
1
+ ---
2
+ title: From write to read
3
+ description: One change followed from a working catalog to a reader, with the requests at each step, and what each part of FDL is in Git.
4
+ ---
5
+
6
+ This page follows one change from the writer's machine to a reader, and names the request made at each step. The files each step leaves in the repository are shown in [Inside a repository](/internals/inside-a-repository), and the rules are in the [specification](/spec).
7
+
8
+ ## The parts and their Git counterparts
9
+
10
+ FDL keeps the roles and names of Git, and stores things its own way. Anyone who knows how Git keeps commits can read a repository by this table.
11
+
12
+ | FDL | Git | Notes |
13
+ | --- | --- | --- |
14
+ | Repository (a prefix of a store) | Bare repository on a server | No server process is needed |
15
+ | `fdl.json` | `config` | Holds the format, the repository's `id` and `data_dir` |
16
+ | `versions/<ID>/version.json` | Commit | Parents, author, message, and the sha256 of the catalog |
17
+ | `versions/<ID>/ducklake.duckdb` | Tree | Which tables there are and which data files they are made of |
18
+ | Data file (Parquet) | Blob | Written once and never changed. One file can be named by many versions |
19
+ | `refs/heads/<name>` | Branch | Moves forward only, on condition |
20
+ | `refs/tags/<name>` | Tag | Created once, never moved |
21
+ | `HEAD` | `HEAD` | Names the default branch |
22
+ | `refs.json` | `info/refs` | Lets a reader over plain HTTP find refs by name |
23
+ | `versions/<ID>/files.json` | Reachability bitmap | An index made from the catalog. A reader can do without it |
24
+ | `version.json.sig` | Signed commit | An SSH signature made with `ssh-keygen -Y sign` |
25
+ | Working catalog (SQLite) | Working tree and index | Made from a version and turned into the next one |
26
+ | `POST <root>/?batch` | Git LFS batch API | Hands out URLs for uploading straight to storage |
27
+ | `expired.json` | None | Says why a version has no catalog any more |
28
+
29
+ Three things differ on purpose:
30
+
31
+ - A version's ID is a UUIDv7, not a hash of its content. Two DuckDB releases can write the same catalog as different bytes, so a content hash would change with the tool. The sha256 in `version.json` checks the content instead
32
+ - There are no packfiles. Each file is its own object in the store, so a reader fetches only the parts of a Parquet file it needs
33
+ - There is no merge. A version has one parent, and a writer whose push is refused builds again on the newer version
34
+
35
+ ## Writing a version
36
+
37
+ ### Checkout
38
+
39
+ `fdl checkout` reads `HEAD`, then the ref it names, then that version's `version.json`, and copies its catalog into a SQLite file on the writer's machine. This is the working catalog. If the version has `expired.json`, checkout stops.
40
+
41
+ The working catalog's data path depends on the store:
42
+
43
+ - A folder or S3: the repository's data folder. DuckDB writes new Parquet files straight into the repository
44
+ - HTTP: a folder on the writer's machine. Files already in the repository are referred to by their URL under `data_dir` and are read over HTTP
45
+
46
+ ### Writing
47
+
48
+ The writer runs SQL on the working catalog with DuckDB, directly or through a tool such as dbt. Each statement makes a DuckLake snapshot in the working catalog. Nothing a reader can see changes yet: the new Parquet files, if they are already in the repository, are named by no version.
49
+
50
+ ### Commit
51
+
52
+ `fdl commit` copies the working catalog into a DuckDB file, sets its `data_path` to the repository's data folder, makes `dataset.json` from it, and writes `version.json` with a new ID and the previous version as its parent. If the tables hold the same rows as the parent's, there is no commit. The new version exists only on the writer's machine until it is pushed.
53
+
54
+ ### Push
55
+
56
+ A push writes files in the order that keeps readers safe: whatever a file points at is written before it.
57
+
58
+ 1. The new data files the repository lacks
59
+ 2. `versions/<ID>/ducklake.duckdb`
60
+ 3. `versions/<ID>/dataset.json` and `files.json`
61
+ 4. `versions/<ID>/version.json`
62
+ 5. The ref, with `If-Match: <ETag read before step 1>`, or `If-None-Match: *` for the first version
63
+ 6. `refs.json`, rewritten from a fresh read of `HEAD` and every ref
64
+
65
+ Readers start from a ref, and the ref is written in step 5. A push that stops before then leaves files that no ref names, and readers never see them. If another writer moved the ref after step 1, the ETag no longer matches and the store refuses step 5. That writer checks out again and builds on the new version.
66
+
67
+ `refs.json` is written after the ref, so it may lag behind for a moment. That is why a reader that needs the current version reads the ref itself.
68
+
69
+ ### Push over HTTP
70
+
71
+ Over HTTP, every key is a path under the repository's URL, and the host answers `GET`, `PUT` and the rest for it (see [Pushing over HTTP](/spec/http)). The order is the same as above. Two steps change:
72
+
73
+ - Before step 1, the writer sends `POST <root>/?batch` with the keys and sizes of the files it is about to upload. For each file, the host answers with an upload URL, says the file is already there, or answers 413 to have it sent through the host. Files with a URL go straight to storage with `PUT`, and the host is not in the path of the data. A host without batches answers 404, 405 or 501, and every file goes through the host
74
+ - In step 5, the host checks the push before it moves the ref: the version is complete, its parent is the version the ref holds now, the catalog matches `version.json`, and every data file the catalog names is in the repository. It answers 409 with the reason when a check fails. A host that moves refs itself also writes `refs.json`, and answers 403 when the writer tries to
75
+
76
+ A host can also keep writers from moving refs at all. The writer then uploads the version and stops after step 4, and the host moves the ref later, for example after someone approves the change.
77
+
78
+ ## Reading a version
79
+
80
+ A reader needs only `GET`. It does not have to know who wrote the repository or whether a host is in front of it.
81
+
82
+ 1. Read `HEAD`, or the ref asked for. `HEAD` holds `ref: refs/heads/main`, so read `refs/heads/main` next. A ref holds a version ID
83
+ 2. Read `versions/<ID>/version.json`. Check that its `id` is the ID asked for and that the reader knows the major version in `fdl`
84
+ 3. Check for `versions/<ID>/expired.json`. If it is there, stop: the version is past retention or withdrawn, and its catalog is gone
85
+ 4. Attach the catalog read-only, with the repository's data folder as the data path
86
+
87
+ ```sql
88
+ ATTACH 'ducklake:https://example.org/lake/versions/019a3c4e-…/ducklake.duckdb' AS lake (
89
+ DATA_PATH 'https://example.org/lake/data/', OVERRIDE_DATA_PATH true, READ_ONLY
90
+ );
91
+ ```
92
+
93
+ From here DuckDB reads the catalog and fetches the parts of the Parquet files a query needs. `fdl url` does steps 1 to 3 and prints the catalog's URL for step 4.
94
+
95
+ The reader passes the data path itself rather than using the one in the catalog. The catalog holds the URL where the writer published, and a mirror serves the same files somewhere else. With the data path given by the reader, the same statement works at the origin and at any mirror.
96
+
97
+ A version's catalog never changes, so it can be cached for good. `HEAD`, `refs/` and `refs.json` are served with `Cache-Control: no-store`, so a reader always sees where a ref points now.
98
+
99
+ ### Readers that do not know FDL
100
+
101
+ A host may serve the catalog of the version `HEAD` holds at `<root>/ducklake.duckdb`. A reader that knows only DuckLake can then attach that one URL. The host sends the version ID in the `FDL-Version` header, so that a reader can find out which version it got. The store holds no file at that key: the host serves the catalog from `versions/`.
102
+
103
+ ## Taking a version away
104
+
105
+ `fdl expire` takes old versions out of retention. It puts `expired.json` first, then deletes the catalog, and leaves `version.json`, so `fdl log` still shows the version and its place in the history. Data files that no live version names are deleted later, after a grace period, because a writer may be about to commit a version that names them.
106
+
107
+ Withdrawing removes data that must not be published, such as personal data published by mistake, from every version that holds it. The writer first makes a new version without the data and moves each affected branch forward to it, and deletes the tags that point at affected versions. Then the data files that hold the data are deleted at once, with no grace period, and each affected version gets `expired.json` with `"reason": "withdrawn"` and loses its catalog. The steps are in [Data files](/spec/data-files#withdrawal).
108
+
109
+ A reader that reaches an expired or withdrawn version stops at step 3 of reading and reports the reason.
@@ -3,5 +3,5 @@ import { defineMeta } from "blume";
3
3
  export default defineMeta({
4
4
  title: "How it works",
5
5
  order: 3,
6
- pages: ["inside-a-repository", "ducklake-formats"],
6
+ pages: ["from-write-to-read", "inside-a-repository", "ducklake-formats"],
7
7
  });
@@ -0,0 +1,109 @@
1
+ ---
2
+ title: 書いてから読まれるまで
3
+ description: 1 つの変更を作業カタログから読み手まで追い、各段階のリクエストと、FDL の各部分が Git の何に当たるかを示します。
4
+ ---
5
+
6
+ このページでは、1 つの変更を書き手のマシンから読み手まで追い、各段階で送るリクエストを示します。各段階でリポジトリに残るファイルは[リポジトリの中身](/ja/internals/inside-a-repository)に、決まりは[仕様](/ja/spec)にあります。
7
+
8
+ ## 各部分と Git での呼び名
9
+
10
+ FDL は Git の役割と名前をそのまま使い、保存の仕方は独自に決めています。Git がコミットをどう持つかを知っていれば、この表でリポジトリを読めます。
11
+
12
+ | FDL | Git | 補足 |
13
+ | --- | --- | --- |
14
+ | リポジトリ(ストアの 1 つのプレフィックス) | サーバー上のベアリポジトリ | サーバーのプロセスは要りません |
15
+ | `fdl.json` | `config` | 形式、リポジトリの `id`、`data_dir` を持ちます |
16
+ | `versions/<ID>/version.json` | コミット | 親、作者、メッセージ、カタログの sha256 を持ちます |
17
+ | `versions/<ID>/ducklake.duckdb` | ツリー | どのテーブルがあり、それぞれがどのデータファイルでできているかを持ちます |
18
+ | データファイル(Parquet) | ブロブ | 一度書いたら変えません。1 つのファイルを多くのバージョンが指せます |
19
+ | `refs/heads/<name>` | ブランチ | 条件付き書き込みで、前にだけ進みます |
20
+ | `refs/tags/<name>` | タグ | 一度作ったら動かしません |
21
+ | `HEAD` | `HEAD` | 既定のブランチの名前を持ちます |
22
+ | `refs.json` | `info/refs` | 素の HTTP の読み手が ref を名前で探せるようにします |
23
+ | `versions/<ID>/files.json` | 到達可能性ビットマップ | カタログから作る索引です。読み手はこれが無くても読めます |
24
+ | `version.json.sig` | 署名付きコミット | `ssh-keygen -Y sign` で作る SSH 署名です |
25
+ | 作業カタログ(SQLite) | 作業ツリーとインデックス | バージョンから作り、次のバージョンにします |
26
+ | `POST <root>/?batch` | Git LFS の batch API | ストレージへ直接アップロードする URL を渡します |
27
+ | `expired.json` | なし | バージョンにカタログが無くなった理由を書きます |
28
+
29
+ 次の 3 つは、意図して Git と変えています。
30
+
31
+ - バージョンの ID は UUIDv7 で、中身のハッシュではありません。DuckDB のリリースが違うと、同じカタログでも違うバイト列で書かれることがあり、中身のハッシュは道具によって変わってしまうためです。中身の確認には `version.json` の sha256 を使います
32
+ - packfile はありません。ファイルはそれぞれストアの 1 つのオブジェクトなので、読み手は Parquet ファイルのうち要る部分だけを取りに行けます
33
+ - マージはありません。バージョンの親は 1 つです。push を断られた書き手は、新しいバージョンの上で作り直します
34
+
35
+ ## バージョンを書く
36
+
37
+ ### checkout
38
+
39
+ `fdl checkout` は `HEAD` を読み、それが指す ref を読み、そのバージョンの `version.json` を読んでから、カタログを書き手のマシンの SQLite ファイルに写します。これが作業カタログです。バージョンに `expired.json` があれば、checkout は止まります。
40
+
41
+ 作業カタログのデータパスは、ストアによって変わります。
42
+
43
+ - フォルダか S3: リポジトリのデータフォルダです。DuckDB は新しい Parquet ファイルをリポジトリへ直接書きます
44
+ - HTTP: 書き手のマシンのフォルダです。リポジトリに既にあるファイルは `data_dir` の下の URL で指し、HTTP で読みます
45
+
46
+ ### 書く
47
+
48
+ 書き手は作業カタログに対して、DuckDB で直接、または dbt などの道具を通して SQL を流します。文ごとに作業カタログに DuckLake のスナップショットができます。読み手から見えるものはまだ何も変わりません。新しい Parquet ファイルがリポジトリに書かれていても、どのバージョンもそれを指していないためです。
49
+
50
+ ### commit
51
+
52
+ `fdl commit` は作業カタログを DuckDB ファイルに写し、その `data_path` をリポジトリのデータフォルダにし、そこから `dataset.json` を作り、新しい ID と、親として前のバージョンを持つ `version.json` を書きます。テーブルの行が親と同じなら、コミットしません。新しいバージョンは、push するまで書き手のマシンにしかありません。
53
+
54
+ ### push
55
+
56
+ push は、読み手を守る順にファイルを書きます。あるファイルが指す先は、そのファイルより先に書きます。
57
+
58
+ 1. リポジトリに無い新しいデータファイル
59
+ 2. `versions/<ID>/ducklake.duckdb`
60
+ 3. `versions/<ID>/dataset.json` と `files.json`
61
+ 4. `versions/<ID>/version.json`
62
+ 5. ref。`If-Match: <1 の前に読んだ ETag>` で書きます。最初のバージョンなら `If-None-Match: *` です
63
+ 6. `refs.json`。`HEAD` とすべての ref を読み直して書き直します
64
+
65
+ 読み手は ref から読み始め、ref は 5 で書きます。5 より前で止まった push は、どの ref も指さないファイルを残すだけで、読み手からは見えません。1 のあとに別の書き手が ref を動かしていれば、ETag が合わず、ストアは 5 を断ります。断られた書き手は checkout し直し、新しいバージョンの上で作り直します。
66
+
67
+ `refs.json` は ref のあとに書くので、少しの間は ref より古いことがあります。そのため、今のバージョンが要る読み手は ref 自体を読みます。
68
+
69
+ ### HTTP での push
70
+
71
+ HTTP では、すべてのキーがリポジトリの URL の下のパスになり、ホストがそこへの `GET` や `PUT` などに答えます([HTTP での push](/ja/spec/http)を参照)。順番は上と同じで、2 つの段階が変わります。
72
+
73
+ - 1 の前に、書き手はこれから上げるファイルのキーと大きさを `POST <root>/?batch` で送ります。ホストはファイルごとに、アップロード用の URL を返すか、既にあると答えるか、ホスト経由で送らせるために 413 を返します。URL をもらったファイルは `PUT` でストレージへ直接送られ、データはホストを通りません。batch の無いホストは 404・405・501 のどれかを返し、すべてのファイルがホストを通ります
74
+ - 5 で、ホストは ref を動かす前に push を確かめます。バージョンがそろっていること、その親が今 ref の指すバージョンであること、カタログが `version.json` と合うこと、カタログが指すデータファイルがすべてリポジトリにあることです。確認に通らなければ、理由を付けて 409 を返します。ref を自分で動かすホストは `refs.json` も自分で書き、書き手が書こうとすると 403 を返します
75
+
76
+ ホストは、書き手に ref を一切動かさせないこともできます。その場合、書き手はバージョンを上げて 4 で止まり、ホストがあとで、たとえば誰かが変更を承認したときに ref を動かします。
77
+
78
+ ## バージョンを読む
79
+
80
+ 読み手に要るのは `GET` だけです。誰がリポジトリを書いたかも、前にホストがいるかどうかも知らなくて構いません。
81
+
82
+ 1. `HEAD` か、求められた ref を読みます。`HEAD` は `ref: refs/heads/main` を持つので、次に `refs/heads/main` を読みます。ref はバージョンの ID を持ちます
83
+ 2. `versions/<ID>/version.json` を読みます。`id` が求めた ID と同じで、`fdl` のメジャーバージョンを読み手が知っていることを確かめます
84
+ 3. `versions/<ID>/expired.json` があるか確かめます。あれば止まります。そのバージョンは保持期間を過ぎたか取り下げられていて、カタログはもうありません
85
+ 4. リポジトリのデータフォルダをデータパスにして、カタログを読み取り専用で ATTACH します
86
+
87
+ ```sql
88
+ ATTACH 'ducklake:https://example.org/lake/versions/019a3c4e-…/ducklake.duckdb' AS lake (
89
+ DATA_PATH 'https://example.org/lake/data/', OVERRIDE_DATA_PATH true, READ_ONLY
90
+ );
91
+ ```
92
+
93
+ ここから先は DuckDB がカタログを読み、クエリに要る Parquet ファイルの部分を取りに行きます。`fdl url` は 1 から 3 までを行い、4 で使うカタログの URL を出します。
94
+
95
+ 読み手はカタログに書かれたデータパスを使わず、自分で渡します。カタログが持つのは書き手が公開した場所の URL で、ミラーは同じファイルを別の場所から配るためです。読み手がデータパスを渡せば、同じ文が元のホストでもどのミラーでも動きます。
96
+
97
+ バージョンのカタログは変わらないので、ずっとキャッシュできます。`HEAD`・`refs/`・`refs.json` は `Cache-Control: no-store` で配るので、読み手は ref が今どこを指しているかを必ず知ることができます。
98
+
99
+ ### FDL を知らない読み手
100
+
101
+ ホストは、`HEAD` が指すバージョンのカタログを `<root>/ducklake.duckdb` で配ってもかまいません。DuckLake しか知らない読み手は、その 1 つの URL を ATTACH できます。ホストは `FDL-Version` ヘッダーでバージョンの ID を送るので、読み手はどのバージョンを受け取ったかを知ることができます。ストアのそのキーにはファイルがありません。ホストが `versions/` のカタログを配ります。
102
+
103
+ ## バージョンを消す
104
+
105
+ `fdl expire` は古いバージョンを保持期間から外します。先に `expired.json` を置き、次にカタログを消し、`version.json` は残します。そのため `fdl log` はそのバージョンと履歴の中の位置を表示し続けます。どの生きているバージョンも指さないデータファイルは、猶予期間のあとで消します。書き手がそのファイルを指すバージョンをこれからコミットするかもしれないためです。
106
+
107
+ 取り下げは、間違って公開した個人データなど、公開してはいけないデータを、それを持つすべてのバージョンから消します。書き手はまず、そのデータを除いた新しいバージョンを作り、影響を受けるブランチをそこへ前に進め、影響を受けるバージョンを指すタグを消します。次に、そのデータを持つデータファイルを猶予期間を置かずにすぐ消し、影響を受けるバージョンに `"reason": "withdrawn"` の `expired.json` を置いてカタログを消します。手順は[データファイル](/ja/spec/data-files#取り下げ)にあります。
108
+
109
+ 保持期間を過ぎたか取り下げられたバージョンにたどり着いた読み手は、読む手順の 3 で止まり、理由を伝えます。
@@ -3,5 +3,5 @@ import { defineMeta } from "blume";
3
3
  export default defineMeta({
4
4
  title: "仕組み",
5
5
  order: 3,
6
- pages: ["inside-a-repository", "ducklake-formats"],
6
+ pages: ["from-write-to-read", "inside-a-repository", "ducklake-formats"],
7
7
  });
@@ -75,7 +75,7 @@ ref が指すバージョンから最初のバージョンまで、最初の親
75
75
  fdl url <remote> [--ref HEAD]
76
76
  ```
77
77
 
78
- ref が指すバージョンのカタログの URL を表示します。読み取り専用で `ATTACH` するのに使います。
78
+ ref が指すバージョンのカタログの URL を表示します。読み取り専用で `ATTACH` するのに使います。保持期間を過ぎたか取り下げたバージョンは、カタログがまだ配られていても断ります。
79
79
 
80
80
  ## fdl tag
81
81
 
@@ -75,6 +75,12 @@ repo.is_ancestor(store, ancestor_id, descendant) -> bool
75
75
 
76
76
  最初の親をたどる、ref が指すバージョンを読む、あるバージョンが別のバージョンより前にあるかを確かめる、の 3 つです。
77
77
 
78
+ ```python
79
+ repo.readable(store, name="HEAD") -> Version
80
+ ```
81
+
82
+ ref が指すバージョンを、読み手がカタログを開く前に行う確認を済ませてから返します。最初のバージョンの前、読めない形式の `version.json`、保持期間を過ぎたか取り下げたバージョンでは `RepoError` を出します。`fdl url` はこれを使います。
83
+
78
84
  ```python
79
85
  repo.expire(store, version_id, *, reason="retention") -> None
80
86
  repo.prune(store, newest, *, keep=1, since=None, reason="retention") -> list[str]
@@ -75,7 +75,7 @@ Versions from the one the ref holds back to the first, by first parent. Each sho
75
75
  fdl url <remote> [--ref HEAD]
76
76
  ```
77
77
 
78
- Prints the URL of the catalog of the version the ref holds, to `ATTACH` read-only.
78
+ Prints the URL of the catalog of the version the ref holds, to `ATTACH` read-only. A version past retention or withdrawn is refused, even where its catalog is still served.
79
79
 
80
80
  ## fdl tag
81
81
 
@@ -75,6 +75,12 @@ repo.is_ancestor(store, ancestor_id, descendant) -> bool
75
75
 
76
76
  Walk the first-parent line, read the version a ref holds, and check whether a version is behind another.
77
77
 
78
+ ```python
79
+ repo.readable(store, name="HEAD") -> Version
80
+ ```
81
+
82
+ The version a ref holds, checked the way a reader must before opening its catalog. Raises `RepoError` before the first version, for a `version.json` of a format this does not read, and for a version past retention or withdrawn. `fdl url` uses it.
83
+
78
84
  ```python
79
85
  repo.expire(store, version_id, *, reason="retention") -> None
80
86
  repo.prune(store, newest, *, keep=1, since=None, reason="retention") -> list[str]
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "fdl-cli"
3
- version = "0.18.0"
3
+ version = "0.18.2"
4
4
  description = "Versions of a DuckLake catalog on object storage, kept the way Git keeps commits"
5
5
  readme = "README.md"
6
6
  license = "MIT"
@@ -659,9 +659,13 @@ def last_snapshot(working: Path) -> int:
659
659
  con.close()
660
660
 
661
661
 
662
- def same_contents(working: Path, since: int) -> bool:
662
+ def same_contents(working: Path, since: int, *, data_path: str | None = None) -> bool:
663
663
  """Whether the working catalog's last snapshot holds what snapshot ``since`` held.
664
664
 
665
+ ``data_path`` is where the working catalog's new data files are, when not
666
+ where it records (a working catalog of a repository over HTTP writes them
667
+ to a folder on this machine).
668
+
665
669
  The same tables and views, each table with the same columns and the same
666
670
  rows, in any order. A table rebuilt from the same input is written to new
667
671
  files, so files are compared first and rows only where the files differ.
@@ -677,7 +681,10 @@ def same_contents(working: Path, since: int) -> bool:
677
681
  con = duckdb.connect()
678
682
  try:
679
683
  for statement in attach_sql(
680
- working, alias="lake", data_path=_working_data_path(working), read_only=True
684
+ working,
685
+ alias="lake",
686
+ data_path=data_path or _working_data_path(working),
687
+ read_only=True,
681
688
  ):
682
689
  if statement:
683
690
  con.execute(statement)
@@ -127,6 +127,7 @@ def commit(
127
127
  message=message,
128
128
  dataset=declared,
129
129
  allow_empty=allow_empty,
130
+ working_data=Path(state["data"]).resolve().as_posix() + "/" if state.get("data") else None,
130
131
  )
131
132
 
132
133
  try:
@@ -167,11 +168,9 @@ def url(remote: str = Remote, ref: str = typer.Option(layout.HEAD)) -> None:
167
168
  """Where the catalog of the version a ref holds is, to ATTACH."""
168
169
  store = open_store(remote)
169
170
  try:
170
- version = repo.head(store, ref)
171
+ version = repo.readable(store, ref)
171
172
  except ERRORS as exc:
172
173
  raise _fail(exc) from exc
173
- if version is None:
174
- raise _fail(repo.RepoError(f"{ref} holds no version yet"))
175
174
  typer.echo(store.url.rstrip("/") + "/" + layout.version_catalog(version.id))
176
175
 
177
176
 
@@ -21,7 +21,7 @@ from typing import Iterable, Iterator
21
21
  from . import catalog, layout, refs
22
22
  from . import dataset as dataset_json
23
23
  from .store import Store, is_conflict, is_missing
24
- from .version import Catalog, DuckLake, File, Version, new_id, parse, utc_now
24
+ from .version import Catalog, DuckLake, File, Version, VersionError, new_id, parse, utc_now
25
25
 
26
26
 
27
27
  class RepoError(RuntimeError):
@@ -109,6 +109,23 @@ def head(store: Store, name: str = layout.HEAD) -> Version | None:
109
109
  return read_version(store, version_id) if version_id else None
110
110
 
111
111
 
112
+ def readable(store: Store, name: str = layout.HEAD) -> Version:
113
+ """The version a ref holds, checked the way a reader must (Reading, steps 1-3).
114
+
115
+ Raises `RepoError` before the first version, for a record of a format this
116
+ does not read, and for a version past retention or withdrawn.
117
+ """
118
+ try:
119
+ version = head(store, name)
120
+ except VersionError as exc:
121
+ raise RepoError(str(exc)) from exc
122
+ if version is None:
123
+ raise RepoError(f"{name} holds no version yet")
124
+ if is_expired(store, version.id):
125
+ raise RepoError(f"{version.id} is past retention or withdrawn and has no catalog")
126
+ return version
127
+
128
+
112
129
  def commit(
113
130
  working: Path,
114
131
  out_dir: Path,
@@ -121,6 +138,7 @@ def commit(
121
138
  annotations: dict | None = None,
122
139
  dataset: dict | None = None,
123
140
  allow_empty: bool = True,
141
+ working_data: str | None = None,
124
142
  ) -> Version:
125
143
  """Freeze the working catalog into a version under ``out_dir/<ID>/``.
126
144
 
@@ -136,6 +154,9 @@ def commit(
136
154
  data_dir), written into the frozen catalog. Every file reference must be
137
155
  relative to it, or the version would point readers at the writer's disk.
138
156
  Nothing is uploaded: that is `push`.
157
+
158
+ ``working_data`` is where the working catalog wrote its new data files,
159
+ when not where it records (see `checkout`): comparing reads them there.
139
160
  """
140
161
  Path(out_dir).mkdir(parents=True, exist_ok=True)
141
162
  frozen = Path(out_dir) / f".freezing-{new_id()}.duckdb"
@@ -154,7 +175,9 @@ def commit(
154
175
  # the contents can read every row of a rebuilt table.
155
176
  if not allow_empty and parent is not None and _same_description(frozen, dataset, parent):
156
177
  try:
157
- unchanged = catalog.same_contents(working, parent.ducklake.snapshot_id)
178
+ unchanged = catalog.same_contents(
179
+ working, parent.ducklake.snapshot_id, data_path=working_data
180
+ )
158
181
  except catalog.CatalogError as exc:
159
182
  raise CannotCompare(str(exc)) from exc
160
183
  if unchanged:
@@ -157,3 +157,26 @@ def test_a_file_the_host_will_not_sign_goes_through_the_host(tmp_path: Path) ->
157
157
  assert local.get("data/small.parquet") == b"ab" and local.get("data/large.parquet") == b"abcdef"
158
158
  puts = [key for method, key in server.requests if method == "PUT"]
159
159
  assert puts == ["storage/data/small.parquet", "data/large.parquet"]
160
+
161
+
162
+ def test_a_rebuild_of_the_same_rows_over_http_is_nothing_to_commit(served, tmp_path: Path) -> None:
163
+ local, url, _ = served
164
+ working = tmp_path / "work" / "lake.sqlite"
165
+ run("checkout", url, working)
166
+ write(working, "CREATE SCHEMA lake.s; CREATE TABLE lake.s.towns AS SELECT 1 AS id")
167
+ run("commit", working, "-m", "first", "--author", "tester")
168
+ # The way dbt rebuilds a table: a new table beside it, then renames.
169
+ again = tmp_path / "again" / "lake.sqlite"
170
+ run("checkout", url, again)
171
+ write(
172
+ again,
173
+ "CREATE TABLE lake.s.towns__dbt_tmp AS SELECT 1 AS id; "
174
+ "ALTER TABLE lake.s.towns RENAME TO towns__dbt_backup; "
175
+ "ALTER TABLE lake.s.towns__dbt_tmp RENAME TO towns; DROP TABLE lake.s.towns__dbt_backup",
176
+ )
177
+ working = again
178
+ result = runner.invoke(app, ["commit", str(working), "-m", "again", "--author", "tester"])
179
+ assert result.exit_code == 0, result.output
180
+ assert "Nothing to commit" in result.output
181
+ assert "could not tell" not in result.output
182
+ assert len(list(repo.log(local))) == 1
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes