fdl-cli 0.18.0__tar.gz → 0.18.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/CHANGELOG.md +21 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/PKG-INFO +1 -1
- fdl_cli-0.18.2/docs/internals/from-write-to-read.md +109 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/internals/meta.ts +1 -1
- fdl_cli-0.18.2/docs/ja/internals/from-write-to-read.md +109 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/internals/meta.ts +1 -1
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/reference/cli.md +1 -1
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/reference/python.md +6 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/reference/cli.md +1 -1
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/reference/python.md +6 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/pyproject.toml +1 -1
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/src/fdl/catalog.py +9 -2
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/src/fdl/cli.py +2 -3
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/src/fdl/repo.py +25 -2
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/tests/test_http.py +23 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/.github/workflows/docs.yml +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/.github/workflows/publish.yml +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/.github/workflows/release.yml +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/.gitignore +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/.node-version +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/.python-version +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/CLAUDE.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/CONTRIBUTING.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/LICENSE +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/README.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/blume.config.ts +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/concepts/meta.ts +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/concepts/repositories.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/concepts/versions-and-refs.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/concepts/working-catalogs.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/getting-started.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/guides/building-a-host.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/guides/meta.ts +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/guides/mirrors.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/guides/public-and-private.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/guides/reading-without-duckdb.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/guides/serving-over-http.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/guides/stores.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/index.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/internals/inside-a-repository.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/concepts/meta.ts +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/concepts/repositories.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/concepts/versions-and-refs.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/concepts/working-catalogs.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/getting-started.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/guides/building-a-host.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/guides/meta.ts +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/guides/mirrors.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/guides/public-and-private.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/guides/reading-without-duckdb.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/guides/serving-over-http.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/guides/stores.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/index.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/internals/inside-a-repository.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/reference/meta.ts +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/spec/data-files.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/spec/http.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/spec/index.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/spec/layout.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/spec/meta.ts +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/spec/reading.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/spec/refs.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/spec/versions.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/ja/spec/writing.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/reference/meta.ts +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/spec/data-files.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/spec/http.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/spec/index.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/spec/layout.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/spec/meta.ts +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/spec/reading.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/spec/refs.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/spec/versions.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/docs/spec/writing.md +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/package-lock.json +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/package.json +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/public/CNAME +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/src/fdl/__init__.py +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/src/fdl/dataset.py +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/src/fdl/layout.py +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/src/fdl/py.typed +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/src/fdl/refs.py +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/src/fdl/store.py +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/src/fdl/stores.py +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/src/fdl/version.py +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/tests/http_repo.py +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/tests/test_cli.py +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/tests/test_dataset.py +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/tests/test_repo.py +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/tests/test_stores.py +0 -0
- {fdl_cli-0.18.0 → fdl_cli-0.18.2}/uv.lock +0 -0
|
@@ -1,6 +1,27 @@
|
|
|
1
1
|
# CHANGELOG
|
|
2
2
|
|
|
3
3
|
|
|
4
|
+
## v0.18.2 (2026-11-03)
|
|
5
|
+
|
|
6
|
+
### Bug Fixes
|
|
7
|
+
|
|
8
|
+
- Refuse a version past retention in fdl url, and explain how FDL works
|
|
9
|
+
([#35](https://github.com/queria-io/fdl/pull/35),
|
|
10
|
+
[`1130813`](https://github.com/queria-io/fdl/commit/1130813857d6dd5596c3b7fbb6ef3717b94c16f6))
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
## v0.18.1 (2026-11-03)
|
|
14
|
+
|
|
15
|
+
### Bug Fixes
|
|
16
|
+
|
|
17
|
+
- Compare a rebuild over HTTP with the new files where they were written
|
|
18
|
+
([`c0f091a`](https://github.com/queria-io/fdl/commit/c0f091a76831b58afa91be16643e3b08ae9ae2c1))
|
|
19
|
+
|
|
20
|
+
A working catalog of a repository over HTTP writes new data files to a local folder, but comparing
|
|
21
|
+
with the parent read them under the repository's URL. The read failed (404), the commit fell back
|
|
22
|
+
to making a version, and every rebuild made one even when nothing changed.
|
|
23
|
+
|
|
24
|
+
|
|
4
25
|
## v0.18.0 (2026-11-03)
|
|
5
26
|
|
|
6
27
|
### Bug Fixes
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: From write to read
|
|
3
|
+
description: One change followed from a working catalog to a reader, with the requests at each step, and what each part of FDL is in Git.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
This page follows one change from the writer's machine to a reader, and names the request made at each step. The files each step leaves in the repository are shown in [Inside a repository](/internals/inside-a-repository), and the rules are in the [specification](/spec).
|
|
7
|
+
|
|
8
|
+
## The parts and their Git counterparts
|
|
9
|
+
|
|
10
|
+
FDL keeps the roles and names of Git, and stores things its own way. Anyone who knows how Git keeps commits can read a repository by this table.
|
|
11
|
+
|
|
12
|
+
| FDL | Git | Notes |
|
|
13
|
+
| --- | --- | --- |
|
|
14
|
+
| Repository (a prefix of a store) | Bare repository on a server | No server process is needed |
|
|
15
|
+
| `fdl.json` | `config` | Holds the format, the repository's `id` and `data_dir` |
|
|
16
|
+
| `versions/<ID>/version.json` | Commit | Parents, author, message, and the sha256 of the catalog |
|
|
17
|
+
| `versions/<ID>/ducklake.duckdb` | Tree | Which tables there are and which data files they are made of |
|
|
18
|
+
| Data file (Parquet) | Blob | Written once and never changed. One file can be named by many versions |
|
|
19
|
+
| `refs/heads/<name>` | Branch | Moves forward only, on condition |
|
|
20
|
+
| `refs/tags/<name>` | Tag | Created once, never moved |
|
|
21
|
+
| `HEAD` | `HEAD` | Names the default branch |
|
|
22
|
+
| `refs.json` | `info/refs` | Lets a reader over plain HTTP find refs by name |
|
|
23
|
+
| `versions/<ID>/files.json` | Reachability bitmap | An index made from the catalog. A reader can do without it |
|
|
24
|
+
| `version.json.sig` | Signed commit | An SSH signature made with `ssh-keygen -Y sign` |
|
|
25
|
+
| Working catalog (SQLite) | Working tree and index | Made from a version and turned into the next one |
|
|
26
|
+
| `POST <root>/?batch` | Git LFS batch API | Hands out URLs for uploading straight to storage |
|
|
27
|
+
| `expired.json` | None | Says why a version has no catalog any more |
|
|
28
|
+
|
|
29
|
+
Three things differ on purpose:
|
|
30
|
+
|
|
31
|
+
- A version's ID is a UUIDv7, not a hash of its content. Two DuckDB releases can write the same catalog as different bytes, so a content hash would change with the tool. The sha256 in `version.json` checks the content instead
|
|
32
|
+
- There are no packfiles. Each file is its own object in the store, so a reader fetches only the parts of a Parquet file it needs
|
|
33
|
+
- There is no merge. A version has one parent, and a writer whose push is refused builds again on the newer version
|
|
34
|
+
|
|
35
|
+
## Writing a version
|
|
36
|
+
|
|
37
|
+
### Checkout
|
|
38
|
+
|
|
39
|
+
`fdl checkout` reads `HEAD`, then the ref it names, then that version's `version.json`, and copies its catalog into a SQLite file on the writer's machine. This is the working catalog. If the version has `expired.json`, checkout stops.
|
|
40
|
+
|
|
41
|
+
The working catalog's data path depends on the store:
|
|
42
|
+
|
|
43
|
+
- A folder or S3: the repository's data folder. DuckDB writes new Parquet files straight into the repository
|
|
44
|
+
- HTTP: a folder on the writer's machine. Files already in the repository are referred to by their URL under `data_dir` and are read over HTTP
|
|
45
|
+
|
|
46
|
+
### Writing
|
|
47
|
+
|
|
48
|
+
The writer runs SQL on the working catalog with DuckDB, directly or through a tool such as dbt. Each statement makes a DuckLake snapshot in the working catalog. Nothing a reader can see changes yet: the new Parquet files, if they are already in the repository, are named by no version.
|
|
49
|
+
|
|
50
|
+
### Commit
|
|
51
|
+
|
|
52
|
+
`fdl commit` copies the working catalog into a DuckDB file, sets its `data_path` to the repository's data folder, makes `dataset.json` from it, and writes `version.json` with a new ID and the previous version as its parent. If the tables hold the same rows as the parent's, there is no commit. The new version exists only on the writer's machine until it is pushed.
|
|
53
|
+
|
|
54
|
+
### Push
|
|
55
|
+
|
|
56
|
+
A push writes files in the order that keeps readers safe: whatever a file points at is written before it.
|
|
57
|
+
|
|
58
|
+
1. The new data files the repository lacks
|
|
59
|
+
2. `versions/<ID>/ducklake.duckdb`
|
|
60
|
+
3. `versions/<ID>/dataset.json` and `files.json`
|
|
61
|
+
4. `versions/<ID>/version.json`
|
|
62
|
+
5. The ref, with `If-Match: <ETag read before step 1>`, or `If-None-Match: *` for the first version
|
|
63
|
+
6. `refs.json`, rewritten from a fresh read of `HEAD` and every ref
|
|
64
|
+
|
|
65
|
+
Readers start from a ref, and the ref is written in step 5. A push that stops before then leaves files that no ref names, and readers never see them. If another writer moved the ref after step 1, the ETag no longer matches and the store refuses step 5. That writer checks out again and builds on the new version.
|
|
66
|
+
|
|
67
|
+
`refs.json` is written after the ref, so it may lag behind for a moment. That is why a reader that needs the current version reads the ref itself.
|
|
68
|
+
|
|
69
|
+
### Push over HTTP
|
|
70
|
+
|
|
71
|
+
Over HTTP, every key is a path under the repository's URL, and the host answers `GET`, `PUT` and the rest for it (see [Pushing over HTTP](/spec/http)). The order is the same as above. Two steps change:
|
|
72
|
+
|
|
73
|
+
- Before step 1, the writer sends `POST <root>/?batch` with the keys and sizes of the files it is about to upload. For each file, the host answers with an upload URL, says the file is already there, or answers 413 to have it sent through the host. Files with a URL go straight to storage with `PUT`, and the host is not in the path of the data. A host without batches answers 404, 405 or 501, and every file goes through the host
|
|
74
|
+
- In step 5, the host checks the push before it moves the ref: the version is complete, its parent is the version the ref holds now, the catalog matches `version.json`, and every data file the catalog names is in the repository. It answers 409 with the reason when a check fails. A host that moves refs itself also writes `refs.json`, and answers 403 when the writer tries to
|
|
75
|
+
|
|
76
|
+
A host can also keep writers from moving refs at all. The writer then uploads the version and stops after step 4, and the host moves the ref later, for example after someone approves the change.
|
|
77
|
+
|
|
78
|
+
## Reading a version
|
|
79
|
+
|
|
80
|
+
A reader needs only `GET`. It does not have to know who wrote the repository or whether a host is in front of it.
|
|
81
|
+
|
|
82
|
+
1. Read `HEAD`, or the ref asked for. `HEAD` holds `ref: refs/heads/main`, so read `refs/heads/main` next. A ref holds a version ID
|
|
83
|
+
2. Read `versions/<ID>/version.json`. Check that its `id` is the ID asked for and that the reader knows the major version in `fdl`
|
|
84
|
+
3. Check for `versions/<ID>/expired.json`. If it is there, stop: the version is past retention or withdrawn, and its catalog is gone
|
|
85
|
+
4. Attach the catalog read-only, with the repository's data folder as the data path
|
|
86
|
+
|
|
87
|
+
```sql
|
|
88
|
+
ATTACH 'ducklake:https://example.org/lake/versions/019a3c4e-…/ducklake.duckdb' AS lake (
|
|
89
|
+
DATA_PATH 'https://example.org/lake/data/', OVERRIDE_DATA_PATH true, READ_ONLY
|
|
90
|
+
);
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
From here DuckDB reads the catalog and fetches the parts of the Parquet files a query needs. `fdl url` does steps 1 to 3 and prints the catalog's URL for step 4.
|
|
94
|
+
|
|
95
|
+
The reader passes the data path itself rather than using the one in the catalog. The catalog holds the URL where the writer published, and a mirror serves the same files somewhere else. With the data path given by the reader, the same statement works at the origin and at any mirror.
|
|
96
|
+
|
|
97
|
+
A version's catalog never changes, so it can be cached for good. `HEAD`, `refs/` and `refs.json` are served with `Cache-Control: no-store`, so a reader always sees where a ref points now.
|
|
98
|
+
|
|
99
|
+
### Readers that do not know FDL
|
|
100
|
+
|
|
101
|
+
A host may serve the catalog of the version `HEAD` holds at `<root>/ducklake.duckdb`. A reader that knows only DuckLake can then attach that one URL. The host sends the version ID in the `FDL-Version` header, so that a reader can find out which version it got. The store holds no file at that key: the host serves the catalog from `versions/`.
|
|
102
|
+
|
|
103
|
+
## Taking a version away
|
|
104
|
+
|
|
105
|
+
`fdl expire` takes old versions out of retention. It puts `expired.json` first, then deletes the catalog, and leaves `version.json`, so `fdl log` still shows the version and its place in the history. Data files that no live version names are deleted later, after a grace period, because a writer may be about to commit a version that names them.
|
|
106
|
+
|
|
107
|
+
Withdrawing removes data that must not be published, such as personal data published by mistake, from every version that holds it. The writer first makes a new version without the data and moves each affected branch forward to it, and deletes the tags that point at affected versions. Then the data files that hold the data are deleted at once, with no grace period, and each affected version gets `expired.json` with `"reason": "withdrawn"` and loses its catalog. The steps are in [Data files](/spec/data-files#withdrawal).
|
|
108
|
+
|
|
109
|
+
A reader that reaches an expired or withdrawn version stops at step 3 of reading and reports the reason.
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: 書いてから読まれるまで
|
|
3
|
+
description: 1 つの変更を作業カタログから読み手まで追い、各段階のリクエストと、FDL の各部分が Git の何に当たるかを示します。
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
このページでは、1 つの変更を書き手のマシンから読み手まで追い、各段階で送るリクエストを示します。各段階でリポジトリに残るファイルは[リポジトリの中身](/ja/internals/inside-a-repository)に、決まりは[仕様](/ja/spec)にあります。
|
|
7
|
+
|
|
8
|
+
## 各部分と Git での呼び名
|
|
9
|
+
|
|
10
|
+
FDL は Git の役割と名前をそのまま使い、保存の仕方は独自に決めています。Git がコミットをどう持つかを知っていれば、この表でリポジトリを読めます。
|
|
11
|
+
|
|
12
|
+
| FDL | Git | 補足 |
|
|
13
|
+
| --- | --- | --- |
|
|
14
|
+
| リポジトリ(ストアの 1 つのプレフィックス) | サーバー上のベアリポジトリ | サーバーのプロセスは要りません |
|
|
15
|
+
| `fdl.json` | `config` | 形式、リポジトリの `id`、`data_dir` を持ちます |
|
|
16
|
+
| `versions/<ID>/version.json` | コミット | 親、作者、メッセージ、カタログの sha256 を持ちます |
|
|
17
|
+
| `versions/<ID>/ducklake.duckdb` | ツリー | どのテーブルがあり、それぞれがどのデータファイルでできているかを持ちます |
|
|
18
|
+
| データファイル(Parquet) | ブロブ | 一度書いたら変えません。1 つのファイルを多くのバージョンが指せます |
|
|
19
|
+
| `refs/heads/<name>` | ブランチ | 条件付き書き込みで、前にだけ進みます |
|
|
20
|
+
| `refs/tags/<name>` | タグ | 一度作ったら動かしません |
|
|
21
|
+
| `HEAD` | `HEAD` | 既定のブランチの名前を持ちます |
|
|
22
|
+
| `refs.json` | `info/refs` | 素の HTTP の読み手が ref を名前で探せるようにします |
|
|
23
|
+
| `versions/<ID>/files.json` | 到達可能性ビットマップ | カタログから作る索引です。読み手はこれが無くても読めます |
|
|
24
|
+
| `version.json.sig` | 署名付きコミット | `ssh-keygen -Y sign` で作る SSH 署名です |
|
|
25
|
+
| 作業カタログ(SQLite) | 作業ツリーとインデックス | バージョンから作り、次のバージョンにします |
|
|
26
|
+
| `POST <root>/?batch` | Git LFS の batch API | ストレージへ直接アップロードする URL を渡します |
|
|
27
|
+
| `expired.json` | なし | バージョンにカタログが無くなった理由を書きます |
|
|
28
|
+
|
|
29
|
+
次の 3 つは、意図して Git と変えています。
|
|
30
|
+
|
|
31
|
+
- バージョンの ID は UUIDv7 で、中身のハッシュではありません。DuckDB のリリースが違うと、同じカタログでも違うバイト列で書かれることがあり、中身のハッシュは道具によって変わってしまうためです。中身の確認には `version.json` の sha256 を使います
|
|
32
|
+
- packfile はありません。ファイルはそれぞれストアの 1 つのオブジェクトなので、読み手は Parquet ファイルのうち要る部分だけを取りに行けます
|
|
33
|
+
- マージはありません。バージョンの親は 1 つです。push を断られた書き手は、新しいバージョンの上で作り直します
|
|
34
|
+
|
|
35
|
+
## バージョンを書く
|
|
36
|
+
|
|
37
|
+
### checkout
|
|
38
|
+
|
|
39
|
+
`fdl checkout` は `HEAD` を読み、それが指す ref を読み、そのバージョンの `version.json` を読んでから、カタログを書き手のマシンの SQLite ファイルに写します。これが作業カタログです。バージョンに `expired.json` があれば、checkout は止まります。
|
|
40
|
+
|
|
41
|
+
作業カタログのデータパスは、ストアによって変わります。
|
|
42
|
+
|
|
43
|
+
- フォルダか S3: リポジトリのデータフォルダです。DuckDB は新しい Parquet ファイルをリポジトリへ直接書きます
|
|
44
|
+
- HTTP: 書き手のマシンのフォルダです。リポジトリに既にあるファイルは `data_dir` の下の URL で指し、HTTP で読みます
|
|
45
|
+
|
|
46
|
+
### 書く
|
|
47
|
+
|
|
48
|
+
書き手は作業カタログに対して、DuckDB で直接、または dbt などの道具を通して SQL を流します。文ごとに作業カタログに DuckLake のスナップショットができます。読み手から見えるものはまだ何も変わりません。新しい Parquet ファイルがリポジトリに書かれていても、どのバージョンもそれを指していないためです。
|
|
49
|
+
|
|
50
|
+
### commit
|
|
51
|
+
|
|
52
|
+
`fdl commit` は作業カタログを DuckDB ファイルに写し、その `data_path` をリポジトリのデータフォルダにし、そこから `dataset.json` を作り、新しい ID と、親として前のバージョンを持つ `version.json` を書きます。テーブルの行が親と同じなら、コミットしません。新しいバージョンは、push するまで書き手のマシンにしかありません。
|
|
53
|
+
|
|
54
|
+
### push
|
|
55
|
+
|
|
56
|
+
push は、読み手を守る順にファイルを書きます。あるファイルが指す先は、そのファイルより先に書きます。
|
|
57
|
+
|
|
58
|
+
1. リポジトリに無い新しいデータファイル
|
|
59
|
+
2. `versions/<ID>/ducklake.duckdb`
|
|
60
|
+
3. `versions/<ID>/dataset.json` と `files.json`
|
|
61
|
+
4. `versions/<ID>/version.json`
|
|
62
|
+
5. ref。`If-Match: <1 の前に読んだ ETag>` で書きます。最初のバージョンなら `If-None-Match: *` です
|
|
63
|
+
6. `refs.json`。`HEAD` とすべての ref を読み直して書き直します
|
|
64
|
+
|
|
65
|
+
読み手は ref から読み始め、ref は 5 で書きます。5 より前で止まった push は、どの ref も指さないファイルを残すだけで、読み手からは見えません。1 のあとに別の書き手が ref を動かしていれば、ETag が合わず、ストアは 5 を断ります。断られた書き手は checkout し直し、新しいバージョンの上で作り直します。
|
|
66
|
+
|
|
67
|
+
`refs.json` は ref のあとに書くので、少しの間は ref より古いことがあります。そのため、今のバージョンが要る読み手は ref 自体を読みます。
|
|
68
|
+
|
|
69
|
+
### HTTP での push
|
|
70
|
+
|
|
71
|
+
HTTP では、すべてのキーがリポジトリの URL の下のパスになり、ホストがそこへの `GET` や `PUT` などに答えます([HTTP での push](/ja/spec/http)を参照)。順番は上と同じで、2 つの段階が変わります。
|
|
72
|
+
|
|
73
|
+
- 1 の前に、書き手はこれから上げるファイルのキーと大きさを `POST <root>/?batch` で送ります。ホストはファイルごとに、アップロード用の URL を返すか、既にあると答えるか、ホスト経由で送らせるために 413 を返します。URL をもらったファイルは `PUT` でストレージへ直接送られ、データはホストを通りません。batch の無いホストは 404・405・501 のどれかを返し、すべてのファイルがホストを通ります
|
|
74
|
+
- 5 で、ホストは ref を動かす前に push を確かめます。バージョンがそろっていること、その親が今 ref の指すバージョンであること、カタログが `version.json` と合うこと、カタログが指すデータファイルがすべてリポジトリにあることです。確認に通らなければ、理由を付けて 409 を返します。ref を自分で動かすホストは `refs.json` も自分で書き、書き手が書こうとすると 403 を返します
|
|
75
|
+
|
|
76
|
+
ホストは、書き手に ref を一切動かさせないこともできます。その場合、書き手はバージョンを上げて 4 で止まり、ホストがあとで、たとえば誰かが変更を承認したときに ref を動かします。
|
|
77
|
+
|
|
78
|
+
## バージョンを読む
|
|
79
|
+
|
|
80
|
+
読み手に要るのは `GET` だけです。誰がリポジトリを書いたかも、前にホストがいるかどうかも知らなくて構いません。
|
|
81
|
+
|
|
82
|
+
1. `HEAD` か、求められた ref を読みます。`HEAD` は `ref: refs/heads/main` を持つので、次に `refs/heads/main` を読みます。ref はバージョンの ID を持ちます
|
|
83
|
+
2. `versions/<ID>/version.json` を読みます。`id` が求めた ID と同じで、`fdl` のメジャーバージョンを読み手が知っていることを確かめます
|
|
84
|
+
3. `versions/<ID>/expired.json` があるか確かめます。あれば止まります。そのバージョンは保持期間を過ぎたか取り下げられていて、カタログはもうありません
|
|
85
|
+
4. リポジトリのデータフォルダをデータパスにして、カタログを読み取り専用で ATTACH します
|
|
86
|
+
|
|
87
|
+
```sql
|
|
88
|
+
ATTACH 'ducklake:https://example.org/lake/versions/019a3c4e-…/ducklake.duckdb' AS lake (
|
|
89
|
+
DATA_PATH 'https://example.org/lake/data/', OVERRIDE_DATA_PATH true, READ_ONLY
|
|
90
|
+
);
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
ここから先は DuckDB がカタログを読み、クエリに要る Parquet ファイルの部分を取りに行きます。`fdl url` は 1 から 3 までを行い、4 で使うカタログの URL を出します。
|
|
94
|
+
|
|
95
|
+
読み手はカタログに書かれたデータパスを使わず、自分で渡します。カタログが持つのは書き手が公開した場所の URL で、ミラーは同じファイルを別の場所から配るためです。読み手がデータパスを渡せば、同じ文が元のホストでもどのミラーでも動きます。
|
|
96
|
+
|
|
97
|
+
バージョンのカタログは変わらないので、ずっとキャッシュできます。`HEAD`・`refs/`・`refs.json` は `Cache-Control: no-store` で配るので、読み手は ref が今どこを指しているかを必ず知ることができます。
|
|
98
|
+
|
|
99
|
+
### FDL を知らない読み手
|
|
100
|
+
|
|
101
|
+
ホストは、`HEAD` が指すバージョンのカタログを `<root>/ducklake.duckdb` で配ってもかまいません。DuckLake しか知らない読み手は、その 1 つの URL を ATTACH できます。ホストは `FDL-Version` ヘッダーでバージョンの ID を送るので、読み手はどのバージョンを受け取ったかを知ることができます。ストアのそのキーにはファイルがありません。ホストが `versions/` のカタログを配ります。
|
|
102
|
+
|
|
103
|
+
## バージョンを消す
|
|
104
|
+
|
|
105
|
+
`fdl expire` は古いバージョンを保持期間から外します。先に `expired.json` を置き、次にカタログを消し、`version.json` は残します。そのため `fdl log` はそのバージョンと履歴の中の位置を表示し続けます。どの生きているバージョンも指さないデータファイルは、猶予期間のあとで消します。書き手がそのファイルを指すバージョンをこれからコミットするかもしれないためです。
|
|
106
|
+
|
|
107
|
+
取り下げは、間違って公開した個人データなど、公開してはいけないデータを、それを持つすべてのバージョンから消します。書き手はまず、そのデータを除いた新しいバージョンを作り、影響を受けるブランチをそこへ前に進め、影響を受けるバージョンを指すタグを消します。次に、そのデータを持つデータファイルを猶予期間を置かずにすぐ消し、影響を受けるバージョンに `"reason": "withdrawn"` の `expired.json` を置いてカタログを消します。手順は[データファイル](/ja/spec/data-files#取り下げ)にあります。
|
|
108
|
+
|
|
109
|
+
保持期間を過ぎたか取り下げられたバージョンにたどり着いた読み手は、読む手順の 3 で止まり、理由を伝えます。
|
|
@@ -75,6 +75,12 @@ repo.is_ancestor(store, ancestor_id, descendant) -> bool
|
|
|
75
75
|
|
|
76
76
|
最初の親をたどる、ref が指すバージョンを読む、あるバージョンが別のバージョンより前にあるかを確かめる、の 3 つです。
|
|
77
77
|
|
|
78
|
+
```python
|
|
79
|
+
repo.readable(store, name="HEAD") -> Version
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
ref が指すバージョンを、読み手がカタログを開く前に行う確認を済ませてから返します。最初のバージョンの前、読めない形式の `version.json`、保持期間を過ぎたか取り下げたバージョンでは `RepoError` を出します。`fdl url` はこれを使います。
|
|
83
|
+
|
|
78
84
|
```python
|
|
79
85
|
repo.expire(store, version_id, *, reason="retention") -> None
|
|
80
86
|
repo.prune(store, newest, *, keep=1, since=None, reason="retention") -> list[str]
|
|
@@ -75,7 +75,7 @@ Versions from the one the ref holds back to the first, by first parent. Each sho
|
|
|
75
75
|
fdl url <remote> [--ref HEAD]
|
|
76
76
|
```
|
|
77
77
|
|
|
78
|
-
Prints the URL of the catalog of the version the ref holds, to `ATTACH` read-only.
|
|
78
|
+
Prints the URL of the catalog of the version the ref holds, to `ATTACH` read-only. A version past retention or withdrawn is refused, even where its catalog is still served.
|
|
79
79
|
|
|
80
80
|
## fdl tag
|
|
81
81
|
|
|
@@ -75,6 +75,12 @@ repo.is_ancestor(store, ancestor_id, descendant) -> bool
|
|
|
75
75
|
|
|
76
76
|
Walk the first-parent line, read the version a ref holds, and check whether a version is behind another.
|
|
77
77
|
|
|
78
|
+
```python
|
|
79
|
+
repo.readable(store, name="HEAD") -> Version
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
The version a ref holds, checked the way a reader must before opening its catalog. Raises `RepoError` before the first version, for a `version.json` of a format this does not read, and for a version past retention or withdrawn. `fdl url` uses it.
|
|
83
|
+
|
|
78
84
|
```python
|
|
79
85
|
repo.expire(store, version_id, *, reason="retention") -> None
|
|
80
86
|
repo.prune(store, newest, *, keep=1, since=None, reason="retention") -> list[str]
|
|
@@ -659,9 +659,13 @@ def last_snapshot(working: Path) -> int:
|
|
|
659
659
|
con.close()
|
|
660
660
|
|
|
661
661
|
|
|
662
|
-
def same_contents(working: Path, since: int) -> bool:
|
|
662
|
+
def same_contents(working: Path, since: int, *, data_path: str | None = None) -> bool:
|
|
663
663
|
"""Whether the working catalog's last snapshot holds what snapshot ``since`` held.
|
|
664
664
|
|
|
665
|
+
``data_path`` is where the working catalog's new data files are, when not
|
|
666
|
+
where it records (a working catalog of a repository over HTTP writes them
|
|
667
|
+
to a folder on this machine).
|
|
668
|
+
|
|
665
669
|
The same tables and views, each table with the same columns and the same
|
|
666
670
|
rows, in any order. A table rebuilt from the same input is written to new
|
|
667
671
|
files, so files are compared first and rows only where the files differ.
|
|
@@ -677,7 +681,10 @@ def same_contents(working: Path, since: int) -> bool:
|
|
|
677
681
|
con = duckdb.connect()
|
|
678
682
|
try:
|
|
679
683
|
for statement in attach_sql(
|
|
680
|
-
working,
|
|
684
|
+
working,
|
|
685
|
+
alias="lake",
|
|
686
|
+
data_path=data_path or _working_data_path(working),
|
|
687
|
+
read_only=True,
|
|
681
688
|
):
|
|
682
689
|
if statement:
|
|
683
690
|
con.execute(statement)
|
|
@@ -127,6 +127,7 @@ def commit(
|
|
|
127
127
|
message=message,
|
|
128
128
|
dataset=declared,
|
|
129
129
|
allow_empty=allow_empty,
|
|
130
|
+
working_data=Path(state["data"]).resolve().as_posix() + "/" if state.get("data") else None,
|
|
130
131
|
)
|
|
131
132
|
|
|
132
133
|
try:
|
|
@@ -167,11 +168,9 @@ def url(remote: str = Remote, ref: str = typer.Option(layout.HEAD)) -> None:
|
|
|
167
168
|
"""Where the catalog of the version a ref holds is, to ATTACH."""
|
|
168
169
|
store = open_store(remote)
|
|
169
170
|
try:
|
|
170
|
-
version = repo.
|
|
171
|
+
version = repo.readable(store, ref)
|
|
171
172
|
except ERRORS as exc:
|
|
172
173
|
raise _fail(exc) from exc
|
|
173
|
-
if version is None:
|
|
174
|
-
raise _fail(repo.RepoError(f"{ref} holds no version yet"))
|
|
175
174
|
typer.echo(store.url.rstrip("/") + "/" + layout.version_catalog(version.id))
|
|
176
175
|
|
|
177
176
|
|
|
@@ -21,7 +21,7 @@ from typing import Iterable, Iterator
|
|
|
21
21
|
from . import catalog, layout, refs
|
|
22
22
|
from . import dataset as dataset_json
|
|
23
23
|
from .store import Store, is_conflict, is_missing
|
|
24
|
-
from .version import Catalog, DuckLake, File, Version, new_id, parse, utc_now
|
|
24
|
+
from .version import Catalog, DuckLake, File, Version, VersionError, new_id, parse, utc_now
|
|
25
25
|
|
|
26
26
|
|
|
27
27
|
class RepoError(RuntimeError):
|
|
@@ -109,6 +109,23 @@ def head(store: Store, name: str = layout.HEAD) -> Version | None:
|
|
|
109
109
|
return read_version(store, version_id) if version_id else None
|
|
110
110
|
|
|
111
111
|
|
|
112
|
+
def readable(store: Store, name: str = layout.HEAD) -> Version:
|
|
113
|
+
"""The version a ref holds, checked the way a reader must (Reading, steps 1-3).
|
|
114
|
+
|
|
115
|
+
Raises `RepoError` before the first version, for a record of a format this
|
|
116
|
+
does not read, and for a version past retention or withdrawn.
|
|
117
|
+
"""
|
|
118
|
+
try:
|
|
119
|
+
version = head(store, name)
|
|
120
|
+
except VersionError as exc:
|
|
121
|
+
raise RepoError(str(exc)) from exc
|
|
122
|
+
if version is None:
|
|
123
|
+
raise RepoError(f"{name} holds no version yet")
|
|
124
|
+
if is_expired(store, version.id):
|
|
125
|
+
raise RepoError(f"{version.id} is past retention or withdrawn and has no catalog")
|
|
126
|
+
return version
|
|
127
|
+
|
|
128
|
+
|
|
112
129
|
def commit(
|
|
113
130
|
working: Path,
|
|
114
131
|
out_dir: Path,
|
|
@@ -121,6 +138,7 @@ def commit(
|
|
|
121
138
|
annotations: dict | None = None,
|
|
122
139
|
dataset: dict | None = None,
|
|
123
140
|
allow_empty: bool = True,
|
|
141
|
+
working_data: str | None = None,
|
|
124
142
|
) -> Version:
|
|
125
143
|
"""Freeze the working catalog into a version under ``out_dir/<ID>/``.
|
|
126
144
|
|
|
@@ -136,6 +154,9 @@ def commit(
|
|
|
136
154
|
data_dir), written into the frozen catalog. Every file reference must be
|
|
137
155
|
relative to it, or the version would point readers at the writer's disk.
|
|
138
156
|
Nothing is uploaded: that is `push`.
|
|
157
|
+
|
|
158
|
+
``working_data`` is where the working catalog wrote its new data files,
|
|
159
|
+
when not where it records (see `checkout`): comparing reads them there.
|
|
139
160
|
"""
|
|
140
161
|
Path(out_dir).mkdir(parents=True, exist_ok=True)
|
|
141
162
|
frozen = Path(out_dir) / f".freezing-{new_id()}.duckdb"
|
|
@@ -154,7 +175,9 @@ def commit(
|
|
|
154
175
|
# the contents can read every row of a rebuilt table.
|
|
155
176
|
if not allow_empty and parent is not None and _same_description(frozen, dataset, parent):
|
|
156
177
|
try:
|
|
157
|
-
unchanged = catalog.same_contents(
|
|
178
|
+
unchanged = catalog.same_contents(
|
|
179
|
+
working, parent.ducklake.snapshot_id, data_path=working_data
|
|
180
|
+
)
|
|
158
181
|
except catalog.CatalogError as exc:
|
|
159
182
|
raise CannotCompare(str(exc)) from exc
|
|
160
183
|
if unchanged:
|
|
@@ -157,3 +157,26 @@ def test_a_file_the_host_will_not_sign_goes_through_the_host(tmp_path: Path) ->
|
|
|
157
157
|
assert local.get("data/small.parquet") == b"ab" and local.get("data/large.parquet") == b"abcdef"
|
|
158
158
|
puts = [key for method, key in server.requests if method == "PUT"]
|
|
159
159
|
assert puts == ["storage/data/small.parquet", "data/large.parquet"]
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def test_a_rebuild_of_the_same_rows_over_http_is_nothing_to_commit(served, tmp_path: Path) -> None:
|
|
163
|
+
local, url, _ = served
|
|
164
|
+
working = tmp_path / "work" / "lake.sqlite"
|
|
165
|
+
run("checkout", url, working)
|
|
166
|
+
write(working, "CREATE SCHEMA lake.s; CREATE TABLE lake.s.towns AS SELECT 1 AS id")
|
|
167
|
+
run("commit", working, "-m", "first", "--author", "tester")
|
|
168
|
+
# The way dbt rebuilds a table: a new table beside it, then renames.
|
|
169
|
+
again = tmp_path / "again" / "lake.sqlite"
|
|
170
|
+
run("checkout", url, again)
|
|
171
|
+
write(
|
|
172
|
+
again,
|
|
173
|
+
"CREATE TABLE lake.s.towns__dbt_tmp AS SELECT 1 AS id; "
|
|
174
|
+
"ALTER TABLE lake.s.towns RENAME TO towns__dbt_backup; "
|
|
175
|
+
"ALTER TABLE lake.s.towns__dbt_tmp RENAME TO towns; DROP TABLE lake.s.towns__dbt_backup",
|
|
176
|
+
)
|
|
177
|
+
working = again
|
|
178
|
+
result = runner.invoke(app, ["commit", str(working), "-m", "again", "--author", "tester"])
|
|
179
|
+
assert result.exit_code == 0, result.output
|
|
180
|
+
assert "Nothing to commit" in result.output
|
|
181
|
+
assert "could not tell" not in result.output
|
|
182
|
+
assert len(list(repo.log(local))) == 1
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|