fdl-cli 0.24.2__tar.gz → 0.25.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/CHANGELOG.md +86 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/PKG-INFO +1 -1
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/getting-started.md +3 -1
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/getting-started.md +3 -1
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/reference/cli.md +11 -3
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/spec/refs.md +2 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/spec/versions.md +3 -2
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/spec/writing.md +1 -1
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/reference/cli.md +11 -3
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/spec/refs.md +2 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/spec/versions.md +3 -2
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/spec/writing.md +1 -1
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/pyproject.toml +1 -1
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/src/fdl/cli.py +88 -14
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/src/fdl/repo.py +67 -2
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/tests/test_profiles.py +64 -1
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/tests/test_repo.py +41 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/.github/workflows/docs.yml +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/.github/workflows/publish.yml +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/.github/workflows/release.yml +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/.github/workflows/test.yml +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/.gitignore +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/.node-version +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/.python-version +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/CLAUDE.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/CONTRIBUTING.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/LICENSE +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/README.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/SECURITY.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/blume.config.ts +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/cases/collection.json +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/concepts/meta.ts +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/concepts/repositories.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/concepts/versions-and-refs.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/concepts/working-catalogs.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/guides/building-a-host.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/guides/meta.ts +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/guides/mirrors.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/guides/public-and-private.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/guides/reading-without-duckdb.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/guides/serving-over-http.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/guides/stores.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/index.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/internals/from-write-to-read.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/internals/inside-a-repository.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/internals/meta.ts +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/concepts/meta.ts +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/concepts/repositories.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/concepts/versions-and-refs.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/concepts/working-catalogs.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/guides/building-a-host.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/guides/meta.ts +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/guides/mirrors.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/guides/public-and-private.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/guides/reading-without-duckdb.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/guides/serving-over-http.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/guides/stores.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/index.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/internals/from-write-to-read.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/internals/inside-a-repository.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/internals/meta.ts +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/reference/meta.ts +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/reference/python.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/spec/collection-cases.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/spec/data-files.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/spec/http.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/spec/index.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/spec/layout.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/spec/meta.ts +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/spec/reading.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/ja/stability.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/reference/meta.ts +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/reference/python.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/spec/collection-cases.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/spec/data-files.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/spec/http.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/spec/index.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/spec/layout.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/spec/meta.ts +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/spec/reading.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/docs/stability.md +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/package-lock.json +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/package.json +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/public/CNAME +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/src/fdl/__init__.py +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/src/fdl/catalog.py +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/src/fdl/dataset.py +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/src/fdl/diff.py +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/src/fdl/layout.py +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/src/fdl/lineage.py +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/src/fdl/merge.py +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/src/fdl/objects.py +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/src/fdl/profiles.py +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/src/fdl/py.typed +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/src/fdl/refs.py +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/src/fdl/store.py +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/src/fdl/stores.py +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/src/fdl/version.py +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/tests/http_repo.py +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/tests/test_cli.py +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/tests/test_collection_cases.py +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/tests/test_dataset.py +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/tests/test_http.py +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/tests/test_merge.py +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/tests/test_objects.py +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/tests/test_stores.py +0 -0
- {fdl_cli-0.24.2 → fdl_cli-0.25.0}/uv.lock +0 -0
|
@@ -1,6 +1,92 @@
|
|
|
1
1
|
# CHANGELOG
|
|
2
2
|
|
|
3
3
|
|
|
4
|
+
## v0.25.0 (2026-11-14)
|
|
5
|
+
|
|
6
|
+
### Bug Fixes
|
|
7
|
+
|
|
8
|
+
- Keep fdl push working when profiling cannot read the working catalog
|
|
9
|
+
([#59](https://github.com/queria-io/fdl/pull/59),
|
|
10
|
+
[`59bb12c`](https://github.com/queria-io/fdl/commit/59bb12cfe2308a2c71d6471f8d5e1e40e8b48393))
|
|
11
|
+
|
|
12
|
+
* feat!: profile tables with fdl profile instead of at commit
|
|
13
|
+
|
|
14
|
+
fdl commit no longer writes profiles. fdl profile <working> profiles the version committed there,
|
|
15
|
+
for fdl push to upload with it; fdl profile <remote> adds the profiles a pushed version lacks. The
|
|
16
|
+
spec lets a version get a missing profile later.
|
|
17
|
+
|
|
18
|
+
BREAKING CHANGE: fdl commit writes no profiles. Run fdl profile between fdl commit and fdl push to
|
|
19
|
+
keep writing them.
|
|
20
|
+
|
|
21
|
+
* feat: profile tables at push instead of commit
|
|
22
|
+
|
|
23
|
+
fdl push profiles the version it uploads (--no-profile skips it), so the steps stay commit and push.
|
|
24
|
+
fdl profile <remote> is left for versions pushed without profiles. Also warns of tables that could
|
|
25
|
+
not be read, lists the version's profiles in one request, and updates the spec wording.
|
|
26
|
+
|
|
27
|
+
* fix: keep fdl push working when profiling cannot read the working catalog
|
|
28
|
+
|
|
29
|
+
Read dataset.json and the working catalog inside the guarded block, so a missing or unreadable
|
|
30
|
+
catalog only warns. Profile again when a stopped push left some tables without profiles.
|
|
31
|
+
|
|
32
|
+
### Features
|
|
33
|
+
|
|
34
|
+
- Profile tables at push instead of commit ([#58](https://github.com/queria-io/fdl/pull/58),
|
|
35
|
+
[`95803d8`](https://github.com/queria-io/fdl/commit/95803d81860ba72a8887b1494a9a5b8bc4aff674))
|
|
36
|
+
|
|
37
|
+
* feat!: profile tables with fdl profile instead of at commit
|
|
38
|
+
|
|
39
|
+
fdl commit no longer writes profiles. fdl profile <working> profiles the version committed there,
|
|
40
|
+
for fdl push to upload with it; fdl profile <remote> adds the profiles a pushed version lacks. The
|
|
41
|
+
spec lets a version get a missing profile later.
|
|
42
|
+
|
|
43
|
+
BREAKING CHANGE: fdl commit writes no profiles. Run fdl profile between fdl commit and fdl push to
|
|
44
|
+
keep writing them.
|
|
45
|
+
|
|
46
|
+
* feat: profile tables at push instead of commit
|
|
47
|
+
|
|
48
|
+
fdl push profiles the version it uploads (--no-profile skips it), so the steps stay commit and push.
|
|
49
|
+
fdl profile <remote> is left for versions pushed without profiles. Also warns of tables that could
|
|
50
|
+
not be read, lists the version's profiles in one request, and updates the spec wording.
|
|
51
|
+
|
|
52
|
+
- Profile tables with fdl profile instead of at commit
|
|
53
|
+
([#57](https://github.com/queria-io/fdl/pull/57),
|
|
54
|
+
[`20f21ac`](https://github.com/queria-io/fdl/commit/20f21acca7ba6effc3af1f9c1c39522f982ce68f))
|
|
55
|
+
|
|
56
|
+
fdl commit no longer writes profiles. fdl profile <working> profiles the version committed there,
|
|
57
|
+
for fdl push to upload with it; fdl profile <remote> adds the profiles a pushed version lacks. The
|
|
58
|
+
spec lets a version get a missing profile later.
|
|
59
|
+
|
|
60
|
+
BREAKING CHANGE: fdl commit writes no profiles. Run fdl profile between fdl commit and fdl push to
|
|
61
|
+
keep writing them.
|
|
62
|
+
|
|
63
|
+
### Breaking Changes
|
|
64
|
+
|
|
65
|
+
- Fdl commit writes no profiles. Run fdl profile between fdl commit and fdl push to keep writing
|
|
66
|
+
them.
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
## v0.24.3 (2026-11-13)
|
|
70
|
+
|
|
71
|
+
### Bug Fixes
|
|
72
|
+
|
|
73
|
+
- Write the ref again when a push finds it already at the version
|
|
74
|
+
([#56](https://github.com/queria-io/fdl/pull/56),
|
|
75
|
+
[`a11abaf`](https://github.com/queria-io/fdl/commit/a11abaf3643d914167ee4ef41421daa68b769955))
|
|
76
|
+
|
|
77
|
+
* fix: write the ref again when a push finds it already at the version
|
|
78
|
+
|
|
79
|
+
A host that does its own work when a ref moves may stop after moving it. Pushing again returned
|
|
80
|
+
early because the ref already held the version, so the host never finished. The writer now writes
|
|
81
|
+
the ref again, to the same version and on condition of the ETag it read.
|
|
82
|
+
|
|
83
|
+
* fix: write the ref again after a retried write and in forward
|
|
84
|
+
|
|
85
|
+
A ref write whose first attempt moved the ref, with its answer lost, may leave the host stopped
|
|
86
|
+
after the move. forward returned early on the same version as push did. Both now write the ref
|
|
87
|
+
again, and the test checks the write is conditional on the ETag read.
|
|
88
|
+
|
|
89
|
+
|
|
4
90
|
## v0.24.2 (2026-11-13)
|
|
5
91
|
|
|
6
92
|
### Bug Fixes
|
|
@@ -83,7 +83,7 @@ fdl commit work/lake.sqlite -m "First towns" --author alice
|
|
|
83
83
|
fdl push work/lake.sqlite
|
|
84
84
|
```
|
|
85
85
|
|
|
86
|
-
`commit` freezes the working catalog into a version on your disk. `push` uploads
|
|
86
|
+
`commit` freezes the working catalog into a version on your disk. `push` measures each table into a small summary (a profile) that readers can show without reading the data, uploads the version with them and moves `main` to it, and prints `refs/heads/main -> 019a3c4e-… (version 1)`. The repository now holds:
|
|
87
87
|
|
|
88
88
|
```
|
|
89
89
|
towns/
|
|
@@ -93,6 +93,8 @@ towns/
|
|
|
93
93
|
versions/019a3c4e-…/
|
|
94
94
|
ducklake.duckdb
|
|
95
95
|
dataset.json
|
|
96
|
+
files.json
|
|
97
|
+
profiles/main/towns.json
|
|
96
98
|
version.json
|
|
97
99
|
data/main/towns/ducklake-….parquet
|
|
98
100
|
```
|
|
@@ -83,7 +83,7 @@ fdl commit work/lake.sqlite -m "First towns" --author alice
|
|
|
83
83
|
fdl push work/lake.sqlite
|
|
84
84
|
```
|
|
85
85
|
|
|
86
|
-
`commit` は作業カタログを固めて手元でバージョンにします。`push`
|
|
86
|
+
`commit` は作業カタログを固めて手元でバージョンにします。`push` はテーブルごとに、データを読まずに表示できる小さな要約(profile)を測り、バージョンと一緒にアップロードして `main` をそこへ動かし、`refs/heads/main -> 019a3c4e-… (version 1)` と表示します。リポジトリは次のようになります。
|
|
87
87
|
|
|
88
88
|
```
|
|
89
89
|
towns/
|
|
@@ -93,6 +93,8 @@ towns/
|
|
|
93
93
|
versions/019a3c4e-…/
|
|
94
94
|
ducklake.duckdb
|
|
95
95
|
dataset.json
|
|
96
|
+
files.json
|
|
97
|
+
profiles/main/towns.json
|
|
96
98
|
version.json
|
|
97
99
|
data/main/towns/ducklake-….parquet
|
|
98
100
|
```
|
|
@@ -57,8 +57,6 @@ fdl commit <working.sqlite> -m "What changed" --author alice [--dataset dataset.
|
|
|
57
57
|
|
|
58
58
|
コマンドは、説明の隣の `README.md` と、そこにある `docs/` より下のファイル(隠しファイルを除く)をバージョンの tree に入れます。そのフォルダからのパスで `tree.json` を書き、push がリポジトリに無いものを書いたままの中身で `assets/<sha256>` にアップロードします([tree.json](/ja/spec/versions#treejson))。
|
|
59
59
|
|
|
60
|
-
あわせてテーブルごとの profile を書きます([profiles/](/ja/spec/versions#profiles))。読むのは、データファイルか列が親から変わったテーブルだけです。読めなかったテーブルには profile を書かず、警告を出してコミットを続けます。
|
|
61
|
-
|
|
62
60
|
`asset: true` の印を付けた列は、ファイルの sha256 を持ちます([アセット](/ja/spec/versions#アセット))。コマンドはその列の値を読み、push がリポジトリに無いファイルを `--assets` のフォルダから `assets/<sha256>` にアップロードします。ファイルは中身で照合するので、フォルダの中のファイル名は問いません。
|
|
63
61
|
|
|
64
62
|
親と同じ中身になるとき([変更がないとき](/ja/spec/writing#変更がないとき))は `Nothing to commit` と表示し、何も作らずに終了コード 0 で終わります。同じ行を新しいファイルに書き直しただけの作り直しも、変更なしとみなします。`--allow-empty` を付けると、それでもバージョンを作ります。行を読めず判定できないときは、警告を出してバージョンを作ります。
|
|
@@ -73,11 +71,13 @@ fdl commit <working.sqlite> -m "What changed" --author alice [--dataset dataset.
|
|
|
73
71
|
## fdl push
|
|
74
72
|
|
|
75
73
|
```bash
|
|
76
|
-
fdl push <working.sqlite>
|
|
74
|
+
fdl push <working.sqlite> [--no-profile]
|
|
77
75
|
```
|
|
78
76
|
|
|
79
77
|
`fdl commit` が作ったバージョンをアップロードし、条件付きでブランチをそこへ動かします([書き込みの順番](/ja/spec/writing#書き込みの順番))。ブランチ、バージョン ID、`seq` を表示します。fast-forward の `fdl merge` のあとは、何もアップロードせず、ブランチをマージしたバージョンへ動かします。コミットしたものが無いときは `Nothing to push` と表示します。
|
|
80
78
|
|
|
79
|
+
アップロードの前に、テーブルごとの profile([profiles/](/ja/spec/versions#profiles))を書き、`version.json` より前にアップロードします。ブランチが動く時点で profile がリポジトリにあります。読むのは、データファイルか列が親から変わったテーブルだけで、ほかのテーブルは親の profile を引き継ぎます。読めなかったテーブルには profile を書かず、警告を出します。コミットのあとに作業カタログが変わっていたときは profile を書かずに警告します。`--no-profile` を付けると profile を書きません。`fdl profile` であとから足せます。
|
|
80
|
+
|
|
81
81
|
作業カタログがリポジトリに書いたデータファイルは push の前からありますが、push するまでどのバージョンも参照しません。push しなかったファイルは猶予期間のあとに削除されます。
|
|
82
82
|
|
|
83
83
|
次のときは断られます。
|
|
@@ -220,6 +220,14 @@ fdl withdraw <remote> <version>... [--yes]
|
|
|
220
220
|
|
|
221
221
|
`--yes` を付けないと、消すものを表示するだけです。残すバージョンが参照しているファイルは残すものとして表示します。それがそのデータを持つなら、そのバージョンも取り下げます。ブランチかリクエストが該当するバージョンを指しているあいだは断られます。
|
|
222
222
|
|
|
223
|
+
## fdl profile
|
|
224
|
+
|
|
225
|
+
```bash
|
|
226
|
+
fdl profile <remote> [--ref refs/heads/main]
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
ブランチかタグが指すバージョンのテーブルのうち、[profile](/ja/spec/versions#profiles) が無いものを測り、profile をバージョンの横に置きます。`fdl push` はアップロードするバージョンの profile を作ります。このコマンドは profile 無しで push したバージョンに使います。fdl が profile を書く前に作ったバージョン、`--no-profile` で push したバージョン、そのときテーブルを読めなかったバージョンです。既にある profile は変えません。保持期間を過ぎたバージョンはカタログが無いので断られます。
|
|
230
|
+
|
|
223
231
|
## fdl expire
|
|
224
232
|
|
|
225
233
|
```bash
|
|
@@ -57,3 +57,5 @@ HEAD ref: refs/heads/main\n
|
|
|
57
57
|
## 送り直した書き込み
|
|
58
58
|
|
|
59
59
|
最初の返事を受け取れずに ref の書き込みを送り直した書き手は、自分の最初の書き込みのせいで条件が合わず断られることがあります。そのとき ref が、動かそうとしていたバージョンを指していれば、動かすのは成功しています。
|
|
60
|
+
|
|
61
|
+
動かそうとしているバージョンを ref が既に指していたら(書き込む前でも、送り直した書き込みが断られた後でも)、書き手は ref を同じバージョンで書き直します。条件には読んだときの ETag を付けます。ref を動かすときに自分の処理(新しいバージョンの記録など)をするホストは、ref を動かした後で止まっていることがあり、この書き込みで処理を終えられます。ホストは、ref を動かさない書き込みも受け付けます。読んだ後に ref が動いていたら、書き手はそのままにします。
|
|
@@ -9,6 +9,7 @@ description: バージョン ID、version.json の項目、固めたカタログ
|
|
|
9
9
|
- 取り下げのとき、`"reason": "withdrawn"` の `expired.json` を足してカタログを消し、`version.json` 自体に取り下げるデータがあればそれも消す([データファイル](/ja/spec/data-files#取り下げ))
|
|
10
10
|
- `catalogs/<format>.duckdb` はあとから足してよい。既にあるファイルは変えない
|
|
11
11
|
- `version.json.sig` は、バージョンを作ったあとに署名する書き手が足してよい
|
|
12
|
+
- `profiles/<schema>/<table>.json` は、profile の無いテーブルについてあとから足してよい。既にある profile は変えない
|
|
12
13
|
|
|
13
14
|
## バージョン ID
|
|
14
15
|
|
|
@@ -207,7 +208,7 @@ DuckLake を使わずに Parquet ファイルを読む読み手は、`deletes`
|
|
|
207
208
|
|
|
208
209
|
profile は、`ducklake.snapshot_id` の時点の 1 つのテーブルの要約です。行数、列ごとの中身、先頭の行、図形の列を持つテーブルでは図形の範囲と見本を持ちます。読み手はこの小さなファイル 1 つで、カタログを開かずデータも読まずにテーブルを表示できます。ホストは検索の索引をこれから作れます。profile が持つのはデータから測った事実で、列が何を意味するかは書き手が `dataset.json` に書きます。`files.json` と同じくカタログとデータから作るもので、食い違うときはカタログとデータが正です。任意で、読み手は profile が無くても読めなければなりません。
|
|
209
210
|
|
|
210
|
-
テーブルごとの profile は `versions/<ID>/profiles/<schema>/<table>.json` に置きます。スキーマ名かテーブル名が空・`.`・`..` のとき、または `/` を含むときは置きません。ビューには置きません。
|
|
211
|
+
テーブルごとの profile は `versions/<ID>/profiles/<schema>/<table>.json` に置きます。スキーマ名かテーブル名が空・`.`・`..` のとき、または `/` を含むときは置きません。ビューには置きません。profile を作らなかった書き手や作るのに失敗した書き手のバージョンには、あとから profile を足せます。ほかの profile と同じく、そのバージョンのカタログとデータから測ります。
|
|
211
212
|
|
|
212
213
|
```json profiles/main/towns.json
|
|
213
214
|
{
|
|
@@ -245,7 +246,7 @@ profile は、`ducklake.snapshot_id` の時点の 1 つのテーブルの要約
|
|
|
245
246
|
- `head` は、テーブルが返す先頭の行を JSON のオブジェクトで持つ。図形の列と、値を JSON で表せない列は入れない
|
|
246
247
|
- `geometry` は、書き手が読めたときに、最初の図形の列を説明する。図形を持つ行の数、全体の `bbox`(列の座標で `[xmin, ymin, xmax, ymax]`)、含まれる図形の種類、地図を描くための、無作為に選んで簡略化した図形の GeoJSON `FeatureCollection` を持つ
|
|
247
248
|
- 書き手は計算できないキーを省いてよい。読み手は知らないキーを無視する
|
|
248
|
-
- 書き手は、データファイル(`files.json`)と列が親と同じテーブルには親の profile
|
|
249
|
+
- 書き手は、データファイル(`files.json`)と列が親と同じテーブルには親の profile を使い回す。profile を作るときに読むのは変わったテーブルだけになる。親の profile の `values` が書き手と異なる上限で書かれているときは、profile を作り直す
|
|
249
250
|
- profile はバージョンの一部で、そのバージョンの中では変わらない。保持期間から外すときはカタログと一緒に消す
|
|
250
251
|
|
|
251
252
|
## 固めたカタログ
|
|
@@ -17,7 +17,7 @@ description: チェックアウト、コミット、push。作業カタログ、
|
|
|
17
17
|
| --- | --- |
|
|
18
18
|
| チェックアウト | ブランチのバージョンのカタログから作業カタログを作り、そのバージョンを覚える。最初のバージョンの前は空のものを作る |
|
|
19
19
|
| コミット | 作業カタログの最後のスナップショットを新しいバージョンに固める。親は覚えておいたバージョン。中身を `dataset.json` で説明する |
|
|
20
|
-
| push |
|
|
20
|
+
| push | データファイルを確かめ、テーブルごとに profile を測り(任意)、カタログ、`dataset.json`・`files.json`・`tree.json`・profile、`version.json` の順にアップロードし、条件付きで ref を動かす |
|
|
21
21
|
|
|
22
22
|
書き手は、Git が `git push` までコミットを手元に置くのと同じく、コミットしたバージョンを手元に置いてあとで push してかまいません。push までリポジトリは変わりません。例外は作業カタログがリポジトリに書いたデータファイルで、どのバージョンもまだ参照していないので、push しないままなら猶予期間のあとに削除されます。
|
|
23
23
|
|
|
@@ -57,8 +57,6 @@ Every version is described by a `dataset.json` (see [Versions](/spec/versions#da
|
|
|
57
57
|
|
|
58
58
|
The command puts `README.md` beside the description and every file below `docs/` there (hidden ones left out) in the version's tree: it writes `tree.json` with their paths from that folder and the push uploads those the repository lacks to `assets/<sha256>`, as written (see [tree.json](/spec/versions#treejson)).
|
|
59
59
|
|
|
60
|
-
It also writes a profile of each table (see [profiles/](/spec/versions#profiles)), reading only the tables whose data files or columns changed since the parent. A table it cannot read gets none, and the commit goes on with a warning.
|
|
61
|
-
|
|
62
60
|
Columns marked `asset: true` hold the sha256 of files (see [Assets](/spec/versions#assets)). The command reads the values those columns hold, and the push uploads the files the repository lacks from the `--assets` folder to `assets/<sha256>`, by their content: file names in the folder do not matter.
|
|
63
61
|
|
|
64
62
|
When the version would hold what its parent holds (see [When nothing changed](/spec/writing#when-nothing-changed)), it prints `Nothing to commit`, makes nothing and exits 0. A rebuild that writes the same rows to new files counts as unchanged. `--allow-empty` makes the version anyway. If the rows cannot be read to tell, it warns and makes the version.
|
|
@@ -72,11 +70,13 @@ Refused when:
|
|
|
72
70
|
## fdl push
|
|
73
71
|
|
|
74
72
|
```bash
|
|
75
|
-
fdl push <working.sqlite>
|
|
73
|
+
fdl push <working.sqlite> [--no-profile]
|
|
76
74
|
```
|
|
77
75
|
|
|
78
76
|
Uploads the version `fdl commit` made and moves the branch to it on condition (see [Order of writes](/spec/writing#order-of-writes)). It prints the branch, the version ID and its `seq`. After a fast-forward `fdl merge`, it moves the branch to the version merged, uploading nothing. With nothing committed, it prints `Nothing to push`.
|
|
79
77
|
|
|
78
|
+
Before uploading, it writes a profile of each table (see [profiles/](/spec/versions#profiles)) and uploads the profiles before `version.json`, so they are there when the branch moves. It reads only the tables whose data files or columns changed since the parent; the others keep the parent's profile. A table it cannot read gets none, with a warning. When the working catalog changed after the commit, it profiles nothing and warns. `--no-profile` skips profiling; `fdl profile` adds the profiles later.
|
|
79
|
+
|
|
80
80
|
Data files the working catalog wrote into the repository are there before the push, but no version names them until it. Files never pushed are deleted after the grace period.
|
|
81
81
|
|
|
82
82
|
Refused when:
|
|
@@ -220,6 +220,14 @@ A version reaches the assets its `tree.json` lists and those its asset columns h
|
|
|
220
220
|
|
|
221
221
|
Without `--yes` it only prints what it would delete. Files a kept version still reaches are listed as kept; if they hold the data, withdraw that version too. Refused while a branch or a request points at one of the versions.
|
|
222
222
|
|
|
223
|
+
## fdl profile
|
|
224
|
+
|
|
225
|
+
```bash
|
|
226
|
+
fdl profile <remote> [--ref refs/heads/main]
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
Profiles the tables of the version the branch or tag holds that have no [profile](/spec/versions#profiles), and puts the profiles beside the version. `fdl push` profiles the version it uploads; this is for a version pushed without them: made before fdl wrote profiles, pushed with `--no-profile`, or whose tables could not be read then. Profiles already there are left as they are. Refused for a version past retention, which has no catalog.
|
|
230
|
+
|
|
223
231
|
## fdl expire
|
|
224
232
|
|
|
225
233
|
```bash
|
|
@@ -57,3 +57,5 @@ A reader over plain HTTP cannot list the store, so it cannot find branches and t
|
|
|
57
57
|
## Retried writes
|
|
58
58
|
|
|
59
59
|
A writer that sends a ref write again after losing the answer to the first one may find the condition refused by its own first attempt. If the ref now holds the version it was moving to, the move succeeded.
|
|
60
|
+
|
|
61
|
+
A writer that finds the ref already holding the version it is moving the ref to, before its write or after a retried one, writes the ref again, to the same version and on condition of the ETag it read. A host that does its own work when a ref moves (recording the new version, say) may have stopped after moving the ref, and this write lets it finish. A host accepts a write that leaves a ref where it is. If the ref has moved since it was read, the writer leaves it.
|
|
@@ -9,6 +9,7 @@ A version is the files under `versions/<version ID>/`. They must not change once
|
|
|
9
9
|
- on withdrawal, `expired.json` is added with `"reason": "withdrawn"` and the catalogs deleted, and `version.json` itself is deleted if it holds the withdrawn data ([Data files](/spec/data-files#withdrawal))
|
|
10
10
|
- `catalogs/<format>.duckdb` may be added later. Files already there are not changed
|
|
11
11
|
- `version.json.sig` may be added later, by a writer that signs a version after making it
|
|
12
|
+
- `profiles/<schema>/<table>.json` may be added later for a table that has none. Profiles already there are not changed
|
|
12
13
|
|
|
13
14
|
## Version ID
|
|
14
15
|
|
|
@@ -207,7 +208,7 @@ When a version is taken out of retention, `files.json` is deleted with its catal
|
|
|
207
208
|
|
|
208
209
|
A profile summarizes one table at `ducklake.snapshot_id`: its row count, what each column holds, its first rows and, for a table with a geometry column, the extent and a sample of its shapes. A reader can show a table, or a host can index it for search, from this one small file, without opening the catalog or reading the data. It holds what was measured from the data; what a column means is the writer's, in `dataset.json`. Like `files.json`, it is made from the catalog and the data, which win where the two differ, and it is optional: a reader must work without it.
|
|
209
210
|
|
|
210
|
-
Each table's profile is at `versions/<ID>/profiles/<schema>/<table>.json`. A table whose schema or name is empty, `.`, `..` or holds `/` has none. Views have none.
|
|
211
|
+
Each table's profile is at `versions/<ID>/profiles/<schema>/<table>.json`. A table whose schema or name is empty, `.`, `..` or holds `/` has none. Views have none. A version made without a profile for a table, by a writer that did not profile or failed to, may get one later; it is measured from the version's catalog and data, like any other profile.
|
|
211
212
|
|
|
212
213
|
```json profiles/main/towns.json
|
|
213
214
|
{
|
|
@@ -245,7 +246,7 @@ Each table's profile is at `versions/<ID>/profiles/<schema>/<table>.json`. A tab
|
|
|
245
246
|
- `head` holds the first rows the table returns, as JSON objects, leaving out geometry and columns whose values have no JSON form
|
|
246
247
|
- `geometry` describes the first geometry column, when the writer could read it: how many rows have a shape, the `bbox` of all of them (`[xmin, ymin, xmax, ymax]` in the column's coordinates), the geometry types present, and a GeoJSON `FeatureCollection` of a random sample of simplified shapes for drawing a map
|
|
247
248
|
- A writer may leave out any key it cannot compute. A reader ignores keys it does not know
|
|
248
|
-
- A writer reuses its parent's profile for a table whose data files (in `files.json`) and columns are the parent's, so
|
|
249
|
+
- A writer reuses its parent's profile for a table whose data files (in `files.json`) and columns are the parent's, so profiling reads only the tables that changed. It makes the profile again when the parent's `values` follow another limit than its own
|
|
249
250
|
- Profiles are part of the version and do not change with it. They are deleted with the catalogs when the version is taken out of retention
|
|
250
251
|
|
|
251
252
|
## The frozen catalog
|
|
@@ -17,7 +17,7 @@ A writer writes to a working catalog: a DuckLake catalog in SQLite, outside the
|
|
|
17
17
|
| --- | --- |
|
|
18
18
|
| checkout | Make a working catalog from the catalog of the branch's version, and remember that version. Before the first version, make an empty one |
|
|
19
19
|
| commit | Freeze the working catalog's last snapshot into a new version, with the remembered version as its parent, and describe it in `dataset.json` |
|
|
20
|
-
| push | Check the data files, upload the catalog, `dataset.json`, `files.json`, `tree.json` and the profiles, then `version.json`, then move the ref on condition |
|
|
20
|
+
| push | Check the data files, measure each table into a profile (optional), upload the catalog, `dataset.json`, `files.json`, `tree.json` and the profiles, then `version.json`, then move the ref on condition |
|
|
21
21
|
|
|
22
22
|
A writer may keep a committed version on its own disk and push it later, as Git keeps commits until `git push`. Until the push, nothing in the repository changes, apart from data files the working catalog wrote there. No version names those yet, so they are deleted after the grace period if the version is never pushed.
|
|
23
23
|
|
|
@@ -20,7 +20,7 @@ from urllib.parse import urlparse
|
|
|
20
20
|
|
|
21
21
|
import typer
|
|
22
22
|
|
|
23
|
-
from . import catalog, dataset, layout, objects, refs, repo
|
|
23
|
+
from . import catalog, dataset, layout, objects, profiles, refs, repo
|
|
24
24
|
from . import merge as merging
|
|
25
25
|
from . import diff as diffs
|
|
26
26
|
from .stores import StoreError, open_store
|
|
@@ -230,16 +230,11 @@ def _commit(
|
|
|
230
230
|
except repo.CannotCompare as exc:
|
|
231
231
|
typer.echo(f"warning: could not tell whether anything changed ({exc}); committing", err=True)
|
|
232
232
|
version = make(True)
|
|
233
|
-
reading =
|
|
234
|
-
setup = _read_setup(store) + _storage_setup(state.get("storage"))
|
|
233
|
+
reading, setup = _reading(store, state)
|
|
235
234
|
described = json.loads((out / version.id / layout.DATASET).read_text(encoding="utf-8"))
|
|
236
235
|
held = objects.asset_values(
|
|
237
236
|
working, objects.asset_columns(described), data_path=reading, setup=setup
|
|
238
237
|
)
|
|
239
|
-
try:
|
|
240
|
-
repo.write_profiles(store, out / version.id, parent, working, data_path=reading, setup=setup)
|
|
241
|
-
except Exception as exc: # noqa: BLE001 - a profile is never worth a failed commit
|
|
242
|
-
typer.echo(f"warning: tables were not profiled ({exc})", err=True)
|
|
243
238
|
pending = _pending_dir(working)
|
|
244
239
|
shutil.rmtree(pending, ignore_errors=True)
|
|
245
240
|
out.rename(pending)
|
|
@@ -280,10 +275,24 @@ def _written(store, state: dict) -> str | None:
|
|
|
280
275
|
return None
|
|
281
276
|
|
|
282
277
|
|
|
278
|
+
def _reading(store, state: dict) -> tuple[str, list[str]]:
|
|
279
|
+
"""Where the working catalog's data files are read from, and the setup to read them."""
|
|
280
|
+
return _written(store, state) or repo.data_url(store), _read_setup(store) + _storage_setup(state.get("storage"))
|
|
281
|
+
|
|
282
|
+
|
|
283
283
|
@app.command()
|
|
284
|
-
def push(
|
|
284
|
+
def push(
|
|
285
|
+
working: Path = Working,
|
|
286
|
+
no_profile: bool = typer.Option(
|
|
287
|
+
False, "--no-profile", help="Upload the version without profiling its tables"
|
|
288
|
+
),
|
|
289
|
+
) -> None:
|
|
285
290
|
"""Upload the committed version and move the branch to it.
|
|
286
291
|
|
|
292
|
+
First it profiles the version's tables (a summary of each, measured from
|
|
293
|
+
the data), reading only those whose data files or columns changed since
|
|
294
|
+
the parent, and uploads the profiles with the version.
|
|
295
|
+
|
|
287
296
|
Refused when someone else moved the branch since the checkout: check out
|
|
288
297
|
again, redo the work, commit and push, or merge the branch into your work.
|
|
289
298
|
"""
|
|
@@ -299,6 +308,8 @@ def push(working: Path = Working) -> None:
|
|
|
299
308
|
folder = _pending_dir(working) / pending["id"]
|
|
300
309
|
try:
|
|
301
310
|
version = parse_version((folder / layout.VERSION_RECORD).read_bytes(), expected_id=pending["id"])
|
|
311
|
+
if not no_profile:
|
|
312
|
+
_profile_pending(store, working, state, folder, version)
|
|
302
313
|
assets_dir = pending.get("assets_dir")
|
|
303
314
|
repo.push(
|
|
304
315
|
store,
|
|
@@ -363,12 +374,8 @@ def revert(
|
|
|
363
374
|
if declared is None:
|
|
364
375
|
raise repo.RepoError(f"{target.id} has no dataset.json to bring back")
|
|
365
376
|
listed = (diffs.listed(store, target) or {}).get("entries", [])
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
target.ducklake.snapshot_id,
|
|
369
|
-
data_path=_written(store, state) or repo.data_url(store),
|
|
370
|
-
setup=_read_setup(store) + _storage_setup(state.get("storage")),
|
|
371
|
-
)
|
|
377
|
+
reading, setup = _reading(store, state)
|
|
378
|
+
catalog.restore(working, target.ducklake.snapshot_id, data_path=reading, setup=setup)
|
|
372
379
|
version = _commit(
|
|
373
380
|
store,
|
|
374
381
|
working,
|
|
@@ -799,6 +806,73 @@ def tag(
|
|
|
799
806
|
typer.echo(f"refs/tags/{name} -> {version_id}")
|
|
800
807
|
|
|
801
808
|
|
|
809
|
+
@app.command()
|
|
810
|
+
def profile(
|
|
811
|
+
remote: str = Remote,
|
|
812
|
+
ref: str = typer.Option(layout.DEFAULT_BRANCH, help="The branch or tag whose version to profile"),
|
|
813
|
+
) -> None:
|
|
814
|
+
"""Profile the tables of a pushed version that have no profile.
|
|
815
|
+
|
|
816
|
+
`fdl push` profiles the version it uploads. This is for a version pushed
|
|
817
|
+
without them: made before fdl wrote profiles, pushed with --no-profile,
|
|
818
|
+
or whose tables could not be read then. Profiles already there are left
|
|
819
|
+
as they are.
|
|
820
|
+
"""
|
|
821
|
+
store = _store(remote)
|
|
822
|
+
try:
|
|
823
|
+
version = repo.readable(store, ref)
|
|
824
|
+
put, unread = repo.add_profiles(store, version, setup=_read_setup(store))
|
|
825
|
+
except (*ERRORS, catalog.CatalogError) as exc:
|
|
826
|
+
raise _fail(exc) from exc
|
|
827
|
+
for relative in put:
|
|
828
|
+
typer.echo(f"put {relative}")
|
|
829
|
+
_warn_unread(unread)
|
|
830
|
+
typer.echo(f"Profiled {len(put)} table(s) of {version.id} (version {version.seq}).")
|
|
831
|
+
|
|
832
|
+
|
|
833
|
+
def _warn_unread(tables: list[str]) -> None:
|
|
834
|
+
if tables:
|
|
835
|
+
typer.echo(f"warning: tables were not profiled, as they could not be read: {', '.join(tables)}", err=True)
|
|
836
|
+
|
|
837
|
+
|
|
838
|
+
def _profile_pending(store, working: Path, state: dict, folder: Path, version) -> None:
|
|
839
|
+
"""Profile the version committed in ``working``, beside it, for the push to upload.
|
|
840
|
+
|
|
841
|
+
A profile is never worth a failed push: when the working catalog no longer
|
|
842
|
+
holds what was committed, or the tables cannot be read, it warns and the
|
|
843
|
+
version goes without them (`fdl profile <remote>` adds them later).
|
|
844
|
+
"""
|
|
845
|
+
described_path = folder / layout.DATASET
|
|
846
|
+
|
|
847
|
+
def unprofiled(wanted: list[dict]) -> list[str]:
|
|
848
|
+
return [
|
|
849
|
+
f"{table['schema']}.{table['name']}"
|
|
850
|
+
for table in wanted
|
|
851
|
+
if not (folder / profiles.path(table["schema"], table["name"])).is_file()
|
|
852
|
+
]
|
|
853
|
+
|
|
854
|
+
try:
|
|
855
|
+
if not described_path.is_file():
|
|
856
|
+
return
|
|
857
|
+
wanted = profiles.tables(json.loads(described_path.read_text(encoding="utf-8")))
|
|
858
|
+
if not unprofiled(wanted):
|
|
859
|
+
return # no tables, or all profiled by a push that stopped before it finished
|
|
860
|
+
if catalog.last_snapshot(working) != version.ducklake.snapshot_id:
|
|
861
|
+
typer.echo(
|
|
862
|
+
"warning: tables were not profiled, as the working catalog changed after the commit; "
|
|
863
|
+
f"commit again, or run fdl profile {state['remote']} after the push",
|
|
864
|
+
err=True,
|
|
865
|
+
)
|
|
866
|
+
return
|
|
867
|
+
parent = repo.read_version(store, version.parents[0]) if version.parents else None
|
|
868
|
+
reading, setup = _reading(store, state)
|
|
869
|
+
repo.write_profiles(store, folder, parent, working, data_path=reading, setup=setup)
|
|
870
|
+
except Exception as exc: # noqa: BLE001 - a profile is never worth a failed push
|
|
871
|
+
typer.echo(f"warning: tables were not profiled ({exc})", err=True)
|
|
872
|
+
return
|
|
873
|
+
_warn_unread(unprofiled(wanted))
|
|
874
|
+
|
|
875
|
+
|
|
802
876
|
@app.command()
|
|
803
877
|
def expire(
|
|
804
878
|
remote: str = Remote,
|
|
@@ -462,7 +462,10 @@ def push(
|
|
|
462
462
|
current_ref = refs.read(store, layout.check_ref(ref))
|
|
463
463
|
if current_ref is not None and current_ref.target == version.id:
|
|
464
464
|
# This push again, after a run whose answer was lost. That run may
|
|
465
|
-
# have stopped before rewriting refs.json
|
|
465
|
+
# have stopped before rewriting refs.json, and a host that does its
|
|
466
|
+
# own work when a ref moves may have stopped after moving it.
|
|
467
|
+
if move_ref:
|
|
468
|
+
_write_ref_again(store, current_ref)
|
|
466
469
|
update_refs_index(store)
|
|
467
470
|
return
|
|
468
471
|
_refuse_unless_fast_forward(store, current_ref, version)
|
|
@@ -489,9 +492,26 @@ def push(
|
|
|
489
492
|
now = refs.read(store, ref)
|
|
490
493
|
if now is None or now.target != version.id:
|
|
491
494
|
raise NotFastForward(f"{ref} moved during the push. Pull and build again.") from exc
|
|
495
|
+
# The first attempt moved it, but its answer was lost: the host may
|
|
496
|
+
# have stopped after the move.
|
|
497
|
+
_write_ref_again(store, now)
|
|
492
498
|
update_refs_index(store)
|
|
493
499
|
|
|
494
500
|
|
|
501
|
+
def _write_ref_again(store: Store, ref: refs.Ref) -> None:
|
|
502
|
+
"""Write ``ref`` to the version it holds, so a host finishes the move.
|
|
503
|
+
|
|
504
|
+
A ref that moved meanwhile was moved by a later push, which did the
|
|
505
|
+
host's work for its own version: nothing is left to finish.
|
|
506
|
+
"""
|
|
507
|
+
if not ref.etag:
|
|
508
|
+
return
|
|
509
|
+
try:
|
|
510
|
+
refs.write(store, ref.name, ref.target, expected=ref)
|
|
511
|
+
except refs.RefConflict:
|
|
512
|
+
pass
|
|
513
|
+
|
|
514
|
+
|
|
495
515
|
def tag(store: Store, name: str, version_id: str) -> None:
|
|
496
516
|
"""Create ``refs/tags/<name>`` at a version. Tags are created, never moved."""
|
|
497
517
|
ref = layout.check_ref(f"refs/tags/{name}")
|
|
@@ -546,7 +566,7 @@ def write_profiles(
|
|
|
546
566
|
"""Write a profile of each table of a committed version under ``local/profiles/``.
|
|
547
567
|
|
|
548
568
|
A table whose data files and columns are the parent's keeps the parent's
|
|
549
|
-
profile, so
|
|
569
|
+
profile, so profiling reads only the tables that changed. Returns how many
|
|
550
570
|
were made from the data.
|
|
551
571
|
"""
|
|
552
572
|
local = Path(local)
|
|
@@ -591,6 +611,48 @@ def write_profiles(
|
|
|
591
611
|
return len(made)
|
|
592
612
|
|
|
593
613
|
|
|
614
|
+
def missing_profiles(store: Store, version: Version) -> list[dict]:
|
|
615
|
+
"""The tables of ``version`` that get a profile and have none in the repository."""
|
|
616
|
+
if version.dataset is None:
|
|
617
|
+
return []
|
|
618
|
+
folder = layout.version_dir(version.id)
|
|
619
|
+
described = json.loads(store.get(folder + version.dataset.path))
|
|
620
|
+
there = set(store.list(folder + layout.PROFILES_DIR))
|
|
621
|
+
return [
|
|
622
|
+
table
|
|
623
|
+
for table in profiles.tables(described)
|
|
624
|
+
if folder + profiles.path(table["schema"], table["name"]) not in there
|
|
625
|
+
]
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def add_profiles(store: Store, version: Version, *, setup: Iterable[str] = ()) -> tuple[list[str], list[str]]:
|
|
629
|
+
"""Profile the tables of a pushed version that have no profile, and put those profiles.
|
|
630
|
+
|
|
631
|
+
For a version made before its writer profiled tables, or whose profiling
|
|
632
|
+
failed. A profile already there is left as it is. Returns the paths put,
|
|
633
|
+
under the version, and the tables (``schema.name``) that could not be read.
|
|
634
|
+
Refused for a version past retention, which has no catalog to check out.
|
|
635
|
+
"""
|
|
636
|
+
wanted = missing_profiles(store, version)
|
|
637
|
+
if not wanted:
|
|
638
|
+
return [], []
|
|
639
|
+
with tempfile.TemporaryDirectory(prefix=".fdl-profile-") as scratch:
|
|
640
|
+
working = Path(scratch) / "work.sqlite"
|
|
641
|
+
checkout(store, working, version=version, local_data=Path(scratch) / "data")
|
|
642
|
+
made = profiles.build(working, wanted, data_path=data_url(store), setup=setup)
|
|
643
|
+
for relative, content in made.items():
|
|
644
|
+
written = Path(scratch) / relative
|
|
645
|
+
written.parent.mkdir(parents=True, exist_ok=True)
|
|
646
|
+
written.write_bytes(content)
|
|
647
|
+
_put_once(store, layout.version_dir(version.id) + relative, written)
|
|
648
|
+
unread = [
|
|
649
|
+
f"{table['schema']}.{table['name']}"
|
|
650
|
+
for table in wanted
|
|
651
|
+
if profiles.path(table["schema"], table["name"]) not in made
|
|
652
|
+
]
|
|
653
|
+
return sorted(made), unread
|
|
654
|
+
|
|
655
|
+
|
|
594
656
|
def upload(store: Store, local: Path, version: Version) -> None:
|
|
595
657
|
"""Put a committed version's files, catalog first and the record last.
|
|
596
658
|
|
|
@@ -1060,6 +1122,8 @@ def forward(store: Store, ref: str, version: Version) -> None:
|
|
|
1060
1122
|
"""
|
|
1061
1123
|
current = refs.read(store, layout.check_ref(ref))
|
|
1062
1124
|
if current is not None and current.target == version.id:
|
|
1125
|
+
# As in `push`: a host may have stopped after a lost move.
|
|
1126
|
+
_write_ref_again(store, current)
|
|
1063
1127
|
update_refs_index(store)
|
|
1064
1128
|
return
|
|
1065
1129
|
_refuse_unless_fast_forward(store, current, version)
|
|
@@ -1070,6 +1134,7 @@ def forward(store: Store, ref: str, version: Version) -> None:
|
|
|
1070
1134
|
now = refs.read(store, ref)
|
|
1071
1135
|
if now is None or now.target != version.id:
|
|
1072
1136
|
raise NotFastForward(f"{ref} moved meanwhile. Pull and merge again.") from exc
|
|
1137
|
+
_write_ref_again(store, now)
|
|
1073
1138
|
update_refs_index(store)
|
|
1074
1139
|
|
|
1075
1140
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""Profiles: a summary of each table, made from the data at
|
|
1
|
+
"""Profiles: a summary of each table, made from the data at push."""
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
@@ -201,3 +201,66 @@ def test_a_geometry_column_gets_its_extent_and_a_sample(repository) -> None:
|
|
|
201
201
|
assert got["geometry"]["types"] == ["POINT"]
|
|
202
202
|
assert len(got["geometry"]["sample"]["features"]) == 3
|
|
203
203
|
assert [r.keys() for r in got["head"]][0] == {"id"}
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def test_profile_adds_the_profiles_a_pushed_version_lacks(repository) -> None:
|
|
207
|
+
store, working = repository
|
|
208
|
+
write(working, store, "CREATE TABLE lake.main.towns AS SELECT i AS id FROM range(4) t(i)")
|
|
209
|
+
write(working, store, "CREATE TABLE lake.main.other AS SELECT 1 AS id")
|
|
210
|
+
commit(working, "-m", "m", "--author", "t")
|
|
211
|
+
head = repo.head(store)
|
|
212
|
+
kept = store.get(f"{layout.version_dir(head.id)}profiles/main/other.json")
|
|
213
|
+
# A version made before its writer profiled: no profile for towns.
|
|
214
|
+
(Path(store.url) / f"{layout.version_dir(head.id)}profiles/main/towns.json").unlink()
|
|
215
|
+
|
|
216
|
+
output = run("profile", store.url)
|
|
217
|
+
assert "put profiles/main/towns.json" in output
|
|
218
|
+
assert "Profiled 1 table(s)" in output
|
|
219
|
+
assert profile_of(store)["rows"] == 4
|
|
220
|
+
assert store.get(f"{layout.version_dir(head.id)}profiles/main/other.json") == kept
|
|
221
|
+
assert "Profiled 0 table(s)" in run("profile", store.url)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def test_push_no_profile_uploads_none(repository) -> None:
|
|
225
|
+
store, working = repository
|
|
226
|
+
write(working, store, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
|
|
227
|
+
run("commit", working, "-m", "m", "--author", "t")
|
|
228
|
+
run("push", working, "--no-profile")
|
|
229
|
+
assert store.list(f"{layout.version_dir(repo.head(store).id)}profiles/") == []
|
|
230
|
+
assert "Profiled 1 table(s)" in run("profile", store.url)
|
|
231
|
+
assert profile_of(store)["rows"] == 1
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def test_push_skips_profiles_when_the_working_catalog_changed_after_the_commit(repository) -> None:
|
|
235
|
+
store, working = repository
|
|
236
|
+
write(working, store, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
|
|
237
|
+
run("commit", working, "-m", "m", "--author", "t")
|
|
238
|
+
write(working, store, "INSERT INTO lake.main.towns VALUES (2)")
|
|
239
|
+
result = runner.invoke(app, ["push", str(working)])
|
|
240
|
+
assert result.exit_code == 0, result.output
|
|
241
|
+
assert "changed after the commit" in result.output
|
|
242
|
+
assert store.list(f"{layout.version_dir(repo.head(store).id)}profiles/") == []
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def test_push_goes_without_profiles_when_the_working_catalog_is_gone(repository) -> None:
|
|
246
|
+
store, working = repository
|
|
247
|
+
write(working, store, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
|
|
248
|
+
run("commit", working, "-m", "m", "--author", "t")
|
|
249
|
+
working.unlink()
|
|
250
|
+
result = runner.invoke(app, ["push", str(working)])
|
|
251
|
+
assert result.exit_code == 0, result.output
|
|
252
|
+
assert "were not profiled" in result.output
|
|
253
|
+
assert store.list(f"{layout.version_dir(repo.head(store).id)}profiles/") == []
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def test_push_profiles_the_tables_a_stopped_push_left_without(repository) -> None:
|
|
257
|
+
store, working = repository
|
|
258
|
+
write(working, store, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
|
|
259
|
+
write(working, store, "CREATE TABLE lake.main.roads AS SELECT 1 AS id")
|
|
260
|
+
run("commit", working, "-m", "m", "--author", "t")
|
|
261
|
+
pending = next((working.parent / f"{working.name}.pending").iterdir())
|
|
262
|
+
partial = pending / "profiles" / "main" / "towns.json"
|
|
263
|
+
partial.parent.mkdir(parents=True)
|
|
264
|
+
partial.write_bytes(b'{"rows": 1, "columns": []}')
|
|
265
|
+
run("push", working)
|
|
266
|
+
assert profile_of(store, "roads")["rows"] == 1
|
|
@@ -17,6 +17,8 @@ from fdl import catalog, diff, layout, refs, repo
|
|
|
17
17
|
from fdl.stores import LocalStore
|
|
18
18
|
from fdl.version import is_id, new_id, parse
|
|
19
19
|
|
|
20
|
+
REAL_READ = refs.read
|
|
21
|
+
|
|
20
22
|
AUTHOR = {"name": "tester"}
|
|
21
23
|
|
|
22
24
|
|
|
@@ -138,6 +140,45 @@ def test_pushing_the_same_version_again_is_accepted(tmp_path: Path, remote) -> N
|
|
|
138
140
|
assert repo.head(remote) == first
|
|
139
141
|
|
|
140
142
|
|
|
143
|
+
def test_a_push_sent_again_writes_the_ref_again(tmp_path: Path, remote, monkeypatch) -> None:
|
|
144
|
+
"""A host that does its work when the ref moves may have stopped after
|
|
145
|
+
moving it. The second push gives it the write again."""
|
|
146
|
+
build(tmp_path / "work.sqlite", remote, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
|
|
147
|
+
version = commit(tmp_path, remote)
|
|
148
|
+
repo.push(remote, tmp_path / "local" / version.id, version)
|
|
149
|
+
seen = refs.read(remote, "refs/heads/main")
|
|
150
|
+
written: list[tuple[str, str | None]] = []
|
|
151
|
+
put = remote.put
|
|
152
|
+
monkeypatch.setattr(
|
|
153
|
+
remote,
|
|
154
|
+
"put",
|
|
155
|
+
lambda key, *a, **kw: written.append((key, kw.get("if_match"))) or put(key, *a, **kw),
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
repo.push(remote, tmp_path / "local" / version.id, version)
|
|
159
|
+
|
|
160
|
+
# Written on condition of the ETag read, so a ref moved since is left.
|
|
161
|
+
assert ("refs/heads/main", seen.etag) in written
|
|
162
|
+
assert repo.head(remote) == version
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def test_a_push_sent_again_leaves_a_ref_moved_since(tmp_path: Path, remote, monkeypatch) -> None:
|
|
166
|
+
build(tmp_path / "work.sqlite", remote, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
|
|
167
|
+
version = commit(tmp_path, remote)
|
|
168
|
+
repo.push(remote, tmp_path / "local" / version.id, version)
|
|
169
|
+
later = new_id()
|
|
170
|
+
seen = refs.read(remote, "refs/heads/main")
|
|
171
|
+
# Another push moves main between this one's read and its write.
|
|
172
|
+
monkeypatch.setattr(
|
|
173
|
+
refs, "read", lambda store, name: seen if name == "refs/heads/main" else REAL_READ(store, name)
|
|
174
|
+
)
|
|
175
|
+
refs.write(remote, "refs/heads/main", later, expected=seen)
|
|
176
|
+
|
|
177
|
+
repo.push(remote, tmp_path / "local" / version.id, version)
|
|
178
|
+
|
|
179
|
+
assert REAL_READ(remote, "refs/heads/main").target == later
|
|
180
|
+
|
|
181
|
+
|
|
141
182
|
def test_push_without_moving_the_ref_leaves_readers_where_they_were(
|
|
142
183
|
tmp_path: Path, remote
|
|
143
184
|
) -> None:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|