fdl-cli 0.14.1__tar.gz → 0.16.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/CHANGELOG.md +36 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/PKG-INFO +1 -1
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/reference/cli.md +1 -1
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/spec/data-files.md +1 -1
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/spec/layout.md +1 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/spec/versions.md +39 -1
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/spec/writing.md +2 -2
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/reference/cli.md +1 -1
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/spec/data-files.md +1 -1
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/spec/layout.md +1 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/spec/versions.md +39 -1
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/spec/writing.md +2 -2
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/pyproject.toml +1 -1
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/src/fdl/catalog.py +97 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/src/fdl/cli.py +2 -2
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/src/fdl/dataset.py +23 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/src/fdl/layout.py +2 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/src/fdl/repo.py +14 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/src/fdl/version.py +20 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/tests/test_dataset.py +13 -1
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/tests/test_repo.py +42 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/uv.lock +1 -1
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/.github/workflows/docs.yml +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/.github/workflows/publish.yml +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/.github/workflows/release.yml +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/.gitignore +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/.node-version +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/.python-version +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/CLAUDE.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/CONTRIBUTING.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/LICENSE +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/README.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/blume.config.ts +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/concepts/meta.ts +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/concepts/repositories.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/concepts/versions-and-refs.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/concepts/working-catalogs.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/getting-started.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/guides/building-a-host.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/guides/meta.ts +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/guides/mirrors.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/guides/public-and-private.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/guides/serving-over-http.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/guides/stores.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/index.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/internals/inside-a-repository.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/internals/meta.ts +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/concepts/meta.ts +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/concepts/repositories.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/concepts/versions-and-refs.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/concepts/working-catalogs.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/getting-started.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/guides/building-a-host.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/guides/meta.ts +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/guides/mirrors.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/guides/public-and-private.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/guides/serving-over-http.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/guides/stores.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/index.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/internals/inside-a-repository.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/internals/meta.ts +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/reference/meta.ts +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/reference/python.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/spec/http.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/spec/index.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/spec/meta.ts +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/spec/reading.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/ja/spec/refs.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/reference/meta.ts +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/reference/python.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/spec/http.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/spec/index.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/spec/meta.ts +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/spec/reading.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/docs/spec/refs.md +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/package-lock.json +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/package.json +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/public/CNAME +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/src/fdl/__init__.py +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/src/fdl/py.typed +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/src/fdl/refs.py +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/src/fdl/store.py +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/src/fdl/stores.py +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/tests/http_repo.py +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/tests/test_cli.py +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/tests/test_http.py +0 -0
- {fdl_cli-0.14.1 → fdl_cli-0.16.0}/tests/test_stores.py +0 -0
|
@@ -1,6 +1,42 @@
|
|
|
1
1
|
# CHANGELOG
|
|
2
2
|
|
|
3
3
|
|
|
4
|
+
## v0.16.0 (2026-11-01)
|
|
5
|
+
|
|
6
|
+
### Documentation
|
|
7
|
+
|
|
8
|
+
- Compare files.json with Git's reachability bitmap
|
|
9
|
+
([`181a738`](https://github.com/queria-io/fdl/commit/181a738c9b09dde2e74adea19b9b520884d76833))
|
|
10
|
+
|
|
11
|
+
### Features
|
|
12
|
+
|
|
13
|
+
- Index each table's files in files.json, not version.json
|
|
14
|
+
([`74412af`](https://github.com/queria-io/fdl/commit/74412af89913284ea48ab66f333da0725fb724b9))
|
|
15
|
+
|
|
16
|
+
files.json sits beside the catalog as an optional index made from it, like Git's commit-graph.
|
|
17
|
+
version.json names it with its size and sha256. Tables with a renamed column are marked, and
|
|
18
|
+
readers without DuckLake must not read a table with deletes or renamed columns. Expiry deletes the
|
|
19
|
+
index with the catalogs.
|
|
20
|
+
|
|
21
|
+
- List each table's files in version.json
|
|
22
|
+
([`1134b2d`](https://github.com/queria-io/fdl/commit/1134b2df324b6eb3fe0fa3598c5bf355bed64561))
|
|
23
|
+
|
|
24
|
+
Readers without DuckDB (a Worker reading Parquet by range) need to know which files make up a table
|
|
25
|
+
at a version. commit now writes tables with each data file's path under data_dir, rows, bytes,
|
|
26
|
+
footer size and its delete file.
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
## v0.15.0 (2026-11-01)
|
|
30
|
+
|
|
31
|
+
### Features
|
|
32
|
+
|
|
33
|
+
- Read tables from *.table.yml files beside the description
|
|
34
|
+
([`3ffa77a`](https://github.com/queria-io/fdl/commit/3ffa77a620ec818ce59ea5f957c7b341bae72f8d))
|
|
35
|
+
|
|
36
|
+
A table can be described in its own file next to what builds it. fdl commit adds the tables in
|
|
37
|
+
*.table.yml / *.table.json below the description's folder, skipping hidden folders.
|
|
38
|
+
|
|
39
|
+
|
|
4
40
|
## v0.14.1 (2026-11-01)
|
|
5
41
|
|
|
6
42
|
### Bug Fixes
|
|
@@ -41,7 +41,7 @@ fdl commit <working.sqlite> -m "What changed" --author alice [--dataset dataset.
|
|
|
41
41
|
|
|
42
42
|
作業カタログを、チェックアウトしたバージョンを親とする新しいバージョンに固め、アップロードし、条件付きでブランチを動かします。ブランチ、新しいバージョン ID、`seq` を表示します。
|
|
43
43
|
|
|
44
|
-
バージョンはそれぞれ `dataset.json`([バージョン](/ja/spec/versions#datasetjson)を参照)で説明します。`--dataset` で指定したファイルか、なければいまのディレクトリの `dataset.json` か `dataset.yml`
|
|
44
|
+
バージョンはそれぞれ `dataset.json`([バージョン](/ja/spec/versions#datasetjson)を参照)で説明します。`--dataset` で指定したファイルか、なければいまのディレクトリの `dataset.json` か `dataset.yml` から読みます。そのフォルダより下にある `*.table.yml`(または `*.table.json`)のテーブルの説明も合わせて読みます。隠しフォルダの中は読みません。コマンドはそれをカタログと照らし合わせ、列の型、行数、カタログのコメントを書き足します。
|
|
45
45
|
|
|
46
46
|
次のときは断られます。
|
|
47
47
|
|
|
@@ -24,7 +24,7 @@ DuckLake 自身の片付け(`ducklake_cleanup_old_files`、`ducklake_expire_sn
|
|
|
24
24
|
バージョンの保持期間は、次の順に終わらせます。
|
|
25
25
|
|
|
26
26
|
1. `versions/<ID>/expired.json` を置く
|
|
27
|
-
2.
|
|
27
|
+
2. バージョンのカタログ(`ducklake.duckdb` と `catalogs/`)と `files.json` を消す
|
|
28
28
|
3. 猶予期間のあと、上の条件に合うデータファイルを消す
|
|
29
29
|
|
|
30
30
|
```json expired.json
|
|
@@ -21,6 +21,7 @@ description: リポジトリのファイル、そのパス、fdl.json の中身
|
|
|
21
21
|
version.json.sig 記録の署名(任意)
|
|
22
22
|
ducklake.duckdb 固めたカタログ
|
|
23
23
|
dataset.json バージョンの中身の説明
|
|
24
|
+
files.json カタログから作った、表ごとのデータファイルの索引(任意)
|
|
24
25
|
catalogs/<format>.duckdb 別の DuckLake 形式のカタログ(任意)
|
|
25
26
|
expired.json 保持期間を過ぎた印(過ぎたときだけ)
|
|
26
27
|
<others> ホストが足すファイル(プレビューなど)
|
|
@@ -43,6 +43,11 @@ description: バージョン ID、version.json の項目、固めたカタログ
|
|
|
43
43
|
"snapshot_id": 345,
|
|
44
44
|
"snapshots": [301, 345]
|
|
45
45
|
},
|
|
46
|
+
"files": {
|
|
47
|
+
"path": "files.json",
|
|
48
|
+
"bytes": 1536,
|
|
49
|
+
"sha256": "7a0e…"
|
|
50
|
+
},
|
|
46
51
|
"changes": [
|
|
47
52
|
{ "table": "main.population", "kind": "add_column", "column": "note", "breaking": false }
|
|
48
53
|
],
|
|
@@ -70,6 +75,7 @@ description: バージョン ID、version.json の項目、固めたカタログ
|
|
|
70
75
|
| `catalog` | must | カタログのファイル名、大きさ、sha256、保存形式(`storage`) |
|
|
71
76
|
| `dataset` | must | カタログの隣の `dataset.json` のファイル名、大きさ、sha256。必須になる前に書かれたバージョンには無い |
|
|
72
77
|
| `ducklake` | must | DuckLake の形式、固めたスナップショット、親から足したスナップショットの範囲(最初と最後) |
|
|
78
|
+
| `files` | may | カタログの隣の `files.json` を書いたときの、そのファイル名、大きさ、sha256([files.json](#filesjson)) |
|
|
73
79
|
| `changes` | may | 親との列の違い。`breaking` は、親に向けて書いた SQL が失敗しうるか(列の削除・改名・作り直し) |
|
|
74
80
|
| `inputs` | may | 使った元のファイル。`source_id` の付け方は書き手が決める |
|
|
75
81
|
| `extensions` | may | ホスト固有の項目。ホストの名前をキーにする |
|
|
@@ -90,6 +96,38 @@ FDL 0.1 では、書き手が書く親は多くても 1 つです。`parents`
|
|
|
90
96
|
|
|
91
97
|
読み手は、`version.json` だけで、そのバージョンを開けるか判断できます。
|
|
92
98
|
|
|
99
|
+
## files.json
|
|
100
|
+
|
|
101
|
+
`files.json` は、`ducklake.snapshot_id` の時点の表ごとのデータファイルの索引です。カタログを開けない読み手のためのものです。書き手はこれをカタログから作り、カタログと食い違ったときはカタログが正です。置くかどうかは任意で、読み手はこれが無くても動かなければなりません。Git の到達可能性ビットマップ(pack の隣の `.bitmap`)に当たります。あるコミットからたどれるオブジェクトの一覧をオブジェクトから作ったもので、読むのを速くし、無くても読めます。
|
|
102
|
+
|
|
103
|
+
```json files.json
|
|
104
|
+
{
|
|
105
|
+
"tables": {
|
|
106
|
+
"main.population": {
|
|
107
|
+
"files": [
|
|
108
|
+
{
|
|
109
|
+
"path": "main/population/ducklake-019a3c4e-….parquet",
|
|
110
|
+
"rows": 1741,
|
|
111
|
+
"bytes": 52311,
|
|
112
|
+
"footer_bytes": 2048,
|
|
113
|
+
"deletes": { "path": "main/population/ducklake-019a3c4f-…-delete.parquet", "rows": 3, "bytes": 812 }
|
|
114
|
+
}
|
|
115
|
+
]
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
- `tables` には、そのスナップショットにあるすべての表を `schema.table` をキーにして入れます。行の無い表の `files` は空です
|
|
122
|
+
- `path` はリポジトリの `data_dir` からの相対パスです。ファイルはカタログの順に並べます
|
|
123
|
+
- `rows` と `bytes` は、カタログが記録している行数と大きさです。`footer_bytes` は Parquet のフッターの大きさで、記録があるときだけ書きます
|
|
124
|
+
- `deletes` は、そのデータファイルから行を消したときの削除ファイルです
|
|
125
|
+
- `renamed_columns` は、その表の列の名前を変えたことがあるときに `true` にします。名前を変える前に書いたデータファイルは、古い名前のままです
|
|
126
|
+
|
|
127
|
+
DuckLake を使わずに Parquet ファイルを読む読み手は、`deletes` を持つファイルがある表や、`renamed_columns` が `true` の表を読んではいけません。DuckLake と同じように削除と改名を当てはめる場合は読んでかまいません。当てはめずに読むと、消した行が返ったり、名前を変えた列が欠けたりします。
|
|
128
|
+
|
|
129
|
+
保持期間から外すとき、`files.json` はカタログと一緒に消します。
|
|
130
|
+
|
|
93
131
|
## 固めたカタログ
|
|
94
132
|
|
|
95
133
|
`ducklake.duckdb` は、DuckDB のファイル形式の DuckLake カタログです。
|
|
@@ -142,7 +180,7 @@ FDL 0.1 では、書き手が書く親は多くても 1 つです。`parents`
|
|
|
142
180
|
```
|
|
143
181
|
|
|
144
182
|
- 必須なのは `name` と `title` だけで、ほかは省いてよい
|
|
145
|
-
- 書き手は説明を YAML(`dataset.yml
|
|
183
|
+
- 書き手は説明を YAML(`dataset.yml`)で持ってもよく、テーブルごとに別のファイル(`<name>.table.yml`、`tables` の 1 項目)に分けてもよい。バージョンには常に説明全体が 1 つの JSON で入る
|
|
146
184
|
- `licenses` の各要素は SPDX の識別子か、`id` に書き手が足したいもの(`spdx`・`url`・`title`)を添えたオブジェクト
|
|
147
185
|
- `tables` には、カタログの最後のスナップショットにある表をすべて並べる。書き手が説明しなかった表は、書き手のツールがカタログから分かることで足す。カタログに無い表や列を説明していたら断る
|
|
148
186
|
- カタログから取るもの: 表ごとの `rows`、列ごとの `type`(DuckLake の型名)と `nullable`。書き手が `description` を書かなかったところには、カタログのコメント(`COMMENT ON`)を使う
|
|
@@ -17,7 +17,7 @@ description: チェックアウト、コミット、push。作業カタログ、
|
|
|
17
17
|
| --- | --- |
|
|
18
18
|
| チェックアウト | ブランチのバージョンのカタログから作業カタログを作り、そのバージョンを覚える。最初のバージョンの前は空のものを作る |
|
|
19
19
|
| コミット | 作業カタログの最後のスナップショットを新しいバージョンに固める。親は覚えておいたバージョン。中身を `dataset.json` で説明する |
|
|
20
|
-
| push | データファイルを確かめ、カタログ、`dataset.json`、`version.json` の順にアップロードし、条件付きで ref を動かす |
|
|
20
|
+
| push | データファイルを確かめ、カタログ、`dataset.json` と `files.json`、`version.json` の順にアップロードし、条件付きで ref を動かす |
|
|
21
21
|
|
|
22
22
|
チェックアウトでは、DuckLake の管理用の表を、固めたカタログから SQLite へそのまま写します。固めるときはその逆で、最後のスナップショットまでの行を残します。どちらもデータファイルは書き換えません。
|
|
23
23
|
|
|
@@ -31,7 +31,7 @@ description: チェックアウト、コミット、push。作業カタログ、
|
|
|
31
31
|
|
|
32
32
|
1. データファイル
|
|
33
33
|
2. `versions/<ID>/ducklake.duckdb`(と `catalogs/`)
|
|
34
|
-
3. `versions/<ID>/dataset.json`
|
|
34
|
+
3. `versions/<ID>/dataset.json` と `versions/<ID>/files.json`
|
|
35
35
|
4. `versions/<ID>/version.json`
|
|
36
36
|
5. ref(条件付き)
|
|
37
37
|
|
|
@@ -41,7 +41,7 @@ fdl commit <working.sqlite> -m "What changed" --author alice [--dataset dataset.
|
|
|
41
41
|
|
|
42
42
|
Freezes the working catalog into a new version whose parent is the version it was checked out at, uploads it, and moves the branch on condition. It prints the branch, the new version ID and its `seq`.
|
|
43
43
|
|
|
44
|
-
Every version is described by a `dataset.json` (see [Versions](/spec/versions#datasetjson)), read from `--dataset` or else from `dataset.json` or `dataset.yml` in the current directory. The command checks it against the catalog and fills in the column types, row counts and catalog comments.
|
|
44
|
+
Every version is described by a `dataset.json` (see [Versions](/spec/versions#datasetjson)), read from `--dataset` or else from `dataset.json` or `dataset.yml` in the current directory. Tables described in `*.table.yml` (or `*.table.json`) files below that folder are added to it; hidden folders are skipped. The command checks it against the catalog and fills in the column types, row counts and catalog comments.
|
|
45
45
|
|
|
46
46
|
Refused when:
|
|
47
47
|
|
|
@@ -24,7 +24,7 @@ DuckLake's own cleanup (`ducklake_cleanup_old_files`, `ducklake_expire_snapshots
|
|
|
24
24
|
A version is taken out of retention in this order:
|
|
25
25
|
|
|
26
26
|
1. Put `versions/<ID>/expired.json`
|
|
27
|
-
2. Delete the version's catalogs
|
|
27
|
+
2. Delete the version's catalogs, `ducklake.duckdb` and `catalogs/`, and its `files.json`
|
|
28
28
|
3. After the grace period, delete the data files that meet the conditions above
|
|
29
29
|
|
|
30
30
|
```json expired.json
|
|
@@ -21,6 +21,7 @@ Paths are relative to the root, lower case, and separated by `/`. The root is wh
|
|
|
21
21
|
version.json.sig a signature of the record (optional)
|
|
22
22
|
ducklake.duckdb the frozen catalog
|
|
23
23
|
dataset.json what the version holds, described
|
|
24
|
+
files.json each table's data files, indexed from the catalog (optional)
|
|
24
25
|
catalogs/<format>.duckdb the catalog in another DuckLake format (optional)
|
|
25
26
|
expired.json the past-retention mark (only once expired)
|
|
26
27
|
<others> files a host adds, such as previews
|
|
@@ -43,6 +43,11 @@ A version is the files under `versions/<version ID>/`. They must not change once
|
|
|
43
43
|
"snapshot_id": 345,
|
|
44
44
|
"snapshots": [301, 345]
|
|
45
45
|
},
|
|
46
|
+
"files": {
|
|
47
|
+
"path": "files.json",
|
|
48
|
+
"bytes": 1536,
|
|
49
|
+
"sha256": "7a0e…"
|
|
50
|
+
},
|
|
46
51
|
"changes": [
|
|
47
52
|
{ "table": "main.population", "kind": "add_column", "column": "note", "breaking": false }
|
|
48
53
|
],
|
|
@@ -70,6 +75,7 @@ A version is the files under `versions/<version ID>/`. They must not change once
|
|
|
70
75
|
| `catalog` | must | The catalog's file name, size, sha256, and storage format (`storage`) |
|
|
71
76
|
| `dataset` | must | The `dataset.json` beside the catalog: its file name, size and sha256. Versions written before it was required lack it |
|
|
72
77
|
| `ducklake` | must | The DuckLake format, the snapshot frozen, and the range of snapshots this version adds after its parent (first and last) |
|
|
78
|
+
| `files` | may | The `files.json` beside the catalog, if written: its file name, size and sha256 ([files.json](#filesjson)) |
|
|
73
79
|
| `changes` | may | Column differences from the parent. `breaking` says whether SQL written for the parent may fail (a column dropped, renamed or recreated) |
|
|
74
80
|
| `inputs` | may | Source files used. How `source_id` is assigned is up to the writer |
|
|
75
81
|
| `extensions` | may | Host-specific fields, keyed by the host's name |
|
|
@@ -90,6 +96,38 @@ The description and SQL of each snapshot stay in the catalog, in DuckLake's `com
|
|
|
90
96
|
|
|
91
97
|
A reader can tell from `version.json` alone whether it can open a version.
|
|
92
98
|
|
|
99
|
+
## files.json
|
|
100
|
+
|
|
101
|
+
`files.json` is an index of each table's data files at `ducklake.snapshot_id`, for readers that cannot open the catalog. The writer makes it from the catalog, and the catalog wins where the two differ. It is optional: a reader must work without it. It plays the part of Git's reachability bitmap (`.bitmap` beside a pack): a list, made from the objects, of what one commit reaches, which speeds up reading and which a reader can do without.
|
|
102
|
+
|
|
103
|
+
```json files.json
|
|
104
|
+
{
|
|
105
|
+
"tables": {
|
|
106
|
+
"main.population": {
|
|
107
|
+
"files": [
|
|
108
|
+
{
|
|
109
|
+
"path": "main/population/ducklake-019a3c4e-….parquet",
|
|
110
|
+
"rows": 1741,
|
|
111
|
+
"bytes": 52311,
|
|
112
|
+
"footer_bytes": 2048,
|
|
113
|
+
"deletes": { "path": "main/population/ducklake-019a3c4f-…-delete.parquet", "rows": 3, "bytes": 812 }
|
|
114
|
+
}
|
|
115
|
+
]
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
- `tables` holds every table alive at the snapshot, keyed `schema.table`. A table with no rows has an empty `files`
|
|
122
|
+
- `path` is relative to the repository's `data_dir`. Files are in the catalog's order
|
|
123
|
+
- `rows` and `bytes` are the row count and size the catalog records. `footer_bytes` is the size of the Parquet footer, when recorded
|
|
124
|
+
- `deletes` is the delete file for that data file, if rows were deleted from it
|
|
125
|
+
- `renamed_columns` is `true` when a column of the table was ever renamed. Data files written before the rename carry the old name
|
|
126
|
+
|
|
127
|
+
A reader that reads the Parquet files without DuckLake must not read a table whose files have `deletes`, or whose `renamed_columns` is `true`, unless it applies them as DuckLake does. Otherwise it returns deleted rows or misses renamed columns.
|
|
128
|
+
|
|
129
|
+
When a version is taken out of retention, `files.json` is deleted with its catalogs.
|
|
130
|
+
|
|
93
131
|
## The frozen catalog
|
|
94
132
|
|
|
95
133
|
`ducklake.duckdb` is a DuckLake catalog in DuckDB's file format.
|
|
@@ -142,7 +180,7 @@ The writer describes; the writer's tool checks the description against the froze
|
|
|
142
180
|
```
|
|
143
181
|
|
|
144
182
|
- `name` and `title` are required. Everything else may be left out
|
|
145
|
-
- A writer may keep its description as YAML (`dataset.yml`)
|
|
183
|
+
- A writer may keep its description as YAML (`dataset.yml`), and may keep each table in its own file (`<name>.table.yml`, one entry of `tables`). The version always holds the whole description as one JSON
|
|
146
184
|
- Each of `licenses` is an SPDX identifier, or an object with `id` and what the writer wants to add (`spdx`, `url`, `title`)
|
|
147
185
|
- `tables` lists every table of the catalog's last snapshot. The writer's tool adds the tables the writer did not describe, with what the catalog knows of them, and refuses a described table or column the catalog lacks
|
|
148
186
|
- From the catalog: each table's `rows`, and each field's `type` (DuckLake's type name) and `nullable`. Comments in the catalog (`COMMENT ON`) are used where the writer gave no `description`
|
|
@@ -17,7 +17,7 @@ A writer writes to a working catalog: a DuckLake catalog in SQLite, outside the
|
|
|
17
17
|
| --- | --- |
|
|
18
18
|
| checkout | Make a working catalog from the catalog of the branch's version, and remember that version. Before the first version, make an empty one |
|
|
19
19
|
| commit | Freeze the working catalog's last snapshot into a new version, with the remembered version as its parent, and describe it in `dataset.json` |
|
|
20
|
-
| push | Check the data files, upload the catalog, `dataset.json`, then `version.json`, then move the ref on condition |
|
|
20
|
+
| push | Check the data files, upload the catalog, `dataset.json` and `files.json`, then `version.json`, then move the ref on condition |
|
|
21
21
|
|
|
22
22
|
Checking out copies DuckLake's bookkeeping tables from the frozen catalog into SQLite as they are. Freezing does the reverse, keeping the rows up to the last snapshot. Neither rewrites a data file.
|
|
23
23
|
|
|
@@ -31,7 +31,7 @@ When freezing, a writer must:
|
|
|
31
31
|
|
|
32
32
|
1. Data files
|
|
33
33
|
2. `versions/<ID>/ducklake.duckdb` (and any `catalogs/`)
|
|
34
|
-
3. `versions/<ID>/dataset.json`
|
|
34
|
+
3. `versions/<ID>/dataset.json` and `versions/<ID>/files.json`
|
|
35
35
|
4. `versions/<ID>/version.json`
|
|
36
36
|
5. The ref, on condition
|
|
37
37
|
|
|
@@ -505,6 +505,103 @@ def storage_version(catalog_path: Path) -> str:
|
|
|
505
505
|
return f"duckdb-{row[0].rstrip('+')}"
|
|
506
506
|
|
|
507
507
|
|
|
508
|
+
def table_files(catalog_path: Path, snapshot_id: int) -> dict[str, dict]:
|
|
509
|
+
"""Each table's files at ``snapshot_id``, keyed ``schema.table``.
|
|
510
|
+
|
|
511
|
+
For readers without DuckDB: what to read for a table, the rows each data
|
|
512
|
+
file has deleted, and whether a column was ever renamed. Paths are
|
|
513
|
+
relative to the data path, as in `data_files`.
|
|
514
|
+
"""
|
|
515
|
+
con = _open(Path(catalog_path))
|
|
516
|
+
live = "begin_snapshot <= ? AND (end_snapshot IS NULL OR end_snapshot > ?)"
|
|
517
|
+
try:
|
|
518
|
+
tables = con.execute(
|
|
519
|
+
"""
|
|
520
|
+
SELECT t.table_id, s.schema_name || '.' || t.table_name,
|
|
521
|
+
coalesce(s.path, '') || coalesce(t.path, '')
|
|
522
|
+
FROM ducklake_table t
|
|
523
|
+
JOIN ducklake_schema s ON s.schema_id = t.schema_id
|
|
524
|
+
WHERE t.begin_snapshot <= ? AND (t.end_snapshot IS NULL OR t.end_snapshot > ?)
|
|
525
|
+
AND s.begin_snapshot <= ? AND (s.end_snapshot IS NULL OR s.end_snapshot > ?)
|
|
526
|
+
ORDER BY 2
|
|
527
|
+
""",
|
|
528
|
+
[snapshot_id] * 4,
|
|
529
|
+
).fetchall()
|
|
530
|
+
files = con.execute(
|
|
531
|
+
f"""
|
|
532
|
+
SELECT data_file_id, table_id, path, path_is_relative, record_count,
|
|
533
|
+
file_size_bytes, footer_size
|
|
534
|
+
FROM ducklake_data_file WHERE {live}
|
|
535
|
+
ORDER BY table_id, file_order NULLS LAST, data_file_id
|
|
536
|
+
""",
|
|
537
|
+
[snapshot_id] * 2,
|
|
538
|
+
).fetchall()
|
|
539
|
+
deletes = (
|
|
540
|
+
con.execute(
|
|
541
|
+
f"""
|
|
542
|
+
SELECT data_file_id, path, path_is_relative, delete_count, file_size_bytes
|
|
543
|
+
FROM ducklake_delete_file WHERE {live}
|
|
544
|
+
""",
|
|
545
|
+
[snapshot_id] * 2,
|
|
546
|
+
).fetchall()
|
|
547
|
+
if _table_exists(con, "ducklake_delete_file")
|
|
548
|
+
else []
|
|
549
|
+
)
|
|
550
|
+
# A renamed column keeps its id under a new name, while files written
|
|
551
|
+
# before the rename still carry the old one.
|
|
552
|
+
renamed = {
|
|
553
|
+
row[0]
|
|
554
|
+
for row in con.execute(
|
|
555
|
+
"""
|
|
556
|
+
SELECT DISTINCT table_id FROM ducklake_column
|
|
557
|
+
WHERE begin_snapshot <= ?
|
|
558
|
+
GROUP BY table_id, column_id
|
|
559
|
+
HAVING count(DISTINCT column_name) > 1
|
|
560
|
+
""",
|
|
561
|
+
[snapshot_id],
|
|
562
|
+
).fetchall()
|
|
563
|
+
}
|
|
564
|
+
except Exception as exc: # noqa: BLE001 - duckdb raises several types
|
|
565
|
+
raise CatalogError(f"{catalog_path} is not a DuckLake catalog: {exc}") from exc
|
|
566
|
+
finally:
|
|
567
|
+
con.close()
|
|
568
|
+
|
|
569
|
+
prefixes = {table_id: prefix for table_id, _, prefix in tables}
|
|
570
|
+
|
|
571
|
+
def where(table_id: int, path: str, relative: bool) -> dict:
|
|
572
|
+
if not relative:
|
|
573
|
+
raise CatalogError(f"{catalog_path} names a data file by an absolute path: {path}")
|
|
574
|
+
return {"path": prefixes[table_id] + path}
|
|
575
|
+
|
|
576
|
+
table_of = {data_id: table_id for data_id, table_id, *_ in files}
|
|
577
|
+
deleted = {
|
|
578
|
+
data_id: {**where(table_of[data_id], path, relative), "rows": rows, "bytes": size}
|
|
579
|
+
for data_id, path, relative, rows, size in deletes
|
|
580
|
+
if data_id in table_of
|
|
581
|
+
}
|
|
582
|
+
listed = {
|
|
583
|
+
table_id: [
|
|
584
|
+
{
|
|
585
|
+
**where(table_id, path, relative),
|
|
586
|
+
"rows": rows,
|
|
587
|
+
"bytes": size,
|
|
588
|
+
**({"footer_bytes": footer} if footer is not None else {}),
|
|
589
|
+
**({"deletes": deleted[data_id]} if data_id in deleted else {}),
|
|
590
|
+
}
|
|
591
|
+
for data_id, owner, path, relative, rows, size, footer in files
|
|
592
|
+
if owner == table_id
|
|
593
|
+
]
|
|
594
|
+
for table_id in prefixes
|
|
595
|
+
}
|
|
596
|
+
return {
|
|
597
|
+
name: {
|
|
598
|
+
"files": listed[table_id],
|
|
599
|
+
**({"renamed_columns": True} if table_id in renamed else {}),
|
|
600
|
+
}
|
|
601
|
+
for table_id, name, _ in tables
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
|
|
508
605
|
def snapshot_range(catalog_path: Path, after: int | None) -> tuple[int, int]:
|
|
509
606
|
"""The first and last snapshot past ``after`` (the parent's last one).
|
|
510
607
|
|
|
@@ -99,7 +99,7 @@ def commit(
|
|
|
99
99
|
describe: Path | None = typer.Option(
|
|
100
100
|
None,
|
|
101
101
|
"--dataset",
|
|
102
|
-
help="The description of what the version holds (default: dataset.json or dataset.yml here)",
|
|
102
|
+
help="The description of what the version holds (default: dataset.json or dataset.yml here), with any *.table.yml below it",
|
|
103
103
|
),
|
|
104
104
|
) -> None:
|
|
105
105
|
"""Freeze the working catalog into a version and push it to its branch."""
|
|
@@ -107,7 +107,7 @@ def commit(
|
|
|
107
107
|
store = open_store(state["remote"])
|
|
108
108
|
out = Path(tempfile.mkdtemp(prefix="fdl-"))
|
|
109
109
|
try:
|
|
110
|
-
declared = dataset.
|
|
110
|
+
declared = dataset.gather(describe or dataset.find())
|
|
111
111
|
parent = repo.read_version(store, state["base"]) if state["base"] else None
|
|
112
112
|
version = repo.commit(
|
|
113
113
|
working,
|
|
@@ -65,6 +65,29 @@ def read(path: Path) -> dict[str, Any]:
|
|
|
65
65
|
return declared
|
|
66
66
|
|
|
67
67
|
|
|
68
|
+
#: Tables may each be described in their own file, next to what builds them.
|
|
69
|
+
TABLE_FILES = ("*.table.json", "*.table.yml", "*.table.yaml")
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def gather(path: Path) -> dict[str, Any]:
|
|
73
|
+
"""The description in ``path`` with the tables described in table files.
|
|
74
|
+
|
|
75
|
+
A table file (``<name>.table.yml``) holds one entry of ``tables``. They are
|
|
76
|
+
looked for below the description's folder, skipping hidden folders.
|
|
77
|
+
"""
|
|
78
|
+
declared = read(path)
|
|
79
|
+
root = Path(path).resolve().parent
|
|
80
|
+
files = sorted(
|
|
81
|
+
found
|
|
82
|
+
for pattern in TABLE_FILES
|
|
83
|
+
for found in root.rglob(pattern)
|
|
84
|
+
if not any(part.startswith(".") for part in found.relative_to(root).parts[:-1])
|
|
85
|
+
)
|
|
86
|
+
if not files:
|
|
87
|
+
return declared
|
|
88
|
+
return {**declared, "tables": [*declared.get("tables", []), *map(read, files)]}
|
|
89
|
+
|
|
90
|
+
|
|
68
91
|
def describe(declared: dict[str, Any], frozen: Path) -> dict[str, Any]:
|
|
69
92
|
"""The `dataset.json` of a version: ``declared`` checked against the catalog.
|
|
70
93
|
|
|
@@ -20,6 +20,8 @@ VERSION_RECORD = "version.json"
|
|
|
20
20
|
CATALOG = "ducklake.duckdb"
|
|
21
21
|
#: What the version holds, described (see `fdl.dataset`).
|
|
22
22
|
DATASET = "dataset.json"
|
|
23
|
+
#: The index of each table's files, made from the catalog (optional).
|
|
24
|
+
FILES = "files.json"
|
|
23
25
|
EXPIRED = "expired.json"
|
|
24
26
|
#: Other formats of the same catalog, under the version (optional).
|
|
25
27
|
CATALOGS_DIR = "catalogs/"
|
|
@@ -243,6 +243,7 @@ def record(
|
|
|
243
243
|
described = File(
|
|
244
244
|
path=layout.DATASET, bytes=written.stat().st_size, sha256=_sha256(written)
|
|
245
245
|
)
|
|
246
|
+
indexed = _write_files_index(target, copied, last)
|
|
246
247
|
version = Version(
|
|
247
248
|
id=version_id,
|
|
248
249
|
seq=parent.seq + 1 if parent else 1,
|
|
@@ -257,6 +258,7 @@ def record(
|
|
|
257
258
|
sha256=_sha256(copied),
|
|
258
259
|
),
|
|
259
260
|
ducklake=DuckLake(format=recorded, snapshot_id=last, snapshots=(first, last)),
|
|
261
|
+
files=indexed,
|
|
260
262
|
inputs=tuple(inputs),
|
|
261
263
|
extensions=dict(extensions or {}),
|
|
262
264
|
dataset=described,
|
|
@@ -270,6 +272,14 @@ def record(
|
|
|
270
272
|
return version
|
|
271
273
|
|
|
272
274
|
|
|
275
|
+
def _write_files_index(target: Path, frozen: Path, snapshot_id: int) -> File:
|
|
276
|
+
"""Write `files.json`: each table's files at the frozen snapshot, from the catalog."""
|
|
277
|
+
written = target / layout.FILES
|
|
278
|
+
index = {"tables": catalog.table_files(frozen, snapshot_id)}
|
|
279
|
+
written.write_bytes((json.dumps(index, ensure_ascii=False, indent=2) + "\n").encode())
|
|
280
|
+
return File(path=layout.FILES, bytes=written.stat().st_size, sha256=_sha256(written))
|
|
281
|
+
|
|
282
|
+
|
|
273
283
|
def push(
|
|
274
284
|
store: Store,
|
|
275
285
|
local: Path,
|
|
@@ -332,6 +342,8 @@ def upload(store: Store, local: Path, version: Version) -> None:
|
|
|
332
342
|
# Before the record, which is what makes a version there at all.
|
|
333
343
|
if version.dataset is not None:
|
|
334
344
|
_put_once(store, layout.version_dir(version.id) + version.dataset.path, local / version.dataset.path)
|
|
345
|
+
if version.files is not None:
|
|
346
|
+
_put_once(store, layout.version_dir(version.id) + version.files.path, local / version.files.path)
|
|
335
347
|
_put_once(store, layout.version_record(version.id), local / layout.VERSION_RECORD)
|
|
336
348
|
|
|
337
349
|
|
|
@@ -393,6 +405,8 @@ def expire(store: Store, version_id: str, *, reason: str = "retention") -> None:
|
|
|
393
405
|
store.delete(
|
|
394
406
|
[
|
|
395
407
|
layout.version_catalog(version_id),
|
|
408
|
+
# The index names files the grace period may delete.
|
|
409
|
+
layout.version_dir(version_id) + layout.FILES,
|
|
396
410
|
*store.list(layout.version_dir(version_id) + layout.CATALOGS_DIR),
|
|
397
411
|
]
|
|
398
412
|
)
|
|
@@ -76,6 +76,8 @@ class Version:
|
|
|
76
76
|
changes: tuple[dict, ...] = ()
|
|
77
77
|
inputs: tuple[dict, ...] = ()
|
|
78
78
|
extensions: dict = field(default_factory=dict)
|
|
79
|
+
#: The version's `files.json`, the index of each table's files. Optional.
|
|
80
|
+
files: File | None = None
|
|
79
81
|
#: The version's `dataset.json`. None only in versions written before it was required.
|
|
80
82
|
dataset: File | None = None
|
|
81
83
|
fdl: str = layout.FDL_VERSION
|
|
@@ -111,6 +113,17 @@ class Version:
|
|
|
111
113
|
"snapshot_id": self.ducklake.snapshot_id,
|
|
112
114
|
"snapshots": list(self.ducklake.snapshots),
|
|
113
115
|
},
|
|
116
|
+
**(
|
|
117
|
+
{
|
|
118
|
+
"files": {
|
|
119
|
+
"path": self.files.path,
|
|
120
|
+
"bytes": self.files.bytes,
|
|
121
|
+
"sha256": self.files.sha256,
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
if self.files
|
|
125
|
+
else {}
|
|
126
|
+
),
|
|
114
127
|
"changes": list(self.changes),
|
|
115
128
|
"inputs": list(self.inputs),
|
|
116
129
|
"extensions": self.extensions,
|
|
@@ -152,6 +165,7 @@ def parse(data: bytes, *, expected_id: str | None = None) -> Version:
|
|
|
152
165
|
changes=tuple(record.get("changes", ())),
|
|
153
166
|
inputs=tuple(record.get("inputs", ())),
|
|
154
167
|
extensions=dict(record.get("extensions", {})),
|
|
168
|
+
files=_file(record.get("files")),
|
|
155
169
|
dataset=(
|
|
156
170
|
File(path=dataset["path"], bytes=int(dataset["bytes"]), sha256=dataset["sha256"])
|
|
157
171
|
if dataset
|
|
@@ -164,5 +178,11 @@ def parse(data: bytes, *, expected_id: str | None = None) -> Version:
|
|
|
164
178
|
raise VersionError(f"not a version record: {exc}") from exc
|
|
165
179
|
|
|
166
180
|
|
|
181
|
+
def _file(entry: dict | None) -> File | None:
|
|
182
|
+
if not entry:
|
|
183
|
+
return None
|
|
184
|
+
return File(path=entry["path"], bytes=int(entry["bytes"]), sha256=entry["sha256"])
|
|
185
|
+
|
|
186
|
+
|
|
167
187
|
def utc_now() -> str:
|
|
168
188
|
return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
@@ -7,7 +7,7 @@ from pathlib import Path
|
|
|
7
7
|
import duckdb
|
|
8
8
|
import pytest
|
|
9
9
|
|
|
10
|
-
from fdl.dataset import DatasetError, describe
|
|
10
|
+
from fdl.dataset import DatasetError, describe, gather
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
@pytest.fixture
|
|
@@ -68,3 +68,15 @@ def test_a_description_can_be_yaml(tmp_path: Path, frozen: Path) -> None:
|
|
|
68
68
|
from fdl.dataset import to_json
|
|
69
69
|
|
|
70
70
|
assert b'"issued": "2026-10-30"' in to_json(described)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def test_tables_can_be_described_in_their_own_files(tmp_path: Path, frozen: Path) -> None:
|
|
74
|
+
(tmp_path / "dataset.yml").write_text("name: towns\ntitle: Towns\n")
|
|
75
|
+
(tmp_path / "models").mkdir()
|
|
76
|
+
(tmp_path / "models" / "towns.table.yml").write_text(
|
|
77
|
+
"schema: main\nname: towns\ntitle: The towns\n"
|
|
78
|
+
)
|
|
79
|
+
(tmp_path / ".fdl").mkdir()
|
|
80
|
+
(tmp_path / ".fdl" / "stale.table.yml").write_text("schema: main\nname: gone\n")
|
|
81
|
+
(table,) = describe(gather(tmp_path / "dataset.yml"), frozen)["tables"]
|
|
82
|
+
assert table["title"] == "The towns"
|
|
@@ -251,3 +251,45 @@ def test_prune_keeps_what_was_made_since(tmp_path: Path, remote) -> None:
|
|
|
251
251
|
versions[1].id,
|
|
252
252
|
versions[0].id,
|
|
253
253
|
]
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def files_index(store: LocalStore, version) -> dict:
|
|
258
|
+
data = store.get(layout.version_dir(version.id) + version.files.path)
|
|
259
|
+
assert len(data) == version.files.bytes
|
|
260
|
+
return json.loads(data)["tables"]
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def test_a_version_indexes_each_tables_files(tmp_path: Path, remote) -> None:
|
|
264
|
+
build(
|
|
265
|
+
tmp_path / "work.sqlite",
|
|
266
|
+
remote,
|
|
267
|
+
"CREATE TABLE lake.main.towns AS SELECT range AS id FROM range(3);"
|
|
268
|
+
"DELETE FROM lake.main.towns WHERE id = 0",
|
|
269
|
+
)
|
|
270
|
+
version = commit(tmp_path, remote)
|
|
271
|
+
repo.push(remote, tmp_path / "local" / version.id, version)
|
|
272
|
+
|
|
273
|
+
assert parse(remote.get(layout.version_record(version.id))).files == version.files
|
|
274
|
+
(data_file,) = files_index(remote, version)["main.towns"]["files"]
|
|
275
|
+
assert data_file["rows"] == 3
|
|
276
|
+
assert (remote.root / layout.DEFAULT_DATA_DIR / data_file["path"]).stat().st_size == (
|
|
277
|
+
data_file["bytes"]
|
|
278
|
+
)
|
|
279
|
+
assert data_file["deletes"]["rows"] == 1
|
|
280
|
+
assert (remote.root / layout.DEFAULT_DATA_DIR / data_file["deletes"]["path"]).exists()
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def test_the_index_leaves_out_dropped_tables_and_marks_renames(tmp_path: Path, remote) -> None:
|
|
284
|
+
build(tmp_path / "work.sqlite", remote, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
|
|
285
|
+
build(tmp_path / "work.sqlite", remote, "CREATE TABLE lake.main.gone AS SELECT 1 AS id")
|
|
286
|
+
build(tmp_path / "work.sqlite", remote, "DROP TABLE lake.main.gone")
|
|
287
|
+
build(tmp_path / "work.sqlite", remote, "CREATE TABLE lake.main.moved AS SELECT 1 AS id")
|
|
288
|
+
build(tmp_path / "work.sqlite", remote, "ALTER TABLE lake.main.moved RENAME COLUMN id TO code")
|
|
289
|
+
version = commit(tmp_path, remote)
|
|
290
|
+
repo.push(remote, tmp_path / "local" / version.id, version)
|
|
291
|
+
|
|
292
|
+
tables = files_index(remote, version)
|
|
293
|
+
assert sorted(tables) == ["main.moved", "main.towns"]
|
|
294
|
+
assert tables["main.moved"]["renamed_columns"] is True
|
|
295
|
+
assert "renamed_columns" not in tables["main.towns"]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|