fdl-cli 0.19.0__tar.gz → 0.20.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/CHANGELOG.md +23 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/PKG-INFO +1 -1
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/reference/cli.md +3 -1
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/spec/data-files.md +1 -1
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/spec/index.md +1 -1
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/spec/layout.md +2 -1
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/spec/versions.md +45 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/spec/writing.md +2 -2
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/reference/cli.md +3 -1
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/spec/data-files.md +1 -1
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/spec/index.md +1 -1
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/spec/layout.md +2 -1
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/spec/versions.md +45 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/spec/writing.md +2 -2
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/pyproject.toml +1 -1
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/src/fdl/cli.py +20 -4
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/src/fdl/layout.py +2 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/src/fdl/objects.py +12 -2
- fdl_cli-0.20.1/src/fdl/profiles.py +296 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/src/fdl/repo.py +67 -1
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/tests/test_http.py +2 -1
- fdl_cli-0.20.1/tests/test_profiles.py +131 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/tests/test_repo.py +17 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/.github/workflows/docs.yml +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/.github/workflows/publish.yml +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/.github/workflows/release.yml +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/.gitignore +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/.node-version +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/.python-version +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/CLAUDE.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/CONTRIBUTING.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/LICENSE +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/README.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/blume.config.ts +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/concepts/meta.ts +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/concepts/repositories.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/concepts/versions-and-refs.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/concepts/working-catalogs.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/getting-started.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/guides/building-a-host.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/guides/meta.ts +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/guides/mirrors.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/guides/public-and-private.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/guides/reading-without-duckdb.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/guides/serving-over-http.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/guides/stores.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/index.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/internals/from-write-to-read.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/internals/inside-a-repository.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/internals/meta.ts +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/concepts/meta.ts +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/concepts/repositories.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/concepts/versions-and-refs.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/concepts/working-catalogs.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/getting-started.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/guides/building-a-host.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/guides/meta.ts +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/guides/mirrors.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/guides/public-and-private.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/guides/reading-without-duckdb.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/guides/serving-over-http.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/guides/stores.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/index.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/internals/from-write-to-read.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/internals/inside-a-repository.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/internals/meta.ts +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/reference/meta.ts +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/reference/python.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/spec/http.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/spec/meta.ts +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/spec/reading.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/ja/spec/refs.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/reference/meta.ts +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/reference/python.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/spec/http.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/spec/meta.ts +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/spec/reading.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/docs/spec/refs.md +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/package-lock.json +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/package.json +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/public/CNAME +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/src/fdl/__init__.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/src/fdl/catalog.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/src/fdl/dataset.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/src/fdl/py.typed +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/src/fdl/refs.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/src/fdl/store.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/src/fdl/stores.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/src/fdl/version.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/tests/http_repo.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/tests/test_cli.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/tests/test_dataset.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/tests/test_objects.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/tests/test_stores.py +0 -0
- {fdl_cli-0.19.0 → fdl_cli-0.20.1}/uv.lock +0 -0
|
@@ -1,6 +1,29 @@
|
|
|
1
1
|
# CHANGELOG
|
|
2
2
|
|
|
3
3
|
|
|
4
|
+
## v0.20.1 (2026-11-06)
|
|
5
|
+
|
|
6
|
+
### Bug Fixes
|
|
7
|
+
|
|
8
|
+
- Withdraw a version already past retention by deleting unread files at once
|
|
9
|
+
([#40](https://github.com/queria-io/fdl/pull/40),
|
|
10
|
+
[`d9e3788`](https://github.com/queria-io/fdl/commit/d9e3788b6ffb429a9d301491a5e2a73eca8d891f))
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
## v0.20.0 (2026-11-06)
|
|
14
|
+
|
|
15
|
+
### Documentation
|
|
16
|
+
|
|
17
|
+
- Drop the note that the draft spec changes as tools are built
|
|
18
|
+
([#36](https://github.com/queria-io/fdl/pull/36),
|
|
19
|
+
[`de531b8`](https://github.com/queria-io/fdl/commit/de531b806a3b1d65fe4e5347ce203d22573981d6))
|
|
20
|
+
|
|
21
|
+
### Features
|
|
22
|
+
|
|
23
|
+
- Write a profile of each table beside a version ([#39](https://github.com/queria-io/fdl/pull/39),
|
|
24
|
+
[`bdabd3f`](https://github.com/queria-io/fdl/commit/bdabd3fcd3f0b777609de78691b8ad64a1c40c26))
|
|
25
|
+
|
|
26
|
+
|
|
4
27
|
## v0.19.0 (2026-11-06)
|
|
5
28
|
|
|
6
29
|
### Features
|
|
@@ -53,6 +53,8 @@ fdl commit <working.sqlite> -m "What changed" --author alice [--dataset dataset.
|
|
|
53
53
|
|
|
54
54
|
コマンドは、説明の隣の `README.md` と、そこにある `docs/` より下のファイル(隠しファイルを除く)をバージョンの tree に入れます。そのフォルダからのパスで `tree.json` を書き、リポジトリに無いものを書いたままの中身で `assets/<sha256>` にアップロードします([tree.json](/ja/spec/versions#treejson))。
|
|
55
55
|
|
|
56
|
+
あわせてテーブルごとの profile を書きます([profiles/](/ja/spec/versions#profiles))。読むのは、データファイルか列が親から変わったテーブルだけです。読めなかったテーブルには profile を書かず、警告を出してコミットを続けます。
|
|
57
|
+
|
|
56
58
|
`asset: true` の印を付けた列は、ファイルの sha256 を持ちます([アセット](/ja/spec/versions#アセット))。コマンドはその列の値を読み、リポジトリに無いファイルを `--assets` のフォルダから `assets/<sha256>` にアップロードします。ファイルは中身で照合するので、フォルダの中のファイル名は問いません。
|
|
57
59
|
|
|
58
60
|
親と同じ中身になるとき([変更がないとき](/ja/spec/writing#変更がないとき))は `Nothing to commit` と表示し、ブランチを動かさずに終了コード 0 で終わります。同じ行を新しいファイルに書き直しただけの作り直しも、変更なしとみなします。`--allow-empty` を付けると、それでもバージョンを作ります。行を読めず判定できないときは、警告を出してバージョンを作ります。
|
|
@@ -106,7 +108,7 @@ fdl tag <remote> <name> --delete
|
|
|
106
108
|
fdl withdraw <remote> <version>... [--yes]
|
|
107
109
|
```
|
|
108
110
|
|
|
109
|
-
間違って公開した個人データなど、公開してはいけないデータを持つバージョンを取り下げます([取り下げ](/ja/spec/data-files#取り下げ))。先にそのデータを除いたバージョンをコミットし、該当するバージョンを指すブランチが無い状態にします。そのうえでこのコマンドが、該当するバージョンを指すタグ、そのカタログが参照し残すバージョンが読まないデータファイル、カタログを消し、`"reason": "withdrawn"` の `expired.json` を置きます。`version.json`
|
|
111
|
+
間違って公開した個人データなど、公開してはいけないデータを持つバージョンを取り下げます([取り下げ](/ja/spec/data-files#取り下げ))。先にそのデータを除いたバージョンをコミットし、該当するバージョンを指すブランチが無い状態にします。そのうえでこのコマンドが、該当するバージョンを指すタグ、そのカタログが参照し残すバージョンが読まないデータファイル、カタログを消し、`"reason": "withdrawn"` の `expired.json` を置きます。`version.json` は残します。保持期間を過ぎたバージョンを含めると、そのカタログはもう無いので、有効なバージョンのどれも読まないデータファイルを、猶予期間を待たずにすべて消します。
|
|
110
112
|
|
|
111
113
|
アセットは、まだ消しません。データを持つものは `assets/` から手で消してください。
|
|
112
114
|
|
|
@@ -55,7 +55,7 @@ DuckLake 自身の片付け(`ducklake_cleanup_old_files`、`ducklake_expire_sn
|
|
|
55
55
|
|
|
56
56
|
1. そのデータを除いた新しいバージョンを作り、該当するバージョンを指しているブランチをすべてそこへ動かす。新しいバージョンの親はいまのバージョンなので、前へ動かすことになる
|
|
57
57
|
2. 該当するバージョンを指すタグを消す
|
|
58
|
-
3.
|
|
58
|
+
3. 個人情報を持つデータファイルや、それが写ったスクリーンショットなど、そのデータを持つ共有オブジェクトをすぐに消す。該当するバージョンから到達し、残すバージョンのどれからも到達しないものが対象。そのオブジェクトに到達するバージョンは、すべて該当するバージョンになる。該当するバージョンに保持期間を過ぎたものがあると、カタログが無いので、それがどのオブジェクトに到達していたかが分からない。その場合は、有効なバージョンのどれからも到達しない共有オブジェクトもすべてすぐに消す
|
|
59
59
|
4. 該当するバージョンそれぞれに `"reason": "withdrawn"` の `expired.json` を置き、カタログを消す
|
|
60
60
|
|
|
61
61
|
共有オブジェクトをカタログより先に消すのは、途中で止まった取り下げをやり直せるようにするためです。カタログが残っていれば、どのファイルがそのデータを持つかが分かります。
|
|
@@ -5,7 +5,7 @@ sidebar:
|
|
|
5
5
|
description: FDL 0.1(草案)の規範となる説明。何を決め、何をホストに任せ、どんな用語を使うか。
|
|
6
6
|
---
|
|
7
7
|
|
|
8
|
-
FDL 0.1
|
|
8
|
+
FDL 0.1 の仕様の草案です。ツールはこの仕様に合わせて作り、ツールと仕様が食い違ったときは、先に仕様を直してからツールを合わせます。この日本語訳は参考で、英語版を正とします。
|
|
9
9
|
|
|
10
10
|
FDL は、[DuckLake](https://ducklake.select/) のカタログのバージョンを残すための、置き場のファイル配置です。Git がコミットを保存する形にならっています。どのファイルを置き場のどこに置き、読み手と書き手がそれをどう扱うかを決めます。サーバーは要りません。置き場に求めるのは、読む・書く・条件付きで書くの 3 つです。
|
|
11
11
|
|
|
@@ -23,9 +23,10 @@ description: リポジトリのファイル、そのパス、fdl.json の中身
|
|
|
23
23
|
dataset.json バージョンの中身の説明
|
|
24
24
|
files.json カタログから作った、表ごとのデータファイルの索引(任意)
|
|
25
25
|
tree.json README など、バージョンのほかのファイルのパスの一覧(任意)
|
|
26
|
+
profiles/<schema>/<table>.json データから測ったテーブルごとの要約(任意)
|
|
26
27
|
catalogs/<format>.duckdb 別の DuckLake 形式のカタログ(任意)
|
|
27
28
|
expired.json 保持期間を過ぎた印(過ぎたときだけ)
|
|
28
|
-
<others>
|
|
29
|
+
<others> ホストが足すファイル
|
|
29
30
|
<data_dir><schema>/<table>/ducklake-<uuid>.parquet データファイル(共有、変えない)
|
|
30
31
|
assets/<sha256> テーブルの行やバージョンの tree.json が指すファイル(共有、変更しない)
|
|
31
32
|
```
|
|
@@ -154,6 +154,51 @@ DuckLake を使わずに Parquet ファイルを読む読み手は、`deletes`
|
|
|
154
154
|
- 書き手は、ファイルを編集する場所(Git のリポジトリの中の YAML の隣など)に置き、書き手のツールが `tree.json` を書く。書き手が手で書くものではない
|
|
155
155
|
- `tree.json` はバージョンの一部で、そのバージョンの中では変わらない。保持期間から外すときはカタログと一緒に消す
|
|
156
156
|
|
|
157
|
+
## profiles/
|
|
158
|
+
|
|
159
|
+
profile は、`ducklake.snapshot_id` の時点の 1 つのテーブルの要約です。行数、列ごとの中身、先頭の行、図形の列を持つテーブルでは図形の範囲と見本を持ちます。読み手はこの小さなファイル 1 つで、カタログを開かずデータも読まずにテーブルを表示できます。ホストは検索の索引をこれから作れます。profile が持つのはデータから測った事実で、列が何を意味するかは書き手が `dataset.json` に書きます。`files.json` と同じくカタログとデータから作るもので、食い違うときはカタログとデータが正です。任意で、読み手は profile が無くても読めなければなりません。
|
|
160
|
+
|
|
161
|
+
テーブルごとの profile は `versions/<ID>/profiles/<schema>/<table>.json` に置きます。スキーマ名かテーブル名が空・`.`・`..` のとき、または `/` を含むときは置きません。ビューには置きません。
|
|
162
|
+
|
|
163
|
+
```json profiles/main/towns.json
|
|
164
|
+
{
|
|
165
|
+
"rows": 1741,
|
|
166
|
+
"columns": [
|
|
167
|
+
{ "name": "town_code", "type": "varchar", "null_count": 0, "distinct_count": 1741,
|
|
168
|
+
"values": [{ "value": "082201", "count": 1 }] },
|
|
169
|
+
{ "name": "prefecture", "type": "varchar", "null_count": 0, "distinct_count": 2,
|
|
170
|
+
"values": [{ "value": "茨城県", "count": 1200 }, { "value": "栃木県", "count": 541 }] },
|
|
171
|
+
{ "name": "population", "type": "int64", "null_count": 3, "distinct_count": 1650,
|
|
172
|
+
"min": 0, "max": 84211, "mean": 3120.5, "std": 4410.2, "q1": 420, "median": 1630, "q3": 3990,
|
|
173
|
+
"values": [{ "value": "0", "count": 12 }],
|
|
174
|
+
"histogram": [{ "start": 0, "end": 4210.55, "count": 1302 }] },
|
|
175
|
+
{ "name": "surveyed_on", "type": "date", "null_count": 0, "distinct_count": 1,
|
|
176
|
+
"min": "2025-10-01", "max": "2025-10-01", "span_days": 0,
|
|
177
|
+
"values": [{ "value": "2025-10-01", "count": 1741 }] },
|
|
178
|
+
{ "name": "coastal", "type": "boolean", "null_count": 0, "true_count": 210, "false_count": 1531 },
|
|
179
|
+
{ "name": "geom", "type": "geometry" }
|
|
180
|
+
],
|
|
181
|
+
"head": [{ "town_code": "082201", "prefecture": "茨城県", "population": 2210, "surveyed_on": "2025-10-01", "coastal": false }],
|
|
182
|
+
"geometry": {
|
|
183
|
+
"column": "geom",
|
|
184
|
+
"count": 1741,
|
|
185
|
+
"bbox": [139.69, 35.74, 140.85, 37.15],
|
|
186
|
+
"types": ["MULTIPOLYGON", "POLYGON"],
|
|
187
|
+
"sample": { "type": "FeatureCollection", "features": [] }
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
- `rows` はテーブルの行数。`columns` にはカタログの順にすべての列を並べ、`name` と `type`(DuckLake の型名)は `dataset.json` と同じにする
|
|
193
|
+
- 図形の列を除くすべての列が `null_count` を持つ。数値・テキスト・日付と時刻の列は `distinct_count` と `values` を持つ。大きなテーブルの `distinct_count` は推定値のことがある
|
|
194
|
+
- `values` は、値(テキスト)とその値を持つ行の数を、多い順、同数なら値の順に並べる。列が持つ異なる値が 1,000 以下なら全部を並べ、そのとき `distinct_count` は正確な数になる。読み手は、`values` の長さが `distinct_count` と等しいことで全部の一覧だと判定する。それより多い列は上位 10 件だけを並べる。ホストは全部の一覧を、地域コードや名前などの値で検索できる
|
|
195
|
+
- 数値の列は `min`・`max`・`mean`・`std`、四分位数の `q1`・`median`・`q3`、`min` から `max` までを等幅に区切った `histogram` を持つ。日付と時刻の列は `min` と `max` をテキストで持ち、日付とタイムスタンプは `span_days` も持つ。真偽値の列は `true_count` と `false_count` を持つ
|
|
196
|
+
- `head` は、テーブルが返す先頭の行を JSON のオブジェクトで持つ。図形の列と、値を JSON で表せない列は入れない
|
|
197
|
+
- `geometry` は、書き手が読めたときに、最初の図形の列を説明する。図形を持つ行の数、全体の `bbox`(列の座標で `[xmin, ymin, xmax, ymax]`)、含まれる図形の種類、地図を描くための、無作為に選んで簡略化した図形の GeoJSON `FeatureCollection` を持つ
|
|
198
|
+
- 書き手は計算できないキーを省いてよい。読み手は知らないキーを無視する
|
|
199
|
+
- 書き手は、データファイル(`files.json`)と列が親と同じテーブルには親の profile を使い回す。コミットで読むのは変わったテーブルだけになる
|
|
200
|
+
- profile はバージョンの一部で、そのバージョンの中では変わらない。保持期間から外すときはカタログと一緒に消す
|
|
201
|
+
|
|
157
202
|
## 固めたカタログ
|
|
158
203
|
|
|
159
204
|
`ducklake.duckdb` は、DuckDB のファイル形式の DuckLake カタログです。
|
|
@@ -17,7 +17,7 @@ description: チェックアウト、コミット、push。作業カタログ、
|
|
|
17
17
|
| --- | --- |
|
|
18
18
|
| チェックアウト | ブランチのバージョンのカタログから作業カタログを作り、そのバージョンを覚える。最初のバージョンの前は空のものを作る |
|
|
19
19
|
| コミット | 作業カタログの最後のスナップショットを新しいバージョンに固める。親は覚えておいたバージョン。中身を `dataset.json` で説明する |
|
|
20
|
-
| push | データファイルを確かめ、カタログ、`dataset.json`・`files.json`・`tree.json
|
|
20
|
+
| push | データファイルを確かめ、カタログ、`dataset.json`・`files.json`・`tree.json`・profile、`version.json` の順にアップロードし、条件付きで ref を動かす |
|
|
21
21
|
|
|
22
22
|
チェックアウトでは、DuckLake の管理用の表を、固めたカタログから SQLite へそのまま写します。固めるときはその逆で、最後のスナップショットまでの行を残します。どちらもデータファイルは書き換えません。
|
|
23
23
|
|
|
@@ -40,7 +40,7 @@ description: チェックアウト、コミット、push。作業カタログ、
|
|
|
40
40
|
|
|
41
41
|
1. データファイル・アセット
|
|
42
42
|
2. `versions/<ID>/ducklake.duckdb`(と `catalogs/`)
|
|
43
|
-
3. `versions/<ID>/dataset.json`・`versions/<ID>/files.json`・`versions/<ID>/tree.json
|
|
43
|
+
3. `versions/<ID>/dataset.json`・`versions/<ID>/files.json`・`versions/<ID>/tree.json`・`versions/<ID>/profiles/`
|
|
44
44
|
4. `versions/<ID>/version.json`
|
|
45
45
|
5. ref(条件付き)
|
|
46
46
|
|
|
@@ -53,6 +53,8 @@ Every version is described by a `dataset.json` (see [Versions](/spec/versions#da
|
|
|
53
53
|
|
|
54
54
|
The command puts `README.md` beside the description and every file below `docs/` there (hidden ones left out) in the version's tree: it writes `tree.json` with their paths from that folder and uploads those the repository lacks to `assets/<sha256>`, as written (see [tree.json](/spec/versions#treejson)).
|
|
55
55
|
|
|
56
|
+
It also writes a profile of each table (see [profiles/](/spec/versions#profiles)), reading only the tables whose data files or columns changed since the parent. A table it cannot read gets none, and the commit goes on with a warning.
|
|
57
|
+
|
|
56
58
|
Columns marked `asset: true` hold the sha256 of files (see [Assets](/spec/versions#assets)). The command reads the values those columns hold and uploads the files the repository lacks from the `--assets` folder to `assets/<sha256>`, by their content: file names in the folder do not matter.
|
|
57
59
|
|
|
58
60
|
When the version would hold what its parent holds (see [When nothing changed](/spec/writing#when-nothing-changed)), it prints `Nothing to commit`, leaves the branch where it is and exits 0. A rebuild that writes the same rows to new files counts as unchanged. `--allow-empty` makes the version anyway. If the rows cannot be read to tell, it warns and makes the version.
|
|
@@ -106,7 +108,7 @@ fdl tag <remote> <name> --delete
|
|
|
106
108
|
fdl withdraw <remote> <version>... [--yes]
|
|
107
109
|
```
|
|
108
110
|
|
|
109
|
-
Removes versions that hold data that must not be published, such as personal data published by mistake (see [Withdrawal](/spec/data-files#withdrawal)). First commit a version without the data, so that no branch points at an affected version. The command then deletes the tags that point at the versions, the data files their catalogs name that no kept version reads, and their catalogs, and puts `expired.json` with `"reason": "withdrawn"`. `version.json` stays.
|
|
111
|
+
Removes versions that hold data that must not be published, such as personal data published by mistake (see [Withdrawal](/spec/data-files#withdrawal)). First commit a version without the data, so that no branch points at an affected version. The command then deletes the tags that point at the versions, the data files their catalogs name that no kept version reads, and their catalogs, and puts `expired.json` with `"reason": "withdrawn"`. `version.json` stays. When one of the versions is already past retention, its catalog is gone, so every data file no live version reads is deleted at once, without waiting for the grace period.
|
|
110
112
|
|
|
111
113
|
Assets are not deleted yet: delete those that hold the data from `assets/` by hand.
|
|
112
114
|
|
|
@@ -55,7 +55,7 @@ Withdrawal removes data that must not be published, such as personal data publis
|
|
|
55
55
|
|
|
56
56
|
1. Make a new version without the data and move every branch that holds an affected version to it. This is a forward move: the new version's parent is the current one
|
|
57
57
|
2. Delete the tags that point at affected versions
|
|
58
|
-
3. Delete the shared objects that hold the data, at once, such as a data file with personal data or a screenshot showing it: those an affected version reaches and no version kept reaches. Every version that reaches such an object is an affected version
|
|
58
|
+
3. Delete the shared objects that hold the data, at once, such as a data file with personal data or a screenshot showing it: those an affected version reaches and no version kept reaches. Every version that reaches such an object is an affected version. An affected version already past retention has no catalog to say which objects it reached, so when there is one, every shared object no live version reaches is deleted at once too
|
|
59
59
|
4. For every affected version, put `expired.json` with `"reason": "withdrawn"` and delete its catalogs
|
|
60
60
|
|
|
61
61
|
The shared objects go before the catalogs, so a withdrawal that stops half way can be run again: the catalogs still say which files hold the data.
|
|
@@ -5,7 +5,7 @@ sidebar:
|
|
|
5
5
|
description: The normative description of FDL 0.1 (draft) - what it decides, what it leaves to hosts, and the terms it uses.
|
|
6
6
|
---
|
|
7
7
|
|
|
8
|
-
This is the specification of FDL 0.1, a draft.
|
|
8
|
+
This is the specification of FDL 0.1, a draft. Tools are written against this specification, and where a tool and this text disagree, the text is changed first and the tool follows.
|
|
9
9
|
|
|
10
10
|
FDL is a storage layout for keeping versions of a [DuckLake](https://ducklake.select/) catalog, modelled on how Git stores commits. It says which files go where in a store and how readers and writers handle them. No server is needed: the store has to read, write and write on condition.
|
|
11
11
|
|
|
@@ -23,9 +23,10 @@ Paths are relative to the root, lower case, and separated by `/`. The root is wh
|
|
|
23
23
|
dataset.json what the version holds, described
|
|
24
24
|
files.json each table's data files, indexed from the catalog (optional)
|
|
25
25
|
tree.json the other files of the version, such as a README, by path (optional)
|
|
26
|
+
profiles/<schema>/<table>.json a summary of each table, measured from the data (optional)
|
|
26
27
|
catalogs/<format>.duckdb the catalog in another DuckLake format (optional)
|
|
27
28
|
expired.json the past-retention mark (only once expired)
|
|
28
|
-
<others> files a host adds
|
|
29
|
+
<others> files a host adds
|
|
29
30
|
<data_dir><schema>/<table>/ducklake-<uuid>.parquet data files (shared, immutable)
|
|
30
31
|
assets/<sha256> files the rows of a table or a version's tree.json point at (shared, immutable)
|
|
31
32
|
```
|
|
@@ -154,6 +154,51 @@ When a version is taken out of retention, `files.json` is deleted with its catal
|
|
|
154
154
|
- A writer keeps the files where it edits them, such as beside its YAML in a Git repository, and its tool writes `tree.json`. The writer does not write it by hand
|
|
155
155
|
- `tree.json` is part of the version and does not change with it. It is deleted with the catalogs when the version is taken out of retention
|
|
156
156
|
|
|
157
|
+
## profiles/
|
|
158
|
+
|
|
159
|
+
A profile summarizes one table at `ducklake.snapshot_id`: its row count, what each column holds, its first rows and, for a table with a geometry column, the extent and a sample of its shapes. A reader can show a table, or a host can index it for search, from this one small file, without opening the catalog or reading the data. It holds what was measured from the data; what a column means is the writer's, in `dataset.json`. Like `files.json`, it is made from the catalog and the data, which win where the two differ, and it is optional: a reader must work without it.
|
|
160
|
+
|
|
161
|
+
Each table's profile is at `versions/<ID>/profiles/<schema>/<table>.json`. A table whose schema or name is empty, `.`, `..` or holds `/` has none. Views have none.
|
|
162
|
+
|
|
163
|
+
```json profiles/main/towns.json
|
|
164
|
+
{
|
|
165
|
+
"rows": 1741,
|
|
166
|
+
"columns": [
|
|
167
|
+
{ "name": "town_code", "type": "varchar", "null_count": 0, "distinct_count": 1741,
|
|
168
|
+
"values": [{ "value": "082201", "count": 1 }] },
|
|
169
|
+
{ "name": "prefecture", "type": "varchar", "null_count": 0, "distinct_count": 2,
|
|
170
|
+
"values": [{ "value": "Ibaraki", "count": 1200 }, { "value": "Tochigi", "count": 541 }] },
|
|
171
|
+
{ "name": "population", "type": "int64", "null_count": 3, "distinct_count": 1650,
|
|
172
|
+
"min": 0, "max": 84211, "mean": 3120.5, "std": 4410.2, "q1": 420, "median": 1630, "q3": 3990,
|
|
173
|
+
"values": [{ "value": "0", "count": 12 }],
|
|
174
|
+
"histogram": [{ "start": 0, "end": 4210.55, "count": 1302 }] },
|
|
175
|
+
{ "name": "surveyed_on", "type": "date", "null_count": 0, "distinct_count": 1,
|
|
176
|
+
"min": "2025-10-01", "max": "2025-10-01", "span_days": 0,
|
|
177
|
+
"values": [{ "value": "2025-10-01", "count": 1741 }] },
|
|
178
|
+
{ "name": "coastal", "type": "boolean", "null_count": 0, "true_count": 210, "false_count": 1531 },
|
|
179
|
+
{ "name": "geom", "type": "geometry" }
|
|
180
|
+
],
|
|
181
|
+
"head": [{ "town_code": "082201", "prefecture": "Ibaraki", "population": 2210, "surveyed_on": "2025-10-01", "coastal": false }],
|
|
182
|
+
"geometry": {
|
|
183
|
+
"column": "geom",
|
|
184
|
+
"count": 1741,
|
|
185
|
+
"bbox": [139.69, 35.74, 140.85, 37.15],
|
|
186
|
+
"types": ["MULTIPOLYGON", "POLYGON"],
|
|
187
|
+
"sample": { "type": "FeatureCollection", "features": [] }
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
- `rows` is the table's row count. `columns` lists every column in the catalog's order, with `name` and `type` (DuckLake's type name) as in `dataset.json`
|
|
193
|
+
- Every column but a geometry one has `null_count`. Numbers, text, dates and times have `distinct_count`, which may be estimated on a large table, and `values`
|
|
194
|
+
- `values` lists each value, as text, with the number of rows that hold it, most common first and then by value. When the column holds at most 1,000 distinct values, it lists them all and `distinct_count` is exact: a reader tells a whole list by its length matching `distinct_count`. Otherwise it lists the 10 most common. A host can search a whole list by value, such as a town's code or name
|
|
195
|
+
- Numbers have `min`, `max`, `mean`, `std`, the quartiles `q1`, `median` and `q3`, and a `histogram` of equal-width buckets between `min` and `max`. Dates and times have `min` and `max` as text, and dates and timestamps `span_days`. Booleans have `true_count` and `false_count`
|
|
196
|
+
- `head` holds the first rows the table returns, as JSON objects, leaving out geometry and columns whose values have no JSON form
|
|
197
|
+
- `geometry` describes the first geometry column, when the writer could read it: how many rows have a shape, the `bbox` of all of them (`[xmin, ymin, xmax, ymax]` in the column's coordinates), the geometry types present, and a GeoJSON `FeatureCollection` of a random sample of simplified shapes for drawing a map
|
|
198
|
+
- A writer may leave out any key it cannot compute. A reader ignores keys it does not know
|
|
199
|
+
- A writer reuses its parent's profile for a table whose data files (in `files.json`) and columns are the parent's, so a commit reads only the tables that changed
|
|
200
|
+
- Profiles are part of the version and do not change with it. They are deleted with the catalogs when the version is taken out of retention
|
|
201
|
+
|
|
157
202
|
## The frozen catalog
|
|
158
203
|
|
|
159
204
|
`ducklake.duckdb` is a DuckLake catalog in DuckDB's file format.
|
|
@@ -17,7 +17,7 @@ A writer writes to a working catalog: a DuckLake catalog in SQLite, outside the
|
|
|
17
17
|
| --- | --- |
|
|
18
18
|
| checkout | Make a working catalog from the catalog of the branch's version, and remember that version. Before the first version, make an empty one |
|
|
19
19
|
| commit | Freeze the working catalog's last snapshot into a new version, with the remembered version as its parent, and describe it in `dataset.json` |
|
|
20
|
-
| push | Check the data files, upload the catalog, `dataset.json`, `files.json
|
|
20
|
+
| push | Check the data files, upload the catalog, `dataset.json`, `files.json`, `tree.json` and the profiles, then `version.json`, then move the ref on condition |
|
|
21
21
|
|
|
22
22
|
Checking out copies DuckLake's bookkeeping tables from the frozen catalog into SQLite as they are. Freezing does the reverse, keeping the rows up to the last snapshot. Neither rewrites a data file.
|
|
23
23
|
|
|
@@ -40,7 +40,7 @@ A table rebuilt from the same input is written to new data files, so comparing f
|
|
|
40
40
|
|
|
41
41
|
1. Data files and assets
|
|
42
42
|
2. `versions/<ID>/ducklake.duckdb` (and any `catalogs/`)
|
|
43
|
-
3. `versions/<ID>/dataset.json`, `versions/<ID>/files.json
|
|
43
|
+
3. `versions/<ID>/dataset.json`, `versions/<ID>/files.json`, `versions/<ID>/tree.json` and `versions/<ID>/profiles/`
|
|
44
44
|
4. `versions/<ID>/version.json`
|
|
45
45
|
5. The ref, on condition
|
|
46
46
|
|
|
@@ -148,12 +148,16 @@ def commit(
|
|
|
148
148
|
typer.echo(f"warning: could not tell whether anything changed ({exc}); committing", err=True)
|
|
149
149
|
version = make(True)
|
|
150
150
|
local = Path(state["data"]) if state.get("data") else None
|
|
151
|
+
reading = Path(state["data"]).resolve().as_posix() + "/" if local else repo.data_url(store)
|
|
152
|
+
setup = _read_setup(store)
|
|
151
153
|
described = json.loads((out / version.id / layout.DATASET).read_text(encoding="utf-8"))
|
|
152
154
|
held = objects.asset_values(
|
|
153
|
-
working,
|
|
154
|
-
objects.asset_columns(described),
|
|
155
|
-
data_path=Path(state["data"]).resolve().as_posix() + "/" if local else repo.data_url(store),
|
|
155
|
+
working, objects.asset_columns(described), data_path=reading, setup=setup
|
|
156
156
|
)
|
|
157
|
+
try:
|
|
158
|
+
repo.write_profiles(store, out / version.id, parent, working, data_path=reading, setup=setup)
|
|
159
|
+
except Exception as exc: # noqa: BLE001 - a profile is never worth a failed commit
|
|
160
|
+
typer.echo(f"warning: tables were not profiled ({exc})", err=True)
|
|
157
161
|
local_objects = objects.ByContent(
|
|
158
162
|
{item["sha256"]: root / item["path"] for item in listed},
|
|
159
163
|
assets_dir,
|
|
@@ -180,6 +184,16 @@ def commit(
|
|
|
180
184
|
typer.echo(f"{state['ref']} -> {version.id} (version {version.seq})")
|
|
181
185
|
|
|
182
186
|
|
|
187
|
+
def _read_setup(store) -> list[str]:
|
|
188
|
+
"""What DuckDB needs to read the repository's data files: the token, over HTTP."""
|
|
189
|
+
token = getattr(store, "token", None)
|
|
190
|
+
if not token:
|
|
191
|
+
return []
|
|
192
|
+
scope = store.url.replace("'", "''")
|
|
193
|
+
secret = token.replace("'", "''")
|
|
194
|
+
return [f"CREATE SECRET fdl_remote (TYPE http, BEARER_TOKEN '{secret}', SCOPE '{scope}')"]
|
|
195
|
+
|
|
196
|
+
|
|
183
197
|
@app.command()
|
|
184
198
|
def assets(folder: Path = typer.Argument(..., help="The folder of files the rows will point at")) -> None:
|
|
185
199
|
"""List the files below a folder as CSV: path, sha256, bytes, media_type.
|
|
@@ -281,7 +295,9 @@ def withdraw(
|
|
|
281
295
|
|
|
282
296
|
Move every branch to a version without the data first. This deletes the
|
|
283
297
|
tags that point at the versions, the data files no kept version reads,
|
|
284
|
-
and the versions' catalogs, at once. Their version.json stays.
|
|
298
|
+
and the versions' catalogs, at once. Their version.json stays. A version
|
|
299
|
+
already past retention has no catalog, so then every data file no live
|
|
300
|
+
version reads is deleted.
|
|
285
301
|
"""
|
|
286
302
|
store = open_store(remote)
|
|
287
303
|
try:
|
|
@@ -29,6 +29,8 @@ EXPIRED = "expired.json"
|
|
|
29
29
|
CATALOGS_DIR = "catalogs/"
|
|
30
30
|
#: The other files of a version (its README, say), by path to the asset that holds each (optional).
|
|
31
31
|
TREE = "tree.json"
|
|
32
|
+
#: A summary of each table, made from the data, under the version (optional).
|
|
33
|
+
PROFILES_DIR = "profiles/"
|
|
32
34
|
#: Files the rows of a table point at, by sha256 (shared).
|
|
33
35
|
ASSETS_DIR = "assets/"
|
|
34
36
|
|
|
@@ -174,9 +174,17 @@ def asset_columns(described: dict[str, Any]) -> list[tuple[str, str, str]]:
|
|
|
174
174
|
|
|
175
175
|
|
|
176
176
|
def asset_values(
|
|
177
|
-
working: Path,
|
|
177
|
+
working: Path,
|
|
178
|
+
columns: Iterable[tuple[str, str, str]],
|
|
179
|
+
*,
|
|
180
|
+
data_path: str,
|
|
181
|
+
setup: Iterable[str] = (),
|
|
178
182
|
) -> set[str]:
|
|
179
|
-
"""The sha256 the marked columns hold in the working catalog's last snapshot.
|
|
183
|
+
"""The sha256 the marked columns hold in the working catalog's last snapshot.
|
|
184
|
+
|
|
185
|
+
``setup`` runs before the catalog is attached: a ``CREATE SECRET`` for data
|
|
186
|
+
files in private storage, say.
|
|
187
|
+
"""
|
|
180
188
|
import duckdb
|
|
181
189
|
|
|
182
190
|
columns = list(columns)
|
|
@@ -189,6 +197,8 @@ def asset_values(
|
|
|
189
197
|
found: set[str] = set()
|
|
190
198
|
con = duckdb.connect()
|
|
191
199
|
try:
|
|
200
|
+
for statement in setup:
|
|
201
|
+
con.execute(statement)
|
|
192
202
|
for statement in catalog.attach_sql(Path(working), alias="lake", data_path=data_path, read_only=True):
|
|
193
203
|
if statement:
|
|
194
204
|
con.execute(statement)
|