fdl-cli 0.25.0__tar.gz → 0.26.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/CHANGELOG.md +48 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/PKG-INFO +1 -1
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/reference/cli.md +2 -2
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/spec/versions.md +3 -3
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/reference/cli.md +2 -2
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/spec/versions.md +3 -3
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/pyproject.toml +1 -1
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/src/fdl/cli.py +4 -3
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/src/fdl/profiles.py +28 -9
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/src/fdl/repo.py +31 -8
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/tests/test_profiles.py +65 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/.github/workflows/docs.yml +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/.github/workflows/publish.yml +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/.github/workflows/release.yml +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/.github/workflows/test.yml +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/.gitignore +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/.node-version +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/.python-version +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/CLAUDE.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/CONTRIBUTING.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/LICENSE +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/README.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/SECURITY.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/blume.config.ts +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/cases/collection.json +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/concepts/meta.ts +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/concepts/repositories.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/concepts/versions-and-refs.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/concepts/working-catalogs.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/getting-started.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/guides/building-a-host.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/guides/meta.ts +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/guides/mirrors.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/guides/public-and-private.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/guides/reading-without-duckdb.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/guides/serving-over-http.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/guides/stores.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/index.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/internals/from-write-to-read.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/internals/inside-a-repository.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/internals/meta.ts +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/concepts/meta.ts +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/concepts/repositories.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/concepts/versions-and-refs.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/concepts/working-catalogs.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/getting-started.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/guides/building-a-host.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/guides/meta.ts +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/guides/mirrors.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/guides/public-and-private.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/guides/reading-without-duckdb.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/guides/serving-over-http.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/guides/stores.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/index.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/internals/from-write-to-read.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/internals/inside-a-repository.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/internals/meta.ts +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/reference/meta.ts +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/reference/python.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/spec/collection-cases.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/spec/data-files.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/spec/http.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/spec/index.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/spec/layout.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/spec/meta.ts +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/spec/reading.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/spec/refs.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/spec/writing.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/ja/stability.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/reference/meta.ts +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/reference/python.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/spec/collection-cases.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/spec/data-files.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/spec/http.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/spec/index.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/spec/layout.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/spec/meta.ts +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/spec/reading.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/spec/refs.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/spec/writing.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/docs/stability.md +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/package-lock.json +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/package.json +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/public/CNAME +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/src/fdl/__init__.py +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/src/fdl/catalog.py +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/src/fdl/dataset.py +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/src/fdl/diff.py +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/src/fdl/layout.py +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/src/fdl/lineage.py +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/src/fdl/merge.py +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/src/fdl/objects.py +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/src/fdl/py.typed +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/src/fdl/refs.py +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/src/fdl/store.py +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/src/fdl/stores.py +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/src/fdl/version.py +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/tests/http_repo.py +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/tests/test_cli.py +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/tests/test_collection_cases.py +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/tests/test_dataset.py +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/tests/test_http.py +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/tests/test_merge.py +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/tests/test_objects.py +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/tests/test_repo.py +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/tests/test_stores.py +0 -0
- {fdl_cli-0.25.0 → fdl_cli-0.26.0}/uv.lock +0 -0
|
@@ -1,6 +1,54 @@
|
|
|
1
1
|
# CHANGELOG
|
|
2
2
|
|
|
3
3
|
|
|
4
|
+
## v0.26.0 (2026-11-14)
|
|
5
|
+
|
|
6
|
+
### Features
|
|
7
|
+
|
|
8
|
+
- Profile views, and measure wide tables a hundred columns at a time
|
|
9
|
+
([#60](https://github.com/queria-io/fdl/pull/60),
|
|
10
|
+
[`e6c3442`](https://github.com/queria-io/fdl/commit/e6c34429590eedfcee7b568f6c030b497c5ef668))
|
|
11
|
+
|
|
12
|
+
* feat!: profile tables with fdl profile instead of at commit
|
|
13
|
+
|
|
14
|
+
fdl commit no longer writes profiles. fdl profile <working> profiles the version committed there,
|
|
15
|
+
for fdl push to upload with it; fdl profile <remote> adds the profiles a pushed version lacks. The
|
|
16
|
+
spec lets a version get a missing profile later.
|
|
17
|
+
|
|
18
|
+
BREAKING CHANGE: fdl commit writes no profiles. Run fdl profile between fdl commit and fdl push to
|
|
19
|
+
keep writing them.
|
|
20
|
+
|
|
21
|
+
* feat: profile tables at push instead of commit
|
|
22
|
+
|
|
23
|
+
fdl push profiles the version it uploads (--no-profile skips it), so the steps stay commit and push.
|
|
24
|
+
fdl profile <remote> is left for versions pushed without profiles. Also warns of tables that could
|
|
25
|
+
not be read, lists the version's profiles in one request, and updates the spec wording.
|
|
26
|
+
|
|
27
|
+
* fix: keep fdl push working when profiling cannot read the working catalog
|
|
28
|
+
|
|
29
|
+
Read dataset.json and the working catalog inside the guarded block, so a missing or unreadable
|
|
30
|
+
catalog only warns. Profile again when a stopped push left some tables without profiles.
|
|
31
|
+
|
|
32
|
+
* feat: profile views, and measure wide tables a hundred columns at a time
|
|
33
|
+
|
|
34
|
+
A view's profile is measured from the rows its query returns, with the columns DESCRIBE gives. It
|
|
35
|
+
keeps the parent's profile only while its sql is the parent's and no table's data files changed.
|
|
36
|
+
|
|
37
|
+
The column statistics now run in queries of at most 100 columns. One query over hundreds of
|
|
38
|
+
count(DISTINCT) ran out of memory, as DuckDB holds a hash table for each at once.
|
|
39
|
+
|
|
40
|
+
* fix: re-profile a view when any view's query changed
|
|
41
|
+
|
|
42
|
+
A view may read another view, so a view keeps its parent's profile only when every view's sql and
|
|
43
|
+
every table's data files are the parent's. Fetch the parent's dataset.json only when there are
|
|
44
|
+
views, list more of DuckDB's temporal types, and describe view profiling in push --help.
|
|
45
|
+
|
|
46
|
+
### Breaking Changes
|
|
47
|
+
|
|
48
|
+
- Fdl commit writes no profiles. Run fdl profile between fdl commit and fdl push to keep writing
|
|
49
|
+
them.
|
|
50
|
+
|
|
51
|
+
|
|
4
52
|
## v0.25.0 (2026-11-14)
|
|
5
53
|
|
|
6
54
|
### Bug Fixes
|
|
@@ -76,7 +76,7 @@ fdl push <working.sqlite> [--no-profile]
|
|
|
76
76
|
|
|
77
77
|
`fdl commit` が作ったバージョンをアップロードし、条件付きでブランチをそこへ動かします([書き込みの順番](/ja/spec/writing#書き込みの順番))。ブランチ、バージョン ID、`seq` を表示します。fast-forward の `fdl merge` のあとは、何もアップロードせず、ブランチをマージしたバージョンへ動かします。コミットしたものが無いときは `Nothing to push` と表示します。
|
|
78
78
|
|
|
79
|
-
|
|
79
|
+
アップロードの前に、テーブルとビューごとの profile([profiles/](/ja/spec/versions#profiles))を書き、`version.json` より前にアップロードします。ブランチが動く時点で profile がリポジトリにあります。読むのは、データファイルか列が親から変わったテーブルだけで、ほかのテーブルは親の profile を引き継ぎます。ビューは、どれかのテーブルのデータファイルかビューのクエリが変わると、クエリを実行して測り直します。読めなかったテーブルには profile を書かず、警告を出します。コミットのあとに作業カタログが変わっていたときは profile を書かずに警告します。`--no-profile` を付けると profile を書きません。`fdl profile` であとから足せます。
|
|
80
80
|
|
|
81
81
|
作業カタログがリポジトリに書いたデータファイルは push の前からありますが、push するまでどのバージョンも参照しません。push しなかったファイルは猶予期間のあとに削除されます。
|
|
82
82
|
|
|
@@ -226,7 +226,7 @@ fdl withdraw <remote> <version>... [--yes]
|
|
|
226
226
|
fdl profile <remote> [--ref refs/heads/main]
|
|
227
227
|
```
|
|
228
228
|
|
|
229
|
-
|
|
229
|
+
ブランチかタグが指すバージョンのテーブルとビューのうち、[profile](/ja/spec/versions#profiles) が無いものを測り、profile をバージョンの横に置きます。`fdl push` はアップロードするバージョンの profile を作ります。このコマンドは profile 無しで push したバージョンに使います。fdl が profile を書く前に作ったバージョン、`--no-profile` で push したバージョン、そのときテーブルを読めなかったバージョンです。既にある profile は変えません。保持期間を過ぎたバージョンはカタログが無いので断られます。
|
|
230
230
|
|
|
231
231
|
## fdl expire
|
|
232
232
|
|
|
@@ -208,7 +208,7 @@ DuckLake を使わずに Parquet ファイルを読む読み手は、`deletes`
|
|
|
208
208
|
|
|
209
209
|
profile は、`ducklake.snapshot_id` の時点の 1 つのテーブルの要約です。行数、列ごとの中身、先頭の行、図形の列を持つテーブルでは図形の範囲と見本を持ちます。読み手はこの小さなファイル 1 つで、カタログを開かずデータも読まずにテーブルを表示できます。ホストは検索の索引をこれから作れます。profile が持つのはデータから測った事実で、列が何を意味するかは書き手が `dataset.json` に書きます。`files.json` と同じくカタログとデータから作るもので、食い違うときはカタログとデータが正です。任意で、読み手は profile が無くても読めなければなりません。
|
|
210
210
|
|
|
211
|
-
テーブルごとの profile は `versions/<ID>/profiles/<schema>/<table>.json`
|
|
211
|
+
テーブルごとの profile は `versions/<ID>/profiles/<schema>/<table>.json` に置きます。ビューにも置き、クエリが返す行から測ります。スキーマ名かテーブル名が空・`.`・`..` のとき、または `/` を含むときは置きません。profile を作らなかった書き手や作るのに失敗した書き手のバージョンには、あとから profile を足せます。ほかの profile と同じく、そのバージョンのカタログとデータから測ります。
|
|
212
212
|
|
|
213
213
|
```json profiles/main/towns.json
|
|
214
214
|
{
|
|
@@ -239,14 +239,14 @@ profile は、`ducklake.snapshot_id` の時点の 1 つのテーブルの要約
|
|
|
239
239
|
}
|
|
240
240
|
```
|
|
241
241
|
|
|
242
|
-
- `rows` はテーブルの行数。`columns` にはカタログの順にすべての列を並べ、`name` と `type`(DuckLake の型名)は `dataset.json`
|
|
242
|
+
- `rows` はテーブルの行数。`columns` にはカタログの順にすべての列を並べ、`name` と `type`(DuckLake の型名)は `dataset.json` と同じにする。`dataset.json` に型の無いビューでは、クエリが返す列を DuckDB の型名で並べる
|
|
243
243
|
- 図形の列を除くすべての列が `null_count` を持つ。数値・テキスト・日付と時刻の列は `distinct_count` と `values` を持つ。大きなテーブルの `distinct_count` は推定値のことがある
|
|
244
244
|
- `values` は、値(テキスト)とその値を持つ行の数を、多い順、同数なら値の順に並べる。列が持つ異なる値が 2,000 以下なら全部を並べ、そのとき `distinct_count` は正確な数になる。読み手は、`values` の長さが `distinct_count` と等しいことで全部の一覧だと判定する。それより多い列は上位 10 件だけを並べる。ホストは全部の一覧を、地域コードや名前などの値で検索できる
|
|
245
245
|
- 数値の列は `min`・`max`・`mean`・`std`、四分位数の `q1`・`median`・`q3`、`min` から `max` までを等幅に区切った `histogram` を持つ。日付と時刻の列は `min` と `max` をテキストで持ち、日付とタイムスタンプは `span_days` も持つ。真偽値の列は `true_count` と `false_count` を持つ
|
|
246
246
|
- `head` は、テーブルが返す先頭の行を JSON のオブジェクトで持つ。図形の列と、値を JSON で表せない列は入れない
|
|
247
247
|
- `geometry` は、書き手が読めたときに、最初の図形の列を説明する。図形を持つ行の数、全体の `bbox`(列の座標で `[xmin, ymin, xmax, ymax]`)、含まれる図形の種類、地図を描くための、無作為に選んで簡略化した図形の GeoJSON `FeatureCollection` を持つ
|
|
248
248
|
- 書き手は計算できないキーを省いてよい。読み手は知らないキーを無視する
|
|
249
|
-
- 書き手は、データファイル(`files.json`)と列が親と同じテーブルには親の profile を使い回す。profile
|
|
249
|
+
- 書き手は、データファイル(`files.json`)と列が親と同じテーブルには親の profile を使い回す。profile を作るときに読むのは変わったテーブルだけになる。ビューは別のビューを読むことがあるので、すべてのビューの `sql` が親と同じで、どのテーブルのデータファイルも変わっていないときに親の profile を使い回す。親の profile の `values` が書き手と異なる上限で書かれているときは、profile を作り直す
|
|
250
250
|
- profile はバージョンの一部で、そのバージョンの中では変わらない。保持期間から外すときはカタログと一緒に消す
|
|
251
251
|
|
|
252
252
|
## 固めたカタログ
|
|
@@ -75,7 +75,7 @@ fdl push <working.sqlite> [--no-profile]
|
|
|
75
75
|
|
|
76
76
|
Uploads the version `fdl commit` made and moves the branch to it on condition (see [Order of writes](/spec/writing#order-of-writes)). It prints the branch, the version ID and its `seq`. After a fast-forward `fdl merge`, it moves the branch to the version merged, uploading nothing. With nothing committed, it prints `Nothing to push`.
|
|
77
77
|
|
|
78
|
-
Before uploading, it writes a profile of each table (see [profiles/](/spec/versions#profiles)) and uploads the profiles before `version.json`, so they are there when the branch moves. It reads only the tables whose data files or columns changed since the parent; the others keep the parent's profile. A table it cannot read gets none, with a warning. When the working catalog changed after the commit, it profiles nothing and warns. `--no-profile` skips profiling; `fdl profile` adds the profiles later.
|
|
78
|
+
Before uploading, it writes a profile of each table and view (see [profiles/](/spec/versions#profiles)) and uploads the profiles before `version.json`, so they are there when the branch moves. It reads only the tables whose data files or columns changed since the parent; the others keep the parent's profile. A view is profiled again, by running its query, whenever a table's data files or a view's query changed. A table it cannot read gets none, with a warning. When the working catalog changed after the commit, it profiles nothing and warns. `--no-profile` skips profiling; `fdl profile` adds the profiles later.
|
|
79
79
|
|
|
80
80
|
Data files the working catalog wrote into the repository are there before the push, but no version names them until it. Files never pushed are deleted after the grace period.
|
|
81
81
|
|
|
@@ -226,7 +226,7 @@ Without `--yes` it only prints what it would delete. Files a kept version still
|
|
|
226
226
|
fdl profile <remote> [--ref refs/heads/main]
|
|
227
227
|
```
|
|
228
228
|
|
|
229
|
-
Profiles the tables of the version the branch or tag holds that have no [profile](/spec/versions#profiles), and puts the profiles beside the version. `fdl push` profiles the version it uploads; this is for a version pushed without them: made before fdl wrote profiles, pushed with `--no-profile`, or whose tables could not be read then. Profiles already there are left as they are. Refused for a version past retention, which has no catalog.
|
|
229
|
+
Profiles the tables and views of the version the branch or tag holds that have no [profile](/spec/versions#profiles), and puts the profiles beside the version. `fdl push` profiles the version it uploads; this is for a version pushed without them: made before fdl wrote profiles, pushed with `--no-profile`, or whose tables could not be read then. Profiles already there are left as they are. Refused for a version past retention, which has no catalog.
|
|
230
230
|
|
|
231
231
|
## fdl expire
|
|
232
232
|
|
|
@@ -208,7 +208,7 @@ When a version is taken out of retention, `files.json` is deleted with its catal
|
|
|
208
208
|
|
|
209
209
|
A profile summarizes one table at `ducklake.snapshot_id`: its row count, what each column holds, its first rows and, for a table with a geometry column, the extent and a sample of its shapes. A reader can show a table, or a host can index it for search, from this one small file, without opening the catalog or reading the data. It holds what was measured from the data; what a column means is the writer's, in `dataset.json`. Like `files.json`, it is made from the catalog and the data, which win where the two differ, and it is optional: a reader must work without it.
|
|
210
210
|
|
|
211
|
-
Each table's profile is at `versions/<ID>/profiles/<schema>/<table>.json`. A table whose schema or name is empty, `.`, `..` or holds `/` has none.
|
|
211
|
+
Each table's profile is at `versions/<ID>/profiles/<schema>/<table>.json`. A view has one too, measured from the rows its query returns. A table or view whose schema or name is empty, `.`, `..` or holds `/` has none. A version made without a profile for a table, by a writer that did not profile or failed to, may get one later; it is measured from the version's catalog and data, like any other profile.
|
|
212
212
|
|
|
213
213
|
```json profiles/main/towns.json
|
|
214
214
|
{
|
|
@@ -239,14 +239,14 @@ Each table's profile is at `versions/<ID>/profiles/<schema>/<table>.json`. A tab
|
|
|
239
239
|
}
|
|
240
240
|
```
|
|
241
241
|
|
|
242
|
-
- `rows` is the table's row count. `columns` lists every column in the catalog's order, with `name` and `type` (DuckLake's type name) as in `dataset.json`
|
|
242
|
+
- `rows` is the table's row count. `columns` lists every column in the catalog's order, with `name` and `type` (DuckLake's type name) as in `dataset.json`. For a view, which has no types in `dataset.json`, they are the columns its query returns, with DuckDB's type names
|
|
243
243
|
- Every column but a geometry one has `null_count`. Numbers, text, dates and times have `distinct_count`, which may be estimated on a large table, and `values`
|
|
244
244
|
- `values` lists each value, as text, with the number of rows that hold it, most common first and then by value. When the column holds at most 2,000 distinct values, it lists them all and `distinct_count` is exact: a reader tells a whole list by its length matching `distinct_count`. Otherwise it lists the 10 most common. A host can search a whole list by value, such as a town's code or name
|
|
245
245
|
- Numbers have `min`, `max`, `mean`, `std`, the quartiles `q1`, `median` and `q3`, and a `histogram` of equal-width buckets between `min` and `max`. Dates and times have `min` and `max` as text, and dates and timestamps `span_days`. Booleans have `true_count` and `false_count`
|
|
246
246
|
- `head` holds the first rows the table returns, as JSON objects, leaving out geometry and columns whose values have no JSON form
|
|
247
247
|
- `geometry` describes the first geometry column, when the writer could read it: how many rows have a shape, the `bbox` of all of them (`[xmin, ymin, xmax, ymax]` in the column's coordinates), the geometry types present, and a GeoJSON `FeatureCollection` of a random sample of simplified shapes for drawing a map
|
|
248
248
|
- A writer may leave out any key it cannot compute. A reader ignores keys it does not know
|
|
249
|
-
- A writer reuses its parent's profile for a table whose data files (in `files.json`) and columns are the parent's, so profiling reads only the tables that changed. It makes the profile again when the parent's `values` follow another limit than its own
|
|
249
|
+
- A writer reuses its parent's profile for a table whose data files (in `files.json`) and columns are the parent's, so profiling reads only the tables that changed. It reuses a view's when every view's `sql` is the parent's and no table's data files changed, as a view may read another view. It makes the profile again when the parent's `values` follow another limit than its own
|
|
250
250
|
- Profiles are part of the version and do not change with it. They are deleted with the catalogs when the version is taken out of retention
|
|
251
251
|
|
|
252
252
|
## The frozen catalog
|
|
@@ -289,9 +289,10 @@ def push(
|
|
|
289
289
|
) -> None:
|
|
290
290
|
"""Upload the committed version and move the branch to it.
|
|
291
291
|
|
|
292
|
-
First it profiles the version's tables (a summary of each,
|
|
293
|
-
the data), reading only
|
|
294
|
-
the parent, and
|
|
292
|
+
First it profiles the version's tables and views (a summary of each,
|
|
293
|
+
measured from the data), reading only the tables whose data files or
|
|
294
|
+
columns changed since the parent, and the views when any table or view
|
|
295
|
+
changed, and uploads the profiles with the version.
|
|
295
296
|
|
|
296
297
|
Refused when someone else moved the branch since the checkout: check out
|
|
297
298
|
again, redo the work, commit and push, or merge the branch into your work.
|
|
@@ -32,6 +32,9 @@ ALL_VALUES = 2000
|
|
|
32
32
|
TOP_VALUES = 10
|
|
33
33
|
#: Above this many rows, distinct values are estimated rather than counted.
|
|
34
34
|
EXACT_DISTINCT_ROWS = 1_000_000
|
|
35
|
+
#: Columns measured by one query. DuckDB holds a hash table for every `count(DISTINCT)`
|
|
36
|
+
#: of a query at once, so a table of many hundred columns in one query runs out of memory.
|
|
37
|
+
COLUMNS_PER_QUERY = 100
|
|
35
38
|
|
|
36
39
|
_NUMERIC = {
|
|
37
40
|
"TINYINT", "SMALLINT", "INTEGER", "BIGINT", "HUGEINT", "UTINYINT", "USMALLINT",
|
|
@@ -39,8 +42,13 @@ _NUMERIC = {
|
|
|
39
42
|
"INT8", "INT16", "INT32", "INT64", "INT128", "UINT8", "UINT16", "UINT32", "UINT64",
|
|
40
43
|
"UINT128", "FLOAT32", "FLOAT64",
|
|
41
44
|
} # fmt: skip
|
|
42
|
-
|
|
43
|
-
|
|
45
|
+
#: Timestamps of another unit: DuckLake's `timestamp_s` and the like, and DuckDB's names for a view's columns.
|
|
46
|
+
_TIMESTAMPS = {"TIMESTAMP_S", "TIMESTAMP_MS", "TIMESTAMP_NS"}
|
|
47
|
+
_TEMPORAL = {
|
|
48
|
+
"DATE", "TIME", "TIMESTAMP", "TIMESTAMPTZ", "TIMESTAMP WITH TIME ZONE", "TIMETZ", "TIME WITH TIME ZONE",
|
|
49
|
+
"INTERVAL", *_TIMESTAMPS,
|
|
50
|
+
} # fmt: skip
|
|
51
|
+
_DATED = {"DATE", "TIMESTAMP", "TIMESTAMPTZ", "TIMESTAMP WITH TIME ZONE", *_TIMESTAMPS}
|
|
44
52
|
_BOOLEAN = {"BOOLEAN"}
|
|
45
53
|
_TEXT = {"VARCHAR", "UUID"}
|
|
46
54
|
|
|
@@ -73,8 +81,8 @@ def path(schema: str, table: str) -> str | None:
|
|
|
73
81
|
|
|
74
82
|
|
|
75
83
|
def tables(described: dict[str, Any]) -> list[dict[str, Any]]:
|
|
76
|
-
"""The tables of a `dataset.json` that get a profile:
|
|
77
|
-
return [t for t in described.get("tables", []) if
|
|
84
|
+
"""The tables and views of a `dataset.json` that get a profile: all whose names a path can hold."""
|
|
85
|
+
return [t for t in described.get("tables", []) if path(t["schema"], t["name"])]
|
|
78
86
|
|
|
79
87
|
|
|
80
88
|
def columns_of(table: dict[str, Any]) -> list[dict[str, str]]:
|
|
@@ -115,14 +123,18 @@ def build(working: Path, wanted: Iterable[dict[str, Any]], *, data_path: str, se
|
|
|
115
123
|
def reusable(previous: bytes, table: dict[str, Any]) -> bool:
|
|
116
124
|
"""Whether a parent's profile describes ``table`` as it is and as this writer would.
|
|
117
125
|
|
|
118
|
-
The same columns and types, and `values` listed by the current `ALL_VALUES`:
|
|
126
|
+
The same columns and types (for a table), and `values` listed by the current `ALL_VALUES`:
|
|
119
127
|
a profile written under another limit is made again rather than carried forward.
|
|
120
128
|
"""
|
|
121
129
|
try:
|
|
122
130
|
held = json.loads(previous).get("columns", [])
|
|
123
131
|
except ValueError:
|
|
124
132
|
return False
|
|
125
|
-
|
|
133
|
+
# A view's columns are not in `dataset.json`; the caller keeps its profile only
|
|
134
|
+
# while its query and every table are unchanged, and so are its columns.
|
|
135
|
+
if table.get("kind") != "view" and [
|
|
136
|
+
{"name": c.get("name"), "type": c.get("type")} for c in held
|
|
137
|
+
] != columns_of(table):
|
|
126
138
|
return False
|
|
127
139
|
return all(_listed_as_now(c) for c in held)
|
|
128
140
|
|
|
@@ -157,12 +169,14 @@ def _q(name: str) -> str:
|
|
|
157
169
|
|
|
158
170
|
def _profile(con, table: dict[str, Any], spatial: bool) -> dict[str, Any]:
|
|
159
171
|
relation = f"lake.{_q(table['schema'])}.{_q(table['name'])}"
|
|
160
|
-
fields = columns_of(table)
|
|
172
|
+
fields = _view_columns(con, relation) if table.get("kind") == "view" else columns_of(table)
|
|
161
173
|
rows = int(con.execute(f"SELECT count(*) FROM {relation}").fetchone()[0])
|
|
162
174
|
|
|
163
175
|
measured = [f for f in fields if kind(f["type"]) != "geometry"]
|
|
164
|
-
|
|
165
|
-
|
|
176
|
+
values: list[Any] = []
|
|
177
|
+
for start in range(0, len(measured), COLUMNS_PER_QUERY):
|
|
178
|
+
exprs = [e for f in measured[start : start + COLUMNS_PER_QUERY] for e in _aggregates(f, rows)]
|
|
179
|
+
values += con.execute(f"SELECT {', '.join(exprs)} FROM {relation}").fetchone()
|
|
166
180
|
stats: list[dict[str, Any]] = []
|
|
167
181
|
cursor = 0
|
|
168
182
|
for field in fields:
|
|
@@ -191,6 +205,11 @@ def _profile(con, table: dict[str, Any], spatial: bool) -> dict[str, Any]:
|
|
|
191
205
|
return profile
|
|
192
206
|
|
|
193
207
|
|
|
208
|
+
def _view_columns(con, relation: str) -> list[dict[str, str]]:
|
|
209
|
+
"""A view's columns as its query returns them: the catalog records none for a view."""
|
|
210
|
+
return [{"name": row[0], "type": row[1]} for row in con.execute(f"DESCRIBE SELECT * FROM {relation}").fetchall()]
|
|
211
|
+
|
|
212
|
+
|
|
194
213
|
def _aggregates(field: dict[str, str], rows: int) -> list[str]:
|
|
195
214
|
c = _q(field["name"])
|
|
196
215
|
distinct = f"count(DISTINCT {c})" if rows <= EXACT_DISTINCT_ROWS else f"approx_count_distinct({c})"
|
|
@@ -566,8 +566,9 @@ def write_profiles(
|
|
|
566
566
|
"""Write a profile of each table of a committed version under ``local/profiles/``.
|
|
567
567
|
|
|
568
568
|
A table whose data files and columns are the parent's keeps the parent's
|
|
569
|
-
profile, so profiling reads only the tables that changed.
|
|
570
|
-
|
|
569
|
+
profile, so profiling reads only the tables that changed. A view keeps it
|
|
570
|
+
when no view's query and no table's data files changed. Returns
|
|
571
|
+
how many were made from the data.
|
|
571
572
|
"""
|
|
572
573
|
local = Path(local)
|
|
573
574
|
described_path = local / layout.DATASET
|
|
@@ -582,18 +583,33 @@ def write_profiles(
|
|
|
582
583
|
except Exception as exc: # noqa: BLE001 - stores raise their own types
|
|
583
584
|
if not is_missing(exc):
|
|
584
585
|
raise
|
|
586
|
+
views_now = _view_sql(described)
|
|
587
|
+
views_unchanged = False
|
|
588
|
+
if views_now and parent is not None and parent.dataset is not None and files_before == files_now:
|
|
589
|
+
# A view reads tables and other views, so it keeps its profile only when
|
|
590
|
+
# no table's data files and no view's query changed.
|
|
591
|
+
try:
|
|
592
|
+
held = json.loads(store.get(layout.version_dir(parent.id) + parent.dataset.path))
|
|
593
|
+
views_unchanged = _view_sql(held) == views_now
|
|
594
|
+
except Exception as exc: # noqa: BLE001 - stores raise their own types
|
|
595
|
+
if not is_missing(exc):
|
|
596
|
+
raise
|
|
585
597
|
kept: dict[str, bytes] = {}
|
|
586
598
|
pending = []
|
|
587
599
|
for table in profiles.tables(described):
|
|
588
600
|
key = f"{table['schema']}.{table['name']}"
|
|
589
601
|
relative = profiles.path(table["schema"], table["name"])
|
|
590
602
|
previous = None
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
603
|
+
if table.get("kind") == "view":
|
|
604
|
+
unchanged = views_unchanged
|
|
605
|
+
else:
|
|
606
|
+
before, now = files_before.get(key), files_now.get(key, {})
|
|
607
|
+
if before is not None and "columns" not in before:
|
|
608
|
+
# A parent written before files.json listed the columns: compare the
|
|
609
|
+
# rest, and leave the names and types to profiles.reusable.
|
|
610
|
+
now = {name: value for name, value in now.items() if name != "columns"}
|
|
611
|
+
unchanged = before is not None and before == now
|
|
612
|
+
if parent is not None and unchanged:
|
|
597
613
|
try:
|
|
598
614
|
previous = store.get(layout.version_dir(parent.id) + relative)
|
|
599
615
|
except Exception as exc: # noqa: BLE001
|
|
@@ -611,6 +627,13 @@ def write_profiles(
|
|
|
611
627
|
return len(made)
|
|
612
628
|
|
|
613
629
|
|
|
630
|
+
def _view_sql(described: dict) -> dict[str, str | None]:
|
|
631
|
+
"""Each view of a `dataset.json` (``schema.name``) and its query."""
|
|
632
|
+
return {
|
|
633
|
+
f"{t['schema']}.{t['name']}": t.get("sql") for t in described.get("tables", []) if t.get("kind") == "view"
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
|
|
614
637
|
def missing_profiles(store: Store, version: Version) -> list[dict]:
|
|
615
638
|
"""The tables of ``version`` that get a profile and have none in the repository."""
|
|
616
639
|
if version.dataset is None:
|
|
@@ -264,3 +264,68 @@ def test_push_profiles_the_tables_a_stopped_push_left_without(repository) -> Non
|
|
|
264
264
|
partial.write_bytes(b'{"rows": 1, "columns": []}')
|
|
265
265
|
run("push", working)
|
|
266
266
|
assert profile_of(store, "roads")["rows"] == 1
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def test_a_view_is_profiled(repository) -> None:
|
|
270
|
+
store, working = repository
|
|
271
|
+
write(working, store, "CREATE TABLE lake.main.towns AS SELECT i AS id FROM range(5) t(i)")
|
|
272
|
+
write(working, store, "CREATE VIEW lake.main.big AS SELECT id FROM lake.main.towns WHERE id >= 2")
|
|
273
|
+
commit(working, "-m", "m", "--author", "t")
|
|
274
|
+
got = profile_of(store, "big")
|
|
275
|
+
assert got["rows"] == 3
|
|
276
|
+
assert got["columns"][0]["min"] == 2
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _spy_builds(monkeypatch) -> list[str]:
|
|
280
|
+
built: list[str] = []
|
|
281
|
+
original = profiles.build
|
|
282
|
+
|
|
283
|
+
def spy(working, wanted, **kwargs):
|
|
284
|
+
wanted = list(wanted)
|
|
285
|
+
built.extend(t["name"] for t in wanted)
|
|
286
|
+
return original(working, wanted, **kwargs)
|
|
287
|
+
|
|
288
|
+
monkeypatch.setattr(profiles, "build", spy)
|
|
289
|
+
return built
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def test_a_view_keeps_its_profile_only_while_no_table_changes(repository, monkeypatch) -> None:
|
|
293
|
+
store, working = repository
|
|
294
|
+
write(working, store, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
|
|
295
|
+
write(working, store, "CREATE VIEW lake.main.big AS SELECT id FROM lake.main.towns")
|
|
296
|
+
commit(working, "-m", "m", "--author", "t")
|
|
297
|
+
built = _spy_builds(monkeypatch)
|
|
298
|
+
|
|
299
|
+
run("commit", working, "-m", "same", "--author", "t", "--allow-empty")
|
|
300
|
+
run("push", working)
|
|
301
|
+
assert built == []
|
|
302
|
+
|
|
303
|
+
write(working, store, "INSERT INTO lake.main.towns VALUES (2)")
|
|
304
|
+
commit(working, "-m", "again", "--author", "t")
|
|
305
|
+
assert sorted(built) == ["big", "towns"]
|
|
306
|
+
assert profile_of(store, "big")["rows"] == 2
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def test_a_view_over_a_view_is_profiled_again_when_that_view_changes(repository, monkeypatch) -> None:
|
|
310
|
+
store, working = repository
|
|
311
|
+
write(working, store, "CREATE TABLE lake.main.towns AS SELECT i AS id FROM range(5) t(i)")
|
|
312
|
+
write(working, store, "CREATE VIEW lake.main.big AS SELECT id FROM lake.main.towns WHERE id >= 2")
|
|
313
|
+
write(working, store, "CREATE VIEW lake.main.top AS SELECT id FROM lake.main.big")
|
|
314
|
+
commit(working, "-m", "m", "--author", "t")
|
|
315
|
+
assert profile_of(store, "top")["rows"] == 3
|
|
316
|
+
|
|
317
|
+
write(working, store, "CREATE OR REPLACE VIEW lake.main.big AS SELECT id FROM lake.main.towns WHERE id >= 4")
|
|
318
|
+
commit(working, "-m", "again", "--author", "t")
|
|
319
|
+
assert profile_of(store, "top")["rows"] == 1
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def test_a_wide_table_is_measured_a_few_columns_at_a_time(repository, monkeypatch) -> None:
|
|
323
|
+
store, working = repository
|
|
324
|
+
monkeypatch.setattr(profiles, "COLUMNS_PER_QUERY", 2)
|
|
325
|
+
write(working, store, "CREATE TABLE lake.main.towns AS SELECT i AS a, i + 1 AS b, i + 2 AS c FROM range(3) t(i)")
|
|
326
|
+
commit(working, "-m", "m", "--author", "t")
|
|
327
|
+
assert [(c["name"], c["min"], c["max"]) for c in profile_of(store)["columns"]] == [
|
|
328
|
+
("a", 0, 2),
|
|
329
|
+
("b", 1, 3),
|
|
330
|
+
("c", 2, 4),
|
|
331
|
+
]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|