fdl-cli 0.24.3__tar.gz → 0.26.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/CHANGELOG.md +113 -0
  2. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/PKG-INFO +1 -1
  3. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/getting-started.md +3 -1
  4. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/getting-started.md +3 -1
  5. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/reference/cli.md +11 -3
  6. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/spec/versions.md +4 -3
  7. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/spec/writing.md +1 -1
  8. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/reference/cli.md +11 -3
  9. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/spec/versions.md +4 -3
  10. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/spec/writing.md +1 -1
  11. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/pyproject.toml +1 -1
  12. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/src/fdl/cli.py +89 -14
  13. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/src/fdl/profiles.py +28 -9
  14. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/src/fdl/repo.py +73 -8
  15. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/tests/test_profiles.py +129 -1
  16. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/.github/workflows/docs.yml +0 -0
  17. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/.github/workflows/publish.yml +0 -0
  18. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/.github/workflows/release.yml +0 -0
  19. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/.github/workflows/test.yml +0 -0
  20. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/.gitignore +0 -0
  21. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/.node-version +0 -0
  22. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/.python-version +0 -0
  23. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/CLAUDE.md +0 -0
  24. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/CONTRIBUTING.md +0 -0
  25. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/LICENSE +0 -0
  26. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/README.md +0 -0
  27. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/SECURITY.md +0 -0
  28. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/blume.config.ts +0 -0
  29. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/cases/collection.json +0 -0
  30. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/concepts/meta.ts +0 -0
  31. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/concepts/repositories.md +0 -0
  32. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/concepts/versions-and-refs.md +0 -0
  33. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/concepts/working-catalogs.md +0 -0
  34. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/guides/building-a-host.md +0 -0
  35. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/guides/meta.ts +0 -0
  36. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/guides/mirrors.md +0 -0
  37. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/guides/public-and-private.md +0 -0
  38. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/guides/reading-without-duckdb.md +0 -0
  39. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/guides/serving-over-http.md +0 -0
  40. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/guides/stores.md +0 -0
  41. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/index.md +0 -0
  42. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/internals/ducklake-formats.md +0 -0
  43. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/internals/from-write-to-read.md +0 -0
  44. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/internals/inside-a-repository.md +0 -0
  45. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/internals/meta.ts +0 -0
  46. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/concepts/meta.ts +0 -0
  47. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/concepts/repositories.md +0 -0
  48. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/concepts/versions-and-refs.md +0 -0
  49. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/concepts/working-catalogs.md +0 -0
  50. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/guides/building-a-host.md +0 -0
  51. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/guides/meta.ts +0 -0
  52. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/guides/mirrors.md +0 -0
  53. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/guides/public-and-private.md +0 -0
  54. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/guides/reading-without-duckdb.md +0 -0
  55. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/guides/serving-over-http.md +0 -0
  56. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/guides/stores.md +0 -0
  57. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/index.md +0 -0
  58. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/internals/ducklake-formats.md +0 -0
  59. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/internals/from-write-to-read.md +0 -0
  60. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/internals/inside-a-repository.md +0 -0
  61. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/internals/meta.ts +0 -0
  62. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/reference/meta.ts +0 -0
  63. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/reference/python.md +0 -0
  64. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/spec/collection-cases.md +0 -0
  65. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/spec/data-files.md +0 -0
  66. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/spec/http.md +0 -0
  67. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/spec/index.md +0 -0
  68. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/spec/layout.md +0 -0
  69. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/spec/meta.ts +0 -0
  70. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/spec/reading.md +0 -0
  71. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/spec/refs.md +0 -0
  72. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/ja/stability.md +0 -0
  73. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/reference/meta.ts +0 -0
  74. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/reference/python.md +0 -0
  75. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/spec/collection-cases.md +0 -0
  76. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/spec/data-files.md +0 -0
  77. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/spec/http.md +0 -0
  78. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/spec/index.md +0 -0
  79. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/spec/layout.md +0 -0
  80. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/spec/meta.ts +0 -0
  81. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/spec/reading.md +0 -0
  82. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/spec/refs.md +0 -0
  83. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/docs/stability.md +0 -0
  84. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/package-lock.json +0 -0
  85. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/package.json +0 -0
  86. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/public/CNAME +0 -0
  87. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/src/fdl/__init__.py +0 -0
  88. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/src/fdl/catalog.py +0 -0
  89. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/src/fdl/dataset.py +0 -0
  90. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/src/fdl/diff.py +0 -0
  91. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/src/fdl/layout.py +0 -0
  92. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/src/fdl/lineage.py +0 -0
  93. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/src/fdl/merge.py +0 -0
  94. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/src/fdl/objects.py +0 -0
  95. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/src/fdl/py.typed +0 -0
  96. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/src/fdl/refs.py +0 -0
  97. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/src/fdl/store.py +0 -0
  98. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/src/fdl/stores.py +0 -0
  99. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/src/fdl/version.py +0 -0
  100. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/tests/http_repo.py +0 -0
  101. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/tests/test_cli.py +0 -0
  102. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/tests/test_collection_cases.py +0 -0
  103. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/tests/test_dataset.py +0 -0
  104. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/tests/test_http.py +0 -0
  105. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/tests/test_merge.py +0 -0
  106. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/tests/test_objects.py +0 -0
  107. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/tests/test_repo.py +0 -0
  108. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/tests/test_stores.py +0 -0
  109. {fdl_cli-0.24.3 → fdl_cli-0.26.0}/uv.lock +0 -0
@@ -1,6 +1,119 @@
1
1
  # CHANGELOG
2
2
 
3
3
 
4
+ ## v0.26.0 (2026-11-14)
5
+
6
+ ### Features
7
+
8
+ - Profile views, and measure wide tables a hundred columns at a time
9
+ ([#60](https://github.com/queria-io/fdl/pull/60),
10
+ [`e6c3442`](https://github.com/queria-io/fdl/commit/e6c34429590eedfcee7b568f6c030b497c5ef668))
11
+
12
+ * feat!: profile tables with fdl profile instead of at commit
13
+
14
+ fdl commit no longer writes profiles. fdl profile <working> profiles the version committed there,
15
+ for fdl push to upload with it; fdl profile <remote> adds the profiles a pushed version lacks. The
16
+ spec lets a version get a missing profile later.
17
+
18
+ BREAKING CHANGE: fdl commit writes no profiles. Run fdl profile between fdl commit and fdl push to
19
+ keep writing them.
20
+
21
+ * feat: profile tables at push instead of commit
22
+
23
+ fdl push profiles the version it uploads (--no-profile skips it), so the steps stay commit and push.
24
+ fdl profile <remote> is left for versions pushed without profiles. Also warns of tables that could
25
+ not be read, lists the version's profiles in one request, and updates the spec wording.
26
+
27
+ * fix: keep fdl push working when profiling cannot read the working catalog
28
+
29
+ Read dataset.json and the working catalog inside the guarded block, so a missing or unreadable
30
+ catalog only warns. Profile again when a stopped push left some tables without profiles.
31
+
32
+ * feat: profile views, and measure wide tables a hundred columns at a time
33
+
34
+ A view's profile is measured from the rows its query returns, with the columns DESCRIBE gives. It
35
+ keeps the parent's profile only while its sql is the parent's and no table's data files changed.
36
+
37
+ The column statistics now run in queries of at most 100 columns. One query over hundreds of
38
+ count(DISTINCT) ran out of memory, as DuckDB holds a hash table for each at once.
39
+
40
+ * fix: re-profile a view when any view's query changed
41
+
42
+ A view may read another view, so a view keeps its parent's profile only when every view's sql and
43
+ every table's data files are the parent's. Fetch the parent's dataset.json only when there are
44
+ views, list more of DuckDB's temporal types, and describe view profiling in push --help.
45
+
46
+ ### Breaking Changes
47
+
48
+ - Fdl commit writes no profiles. Run fdl profile between fdl commit and fdl push to keep writing
49
+ them.
50
+
51
+
52
+ ## v0.25.0 (2026-11-14)
53
+
54
+ ### Bug Fixes
55
+
56
+ - Keep fdl push working when profiling cannot read the working catalog
57
+ ([#59](https://github.com/queria-io/fdl/pull/59),
58
+ [`59bb12c`](https://github.com/queria-io/fdl/commit/59bb12cfe2308a2c71d6471f8d5e1e40e8b48393))
59
+
60
+ * feat!: profile tables with fdl profile instead of at commit
61
+
62
+ fdl commit no longer writes profiles. fdl profile <working> profiles the version committed there,
63
+ for fdl push to upload with it; fdl profile <remote> adds the profiles a pushed version lacks. The
64
+ spec lets a version get a missing profile later.
65
+
66
+ BREAKING CHANGE: fdl commit writes no profiles. Run fdl profile between fdl commit and fdl push to
67
+ keep writing them.
68
+
69
+ * feat: profile tables at push instead of commit
70
+
71
+ fdl push profiles the version it uploads (--no-profile skips it), so the steps stay commit and push.
72
+ fdl profile <remote> is left for versions pushed without profiles. Also warns of tables that could
73
+ not be read, lists the version's profiles in one request, and updates the spec wording.
74
+
75
+ * fix: keep fdl push working when profiling cannot read the working catalog
76
+
77
+ Read dataset.json and the working catalog inside the guarded block, so a missing or unreadable
78
+ catalog only warns. Profile again when a stopped push left some tables without profiles.
79
+
80
+ ### Features
81
+
82
+ - Profile tables at push instead of commit ([#58](https://github.com/queria-io/fdl/pull/58),
83
+ [`95803d8`](https://github.com/queria-io/fdl/commit/95803d81860ba72a8887b1494a9a5b8bc4aff674))
84
+
85
+ * feat!: profile tables with fdl profile instead of at commit
86
+
87
+ fdl commit no longer writes profiles. fdl profile <working> profiles the version committed there,
88
+ for fdl push to upload with it; fdl profile <remote> adds the profiles a pushed version lacks. The
89
+ spec lets a version get a missing profile later.
90
+
91
+ BREAKING CHANGE: fdl commit writes no profiles. Run fdl profile between fdl commit and fdl push to
92
+ keep writing them.
93
+
94
+ * feat: profile tables at push instead of commit
95
+
96
+ fdl push profiles the version it uploads (--no-profile skips it), so the steps stay commit and push.
97
+ fdl profile <remote> is left for versions pushed without profiles. Also warns of tables that could
98
+ not be read, lists the version's profiles in one request, and updates the spec wording.
99
+
100
+ - Profile tables with fdl profile instead of at commit
101
+ ([#57](https://github.com/queria-io/fdl/pull/57),
102
+ [`20f21ac`](https://github.com/queria-io/fdl/commit/20f21acca7ba6effc3af1f9c1c39522f982ce68f))
103
+
104
+ fdl commit no longer writes profiles. fdl profile <working> profiles the version committed there,
105
+ for fdl push to upload with it; fdl profile <remote> adds the profiles a pushed version lacks. The
106
+ spec lets a version get a missing profile later.
107
+
108
+ BREAKING CHANGE: fdl commit writes no profiles. Run fdl profile between fdl commit and fdl push to
109
+ keep writing them.
110
+
111
+ ### Breaking Changes
112
+
113
+ - Fdl commit writes no profiles. Run fdl profile between fdl commit and fdl push to keep writing
114
+ them.
115
+
116
+
4
117
  ## v0.24.3 (2026-11-13)
5
118
 
6
119
  ### Bug Fixes
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: fdl-cli
3
- Version: 0.24.3
3
+ Version: 0.26.0
4
4
  Summary: Versions of a DuckLake catalog on object storage, kept the way Git keeps commits
5
5
  Project-URL: Homepage, https://fdl.queria.io/
6
6
  Project-URL: Documentation, https://fdl.queria.io/
@@ -83,7 +83,7 @@ fdl commit work/lake.sqlite -m "First towns" --author alice
83
83
  fdl push work/lake.sqlite
84
84
  ```
85
85
 
86
- `commit` freezes the working catalog into a version on your disk. `push` uploads it and moves `main` to it, and prints `refs/heads/main -> 019a3c4e-… (version 1)`. The repository now holds:
86
+ `commit` freezes the working catalog into a version on your disk. `push` measures each table into a small summary (a profile) that readers can show without reading the data, uploads the version with them and moves `main` to it, and prints `refs/heads/main -> 019a3c4e-… (version 1)`. The repository now holds:
87
87
 
88
88
  ```
89
89
  towns/
@@ -93,6 +93,8 @@ towns/
93
93
  versions/019a3c4e-…/
94
94
  ducklake.duckdb
95
95
  dataset.json
96
+ files.json
97
+ profiles/main/towns.json
96
98
  version.json
97
99
  data/main/towns/ducklake-….parquet
98
100
  ```
@@ -83,7 +83,7 @@ fdl commit work/lake.sqlite -m "First towns" --author alice
83
83
  fdl push work/lake.sqlite
84
84
  ```
85
85
 
86
- `commit` は作業カタログを固めて手元でバージョンにします。`push` はそれをアップロードして `main` をそこへ動かし、`refs/heads/main -> 019a3c4e-… (version 1)` と表示します。リポジトリは次のようになります。
86
+ `commit` は作業カタログを固めて手元でバージョンにします。`push` はテーブルごとに、データを読まずに表示できる小さな要約(profile)を測り、バージョンと一緒にアップロードして `main` をそこへ動かし、`refs/heads/main -> 019a3c4e-… (version 1)` と表示します。リポジトリは次のようになります。
87
87
 
88
88
  ```
89
89
  towns/
@@ -93,6 +93,8 @@ towns/
93
93
  versions/019a3c4e-…/
94
94
  ducklake.duckdb
95
95
  dataset.json
96
+ files.json
97
+ profiles/main/towns.json
96
98
  version.json
97
99
  data/main/towns/ducklake-….parquet
98
100
  ```
@@ -57,8 +57,6 @@ fdl commit <working.sqlite> -m "What changed" --author alice [--dataset dataset.
57
57
 
58
58
  コマンドは、説明の隣の `README.md` と、そこにある `docs/` より下のファイル(隠しファイルを除く)をバージョンの tree に入れます。そのフォルダからのパスで `tree.json` を書き、push がリポジトリに無いものを書いたままの中身で `assets/<sha256>` にアップロードします([tree.json](/ja/spec/versions#treejson))。
59
59
 
60
- あわせてテーブルごとの profile を書きます([profiles/](/ja/spec/versions#profiles))。読むのは、データファイルか列が親から変わったテーブルだけです。読めなかったテーブルには profile を書かず、警告を出してコミットを続けます。
61
-
62
60
  `asset: true` の印を付けた列は、ファイルの sha256 を持ちます([アセット](/ja/spec/versions#アセット))。コマンドはその列の値を読み、push がリポジトリに無いファイルを `--assets` のフォルダから `assets/<sha256>` にアップロードします。ファイルは中身で照合するので、フォルダの中のファイル名は問いません。
63
61
 
64
62
  親と同じ中身になるとき([変更がないとき](/ja/spec/writing#変更がないとき))は `Nothing to commit` と表示し、何も作らずに終了コード 0 で終わります。同じ行を新しいファイルに書き直しただけの作り直しも、変更なしとみなします。`--allow-empty` を付けると、それでもバージョンを作ります。行を読めず判定できないときは、警告を出してバージョンを作ります。
@@ -73,11 +71,13 @@ fdl commit <working.sqlite> -m "What changed" --author alice [--dataset dataset.
73
71
  ## fdl push
74
72
 
75
73
  ```bash
76
- fdl push <working.sqlite>
74
+ fdl push <working.sqlite> [--no-profile]
77
75
  ```
78
76
 
79
77
  `fdl commit` が作ったバージョンをアップロードし、条件付きでブランチをそこへ動かします([書き込みの順番](/ja/spec/writing#書き込みの順番))。ブランチ、バージョン ID、`seq` を表示します。fast-forward の `fdl merge` のあとは、何もアップロードせず、ブランチをマージしたバージョンへ動かします。コミットしたものが無いときは `Nothing to push` と表示します。
80
78
 
79
+ アップロードの前に、テーブルとビューごとの profile([profiles/](/ja/spec/versions#profiles))を書き、`version.json` より前にアップロードします。ブランチが動く時点で profile がリポジトリにあります。読むのは、データファイルか列が親から変わったテーブルだけで、ほかのテーブルは親の profile を引き継ぎます。ビューは、どれかのテーブルのデータファイルかビューのクエリが変わると、クエリを実行して測り直します。読めなかったテーブルには profile を書かず、警告を出します。コミットのあとに作業カタログが変わっていたときは profile を書かずに警告します。`--no-profile` を付けると profile を書きません。`fdl profile` であとから足せます。
80
+
81
81
  作業カタログがリポジトリに書いたデータファイルは push の前からありますが、push するまでどのバージョンも参照しません。push しなかったファイルは猶予期間のあとに削除されます。
82
82
 
83
83
  次のときは断られます。
@@ -220,6 +220,14 @@ fdl withdraw <remote> <version>... [--yes]
220
220
 
221
221
  `--yes` を付けないと、消すものを表示するだけです。残すバージョンが参照しているファイルは残すものとして表示します。それがそのデータを持つなら、そのバージョンも取り下げます。ブランチかリクエストが該当するバージョンを指しているあいだは断られます。
222
222
 
223
+ ## fdl profile
224
+
225
+ ```bash
226
+ fdl profile <remote> [--ref refs/heads/main]
227
+ ```
228
+
229
+ ブランチかタグが指すバージョンのテーブルとビューのうち、[profile](/ja/spec/versions#profiles) が無いものを測り、profile をバージョンの横に置きます。`fdl push` はアップロードするバージョンの profile を作ります。このコマンドは profile 無しで push したバージョンに使います。fdl が profile を書く前に作ったバージョン、`--no-profile` で push したバージョン、そのときテーブルを読めなかったバージョンです。既にある profile は変えません。保持期間を過ぎたバージョンはカタログが無いので断られます。
230
+
223
231
  ## fdl expire
224
232
 
225
233
  ```bash
@@ -9,6 +9,7 @@ description: バージョン ID、version.json の項目、固めたカタログ
9
9
  - 取り下げのとき、`"reason": "withdrawn"` の `expired.json` を足してカタログを消し、`version.json` 自体に取り下げるデータがあればそれも消す([データファイル](/ja/spec/data-files#取り下げ))
10
10
  - `catalogs/<format>.duckdb` はあとから足してよい。既にあるファイルは変えない
11
11
  - `version.json.sig` は、バージョンを作ったあとに署名する書き手が足してよい
12
+ - `profiles/<schema>/<table>.json` は、profile の無いテーブルについてあとから足してよい。既にある profile は変えない
12
13
 
13
14
  ## バージョン ID
14
15
 
@@ -207,7 +208,7 @@ DuckLake を使わずに Parquet ファイルを読む読み手は、`deletes`
207
208
 
208
209
  profile は、`ducklake.snapshot_id` の時点の 1 つのテーブルの要約です。行数、列ごとの中身、先頭の行、図形の列を持つテーブルでは図形の範囲と見本を持ちます。読み手はこの小さなファイル 1 つで、カタログを開かずデータも読まずにテーブルを表示できます。ホストは検索の索引をこれから作れます。profile が持つのはデータから測った事実で、列が何を意味するかは書き手が `dataset.json` に書きます。`files.json` と同じくカタログとデータから作るもので、食い違うときはカタログとデータが正です。任意で、読み手は profile が無くても読めなければなりません。
209
210
 
210
- テーブルごとの profile は `versions/<ID>/profiles/<schema>/<table>.json` に置きます。スキーマ名かテーブル名が空・`.`・`..` のとき、または `/` を含むときは置きません。ビューには置きません。
211
+ テーブルごとの profile は `versions/<ID>/profiles/<schema>/<table>.json` に置きます。ビューにも置き、クエリが返す行から測ります。スキーマ名かテーブル名が空・`.`・`..` のとき、または `/` を含むときは置きません。profile を作らなかった書き手や作るのに失敗した書き手のバージョンには、あとから profile を足せます。ほかの profile と同じく、そのバージョンのカタログとデータから測ります。
211
212
 
212
213
  ```json profiles/main/towns.json
213
214
  {
@@ -238,14 +239,14 @@ profile は、`ducklake.snapshot_id` の時点の 1 つのテーブルの要約
238
239
  }
239
240
  ```
240
241
 
241
- - `rows` はテーブルの行数。`columns` にはカタログの順にすべての列を並べ、`name` と `type`(DuckLake の型名)は `dataset.json` と同じにする
242
+ - `rows` はテーブルの行数。`columns` にはカタログの順にすべての列を並べ、`name` と `type`(DuckLake の型名)は `dataset.json` と同じにする。`dataset.json` に型の無いビューでは、クエリが返す列を DuckDB の型名で並べる
242
243
  - 図形の列を除くすべての列が `null_count` を持つ。数値・テキスト・日付と時刻の列は `distinct_count` と `values` を持つ。大きなテーブルの `distinct_count` は推定値のことがある
243
244
  - `values` は、値(テキスト)とその値を持つ行の数を、多い順、同数なら値の順に並べる。列が持つ異なる値が 2,000 以下なら全部を並べ、そのとき `distinct_count` は正確な数になる。読み手は、`values` の長さが `distinct_count` と等しいことで全部の一覧だと判定する。それより多い列は上位 10 件だけを並べる。ホストは全部の一覧を、地域コードや名前などの値で検索できる
244
245
  - 数値の列は `min`・`max`・`mean`・`std`、四分位数の `q1`・`median`・`q3`、`min` から `max` までを等幅に区切った `histogram` を持つ。日付と時刻の列は `min` と `max` をテキストで持ち、日付とタイムスタンプは `span_days` も持つ。真偽値の列は `true_count` と `false_count` を持つ
245
246
  - `head` は、テーブルが返す先頭の行を JSON のオブジェクトで持つ。図形の列と、値を JSON で表せない列は入れない
246
247
  - `geometry` は、書き手が読めたときに、最初の図形の列を説明する。図形を持つ行の数、全体の `bbox`(列の座標で `[xmin, ymin, xmax, ymax]`)、含まれる図形の種類、地図を描くための、無作為に選んで簡略化した図形の GeoJSON `FeatureCollection` を持つ
247
248
  - 書き手は計算できないキーを省いてよい。読み手は知らないキーを無視する
248
- - 書き手は、データファイル(`files.json`)と列が親と同じテーブルには親の profile を使い回す。コミットで読むのは変わったテーブルだけになる。親の profile の `values` が書き手と異なる上限で書かれているときは、profile を作り直す
249
+ - 書き手は、データファイル(`files.json`)と列が親と同じテーブルには親の profile を使い回す。profile を作るときに読むのは変わったテーブルだけになる。ビューは別のビューを読むことがあるので、すべてのビューの `sql` が親と同じで、どのテーブルのデータファイルも変わっていないときに親の profile を使い回す。親の profile の `values` が書き手と異なる上限で書かれているときは、profile を作り直す
249
250
  - profile はバージョンの一部で、そのバージョンの中では変わらない。保持期間から外すときはカタログと一緒に消す
250
251
 
251
252
  ## 固めたカタログ
@@ -17,7 +17,7 @@ description: チェックアウト、コミット、push。作業カタログ、
17
17
  | --- | --- |
18
18
  | チェックアウト | ブランチのバージョンのカタログから作業カタログを作り、そのバージョンを覚える。最初のバージョンの前は空のものを作る |
19
19
  | コミット | 作業カタログの最後のスナップショットを新しいバージョンに固める。親は覚えておいたバージョン。中身を `dataset.json` で説明する |
20
- | push | データファイルを確かめ、カタログ、`dataset.json`・`files.json`・`tree.json`・profile、`version.json` の順にアップロードし、条件付きで ref を動かす |
20
+ | push | データファイルを確かめ、テーブルごとに profile を測り(任意)、カタログ、`dataset.json`・`files.json`・`tree.json`・profile、`version.json` の順にアップロードし、条件付きで ref を動かす |
21
21
 
22
22
  書き手は、Git が `git push` までコミットを手元に置くのと同じく、コミットしたバージョンを手元に置いてあとで push してかまいません。push までリポジトリは変わりません。例外は作業カタログがリポジトリに書いたデータファイルで、どのバージョンもまだ参照していないので、push しないままなら猶予期間のあとに削除されます。
23
23
 
@@ -57,8 +57,6 @@ Every version is described by a `dataset.json` (see [Versions](/spec/versions#da
57
57
 
58
58
  The command puts `README.md` beside the description and every file below `docs/` there (hidden ones left out) in the version's tree: it writes `tree.json` with their paths from that folder and the push uploads those the repository lacks to `assets/<sha256>`, as written (see [tree.json](/spec/versions#treejson)).
59
59
 
60
- It also writes a profile of each table (see [profiles/](/spec/versions#profiles)), reading only the tables whose data files or columns changed since the parent. A table it cannot read gets none, and the commit goes on with a warning.
61
-
62
60
  Columns marked `asset: true` hold the sha256 of files (see [Assets](/spec/versions#assets)). The command reads the values those columns hold, and the push uploads the files the repository lacks from the `--assets` folder to `assets/<sha256>`, by their content: file names in the folder do not matter.
63
61
 
64
62
  When the version would hold what its parent holds (see [When nothing changed](/spec/writing#when-nothing-changed)), it prints `Nothing to commit`, makes nothing and exits 0. A rebuild that writes the same rows to new files counts as unchanged. `--allow-empty` makes the version anyway. If the rows cannot be read to tell, it warns and makes the version.
@@ -72,11 +70,13 @@ Refused when:
72
70
  ## fdl push
73
71
 
74
72
  ```bash
75
- fdl push <working.sqlite>
73
+ fdl push <working.sqlite> [--no-profile]
76
74
  ```
77
75
 
78
76
  Uploads the version `fdl commit` made and moves the branch to it on condition (see [Order of writes](/spec/writing#order-of-writes)). It prints the branch, the version ID and its `seq`. After a fast-forward `fdl merge`, it moves the branch to the version merged, uploading nothing. With nothing committed, it prints `Nothing to push`.
79
77
 
78
+ Before uploading, it writes a profile of each table and view (see [profiles/](/spec/versions#profiles)) and uploads the profiles before `version.json`, so they are there when the branch moves. It reads only the tables whose data files or columns changed since the parent; the others keep the parent's profile. A view is profiled again, by running its query, whenever a table's data files or a view's query changed. A table it cannot read gets none, with a warning. When the working catalog changed after the commit, it profiles nothing and warns. `--no-profile` skips profiling; `fdl profile` adds the profiles later.
79
+
80
80
  Data files the working catalog wrote into the repository are there before the push, but no version names them until it. Files never pushed are deleted after the grace period.
81
81
 
82
82
  Refused when:
@@ -220,6 +220,14 @@ A version reaches the assets its `tree.json` lists and those its asset columns h
220
220
 
221
221
  Without `--yes` it only prints what it would delete. Files a kept version still reaches are listed as kept; if they hold the data, withdraw that version too. Refused while a branch or a request points at one of the versions.
222
222
 
223
+ ## fdl profile
224
+
225
+ ```bash
226
+ fdl profile <remote> [--ref refs/heads/main]
227
+ ```
228
+
229
+ Profiles the tables and views of the version the branch or tag holds that have no [profile](/spec/versions#profiles), and puts the profiles beside the version. `fdl push` profiles the version it uploads; this is for a version pushed without them: made before fdl wrote profiles, pushed with `--no-profile`, or whose tables could not be read then. Profiles already there are left as they are. Refused for a version past retention, which has no catalog.
230
+
223
231
  ## fdl expire
224
232
 
225
233
  ```bash
@@ -9,6 +9,7 @@ A version is the files under `versions/<version ID>/`. They must not change once
9
9
  - on withdrawal, `expired.json` is added with `"reason": "withdrawn"` and the catalogs deleted, and `version.json` itself is deleted if it holds the withdrawn data ([Data files](/spec/data-files#withdrawal))
10
10
  - `catalogs/<format>.duckdb` may be added later. Files already there are not changed
11
11
  - `version.json.sig` may be added later, by a writer that signs a version after making it
12
+ - `profiles/<schema>/<table>.json` may be added later for a table that has none. Profiles already there are not changed
12
13
 
13
14
  ## Version ID
14
15
 
@@ -207,7 +208,7 @@ When a version is taken out of retention, `files.json` is deleted with its catal
207
208
 
208
209
  A profile summarizes one table at `ducklake.snapshot_id`: its row count, what each column holds, its first rows and, for a table with a geometry column, the extent and a sample of its shapes. A reader can show a table, or a host can index it for search, from this one small file, without opening the catalog or reading the data. It holds what was measured from the data; what a column means is the writer's, in `dataset.json`. Like `files.json`, it is made from the catalog and the data, which win where the two differ, and it is optional: a reader must work without it.
209
210
 
210
- Each table's profile is at `versions/<ID>/profiles/<schema>/<table>.json`. A table whose schema or name is empty, `.`, `..` or holds `/` has none. Views have none.
211
+ Each table's profile is at `versions/<ID>/profiles/<schema>/<table>.json`. A view has one too, measured from the rows its query returns. A table or view whose schema or name is empty, `.`, `..` or holds `/` has none. A version made without a profile for a table, by a writer that did not profile or failed to, may get one later; it is measured from the version's catalog and data, like any other profile.
211
212
 
212
213
  ```json profiles/main/towns.json
213
214
  {
@@ -238,14 +239,14 @@ Each table's profile is at `versions/<ID>/profiles/<schema>/<table>.json`. A tab
238
239
  }
239
240
  ```
240
241
 
241
- - `rows` is the table's row count. `columns` lists every column in the catalog's order, with `name` and `type` (DuckLake's type name) as in `dataset.json`
242
+ - `rows` is the table's row count. `columns` lists every column in the catalog's order, with `name` and `type` (DuckLake's type name) as in `dataset.json`. For a view, which has no types in `dataset.json`, they are the columns its query returns, with DuckDB's type names
242
243
  - Every column but a geometry one has `null_count`. Numbers, text, dates and times have `distinct_count`, which may be estimated on a large table, and `values`
243
244
  - `values` lists each value, as text, with the number of rows that hold it, most common first and then by value. When the column holds at most 2,000 distinct values, it lists them all and `distinct_count` is exact: a reader tells a whole list by its length matching `distinct_count`. Otherwise it lists the 10 most common. A host can search a whole list by value, such as a town's code or name
244
245
  - Numbers have `min`, `max`, `mean`, `std`, the quartiles `q1`, `median` and `q3`, and a `histogram` of equal-width buckets between `min` and `max`. Dates and times have `min` and `max` as text, and dates and timestamps `span_days`. Booleans have `true_count` and `false_count`
245
246
  - `head` holds the first rows the table returns, as JSON objects, leaving out geometry and columns whose values have no JSON form
246
247
  - `geometry` describes the first geometry column, when the writer could read it: how many rows have a shape, the `bbox` of all of them (`[xmin, ymin, xmax, ymax]` in the column's coordinates), the geometry types present, and a GeoJSON `FeatureCollection` of a random sample of simplified shapes for drawing a map
247
248
  - A writer may leave out any key it cannot compute. A reader ignores keys it does not know
248
- - A writer reuses its parent's profile for a table whose data files (in `files.json`) and columns are the parent's, so a commit reads only the tables that changed. It makes the profile again when the parent's `values` follow another limit than its own
249
+ - A writer reuses its parent's profile for a table whose data files (in `files.json`) and columns are the parent's, so profiling reads only the tables that changed. It reuses a view's when every view's `sql` is the parent's and no table's data files changed, as a view may read another view. It makes the profile again when the parent's `values` follow another limit than its own
249
250
  - Profiles are part of the version and do not change with it. They are deleted with the catalogs when the version is taken out of retention
250
251
 
251
252
  ## The frozen catalog
@@ -17,7 +17,7 @@ A writer writes to a working catalog: a DuckLake catalog in SQLite, outside the
17
17
  | --- | --- |
18
18
  | checkout | Make a working catalog from the catalog of the branch's version, and remember that version. Before the first version, make an empty one |
19
19
  | commit | Freeze the working catalog's last snapshot into a new version, with the remembered version as its parent, and describe it in `dataset.json` |
20
- | push | Check the data files, upload the catalog, `dataset.json`, `files.json`, `tree.json` and the profiles, then `version.json`, then move the ref on condition |
20
+ | push | Check the data files, measure each table into a profile (optional), upload the catalog, `dataset.json`, `files.json`, `tree.json` and the profiles, then `version.json`, then move the ref on condition |
21
21
 
22
22
  A writer may keep a committed version on its own disk and push it later, as Git keeps commits until `git push`. Until the push, nothing in the repository changes, apart from data files the working catalog wrote there. No version names those yet, so they are deleted after the grace period if the version is never pushed.
23
23
 
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "fdl-cli"
3
- version = "0.24.3"
3
+ version = "0.26.0"
4
4
  description = "Versions of a DuckLake catalog on object storage, kept the way Git keeps commits"
5
5
  readme = "README.md"
6
6
  license = "MIT"
@@ -20,7 +20,7 @@ from urllib.parse import urlparse
20
20
 
21
21
  import typer
22
22
 
23
- from . import catalog, dataset, layout, objects, refs, repo
23
+ from . import catalog, dataset, layout, objects, profiles, refs, repo
24
24
  from . import merge as merging
25
25
  from . import diff as diffs
26
26
  from .stores import StoreError, open_store
@@ -230,16 +230,11 @@ def _commit(
230
230
  except repo.CannotCompare as exc:
231
231
  typer.echo(f"warning: could not tell whether anything changed ({exc}); committing", err=True)
232
232
  version = make(True)
233
- reading = written or repo.data_url(store)
234
- setup = _read_setup(store) + _storage_setup(state.get("storage"))
233
+ reading, setup = _reading(store, state)
235
234
  described = json.loads((out / version.id / layout.DATASET).read_text(encoding="utf-8"))
236
235
  held = objects.asset_values(
237
236
  working, objects.asset_columns(described), data_path=reading, setup=setup
238
237
  )
239
- try:
240
- repo.write_profiles(store, out / version.id, parent, working, data_path=reading, setup=setup)
241
- except Exception as exc: # noqa: BLE001 - a profile is never worth a failed commit
242
- typer.echo(f"warning: tables were not profiled ({exc})", err=True)
243
238
  pending = _pending_dir(working)
244
239
  shutil.rmtree(pending, ignore_errors=True)
245
240
  out.rename(pending)
@@ -280,10 +275,25 @@ def _written(store, state: dict) -> str | None:
280
275
  return None
281
276
 
282
277
 
278
+ def _reading(store, state: dict) -> tuple[str, list[str]]:
279
+ """Where the working catalog's data files are read from, and the setup to read them."""
280
+ return _written(store, state) or repo.data_url(store), _read_setup(store) + _storage_setup(state.get("storage"))
281
+
282
+
283
283
  @app.command()
284
- def push(working: Path = Working) -> None:
284
+ def push(
285
+ working: Path = Working,
286
+ no_profile: bool = typer.Option(
287
+ False, "--no-profile", help="Upload the version without profiling its tables"
288
+ ),
289
+ ) -> None:
285
290
  """Upload the committed version and move the branch to it.
286
291
 
292
+ First it profiles the version's tables and views (a summary of each,
293
+ measured from the data), reading only the tables whose data files or
294
+ columns changed since the parent, and the views when any table or view
295
+ changed, and uploads the profiles with the version.
296
+
287
297
  Refused when someone else moved the branch since the checkout: check out
288
298
  again, redo the work, commit and push, or merge the branch into your work.
289
299
  """
@@ -299,6 +309,8 @@ def push(working: Path = Working) -> None:
299
309
  folder = _pending_dir(working) / pending["id"]
300
310
  try:
301
311
  version = parse_version((folder / layout.VERSION_RECORD).read_bytes(), expected_id=pending["id"])
312
+ if not no_profile:
313
+ _profile_pending(store, working, state, folder, version)
302
314
  assets_dir = pending.get("assets_dir")
303
315
  repo.push(
304
316
  store,
@@ -363,12 +375,8 @@ def revert(
363
375
  if declared is None:
364
376
  raise repo.RepoError(f"{target.id} has no dataset.json to bring back")
365
377
  listed = (diffs.listed(store, target) or {}).get("entries", [])
366
- catalog.restore(
367
- working,
368
- target.ducklake.snapshot_id,
369
- data_path=_written(store, state) or repo.data_url(store),
370
- setup=_read_setup(store) + _storage_setup(state.get("storage")),
371
- )
378
+ reading, setup = _reading(store, state)
379
+ catalog.restore(working, target.ducklake.snapshot_id, data_path=reading, setup=setup)
372
380
  version = _commit(
373
381
  store,
374
382
  working,
@@ -799,6 +807,73 @@ def tag(
799
807
  typer.echo(f"refs/tags/{name} -> {version_id}")
800
808
 
801
809
 
810
+ @app.command()
811
+ def profile(
812
+ remote: str = Remote,
813
+ ref: str = typer.Option(layout.DEFAULT_BRANCH, help="The branch or tag whose version to profile"),
814
+ ) -> None:
815
+ """Profile the tables of a pushed version that have no profile.
816
+
817
+ `fdl push` profiles the version it uploads. This is for a version pushed
818
+ without them: made before fdl wrote profiles, pushed with --no-profile,
819
+ or whose tables could not be read then. Profiles already there are left
820
+ as they are.
821
+ """
822
+ store = _store(remote)
823
+ try:
824
+ version = repo.readable(store, ref)
825
+ put, unread = repo.add_profiles(store, version, setup=_read_setup(store))
826
+ except (*ERRORS, catalog.CatalogError) as exc:
827
+ raise _fail(exc) from exc
828
+ for relative in put:
829
+ typer.echo(f"put {relative}")
830
+ _warn_unread(unread)
831
+ typer.echo(f"Profiled {len(put)} table(s) of {version.id} (version {version.seq}).")
832
+
833
+
834
+ def _warn_unread(tables: list[str]) -> None:
835
+ if tables:
836
+ typer.echo(f"warning: tables were not profiled, as they could not be read: {', '.join(tables)}", err=True)
837
+
838
+
839
+ def _profile_pending(store, working: Path, state: dict, folder: Path, version) -> None:
840
+ """Profile the version committed in ``working``, beside it, for the push to upload.
841
+
842
+ A profile is never worth a failed push: when the working catalog no longer
843
+ holds what was committed, or the tables cannot be read, it warns and the
844
+ version goes without them (`fdl profile <remote>` adds them later).
845
+ """
846
+ described_path = folder / layout.DATASET
847
+
848
+ def unprofiled(wanted: list[dict]) -> list[str]:
849
+ return [
850
+ f"{table['schema']}.{table['name']}"
851
+ for table in wanted
852
+ if not (folder / profiles.path(table["schema"], table["name"])).is_file()
853
+ ]
854
+
855
+ try:
856
+ if not described_path.is_file():
857
+ return
858
+ wanted = profiles.tables(json.loads(described_path.read_text(encoding="utf-8")))
859
+ if not unprofiled(wanted):
860
+ return # no tables, or all profiled by a push that stopped before it finished
861
+ if catalog.last_snapshot(working) != version.ducklake.snapshot_id:
862
+ typer.echo(
863
+ "warning: tables were not profiled, as the working catalog changed after the commit; "
864
+ f"commit again, or run fdl profile {state['remote']} after the push",
865
+ err=True,
866
+ )
867
+ return
868
+ parent = repo.read_version(store, version.parents[0]) if version.parents else None
869
+ reading, setup = _reading(store, state)
870
+ repo.write_profiles(store, folder, parent, working, data_path=reading, setup=setup)
871
+ except Exception as exc: # noqa: BLE001 - a profile is never worth a failed push
872
+ typer.echo(f"warning: tables were not profiled ({exc})", err=True)
873
+ return
874
+ _warn_unread(unprofiled(wanted))
875
+
876
+
802
877
  @app.command()
803
878
  def expire(
804
879
  remote: str = Remote,
@@ -32,6 +32,9 @@ ALL_VALUES = 2000
32
32
  TOP_VALUES = 10
33
33
  #: Above this many rows, distinct values are estimated rather than counted.
34
34
  EXACT_DISTINCT_ROWS = 1_000_000
35
+ #: Columns measured by one query. DuckDB holds a hash table for every `count(DISTINCT)`
36
+ #: of a query at once, so a table of many hundred columns in one query runs out of memory.
37
+ COLUMNS_PER_QUERY = 100
35
38
 
36
39
  _NUMERIC = {
37
40
  "TINYINT", "SMALLINT", "INTEGER", "BIGINT", "HUGEINT", "UTINYINT", "USMALLINT",
@@ -39,8 +42,13 @@ _NUMERIC = {
39
42
  "INT8", "INT16", "INT32", "INT64", "INT128", "UINT8", "UINT16", "UINT32", "UINT64",
40
43
  "UINT128", "FLOAT32", "FLOAT64",
41
44
  } # fmt: skip
42
- _TEMPORAL = {"DATE", "TIME", "TIMESTAMP", "TIMESTAMPTZ", "TIMESTAMP WITH TIME ZONE", "TIMETZ", "INTERVAL"}
43
- _DATED = {"DATE", "TIMESTAMP", "TIMESTAMPTZ", "TIMESTAMP WITH TIME ZONE"}
45
+ #: Timestamps of another unit: DuckLake's `timestamp_s` and the like, and DuckDB's names for a view's columns.
46
+ _TIMESTAMPS = {"TIMESTAMP_S", "TIMESTAMP_MS", "TIMESTAMP_NS"}
47
+ _TEMPORAL = {
48
+ "DATE", "TIME", "TIMESTAMP", "TIMESTAMPTZ", "TIMESTAMP WITH TIME ZONE", "TIMETZ", "TIME WITH TIME ZONE",
49
+ "INTERVAL", *_TIMESTAMPS,
50
+ } # fmt: skip
51
+ _DATED = {"DATE", "TIMESTAMP", "TIMESTAMPTZ", "TIMESTAMP WITH TIME ZONE", *_TIMESTAMPS}
44
52
  _BOOLEAN = {"BOOLEAN"}
45
53
  _TEXT = {"VARCHAR", "UUID"}
46
54
 
@@ -73,8 +81,8 @@ def path(schema: str, table: str) -> str | None:
73
81
 
74
82
 
75
83
  def tables(described: dict[str, Any]) -> list[dict[str, Any]]:
76
- """The tables of a `dataset.json` that get a profile: every table, no view."""
77
- return [t for t in described.get("tables", []) if t.get("kind") != "view" and path(t["schema"], t["name"])]
84
+ """The tables and views of a `dataset.json` that get a profile: all whose names a path can hold."""
85
+ return [t for t in described.get("tables", []) if path(t["schema"], t["name"])]
78
86
 
79
87
 
80
88
  def columns_of(table: dict[str, Any]) -> list[dict[str, str]]:
@@ -115,14 +123,18 @@ def build(working: Path, wanted: Iterable[dict[str, Any]], *, data_path: str, se
115
123
  def reusable(previous: bytes, table: dict[str, Any]) -> bool:
116
124
  """Whether a parent's profile describes ``table`` as it is and as this writer would.
117
125
 
118
- The same columns and types, and `values` listed by the current `ALL_VALUES`:
126
+ The same columns and types (for a table), and `values` listed by the current `ALL_VALUES`:
119
127
  a profile written under another limit is made again rather than carried forward.
120
128
  """
121
129
  try:
122
130
  held = json.loads(previous).get("columns", [])
123
131
  except ValueError:
124
132
  return False
125
- if [{"name": c.get("name"), "type": c.get("type")} for c in held] != columns_of(table):
133
+ # A view's columns are not in `dataset.json`; the caller keeps its profile only
134
+ # while its query and every table are unchanged, and so are its columns.
135
+ if table.get("kind") != "view" and [
136
+ {"name": c.get("name"), "type": c.get("type")} for c in held
137
+ ] != columns_of(table):
126
138
  return False
127
139
  return all(_listed_as_now(c) for c in held)
128
140
 
@@ -157,12 +169,14 @@ def _q(name: str) -> str:
157
169
 
158
170
  def _profile(con, table: dict[str, Any], spatial: bool) -> dict[str, Any]:
159
171
  relation = f"lake.{_q(table['schema'])}.{_q(table['name'])}"
160
- fields = columns_of(table)
172
+ fields = _view_columns(con, relation) if table.get("kind") == "view" else columns_of(table)
161
173
  rows = int(con.execute(f"SELECT count(*) FROM {relation}").fetchone()[0])
162
174
 
163
175
  measured = [f for f in fields if kind(f["type"]) != "geometry"]
164
- exprs = [e for f in measured for e in _aggregates(f, rows)]
165
- values = con.execute(f"SELECT {', '.join(exprs)} FROM {relation}").fetchone() if exprs else ()
176
+ values: list[Any] = []
177
+ for start in range(0, len(measured), COLUMNS_PER_QUERY):
178
+ exprs = [e for f in measured[start : start + COLUMNS_PER_QUERY] for e in _aggregates(f, rows)]
179
+ values += con.execute(f"SELECT {', '.join(exprs)} FROM {relation}").fetchone()
166
180
  stats: list[dict[str, Any]] = []
167
181
  cursor = 0
168
182
  for field in fields:
@@ -191,6 +205,11 @@ def _profile(con, table: dict[str, Any], spatial: bool) -> dict[str, Any]:
191
205
  return profile
192
206
 
193
207
 
208
+ def _view_columns(con, relation: str) -> list[dict[str, str]]:
209
+ """A view's columns as its query returns them: the catalog records none for a view."""
210
+ return [{"name": row[0], "type": row[1]} for row in con.execute(f"DESCRIBE SELECT * FROM {relation}").fetchall()]
211
+
212
+
194
213
  def _aggregates(field: dict[str, str], rows: int) -> list[str]:
195
214
  c = _q(field["name"])
196
215
  distinct = f"count(DISTINCT {c})" if rows <= EXACT_DISTINCT_ROWS else f"approx_count_distinct({c})"
@@ -566,8 +566,9 @@ def write_profiles(
566
566
  """Write a profile of each table of a committed version under ``local/profiles/``.
567
567
 
568
568
  A table whose data files and columns are the parent's keeps the parent's
569
- profile, so a commit reads only the tables that changed. Returns how many
570
- were made from the data.
569
+ profile, so profiling reads only the tables that changed. A view keeps it
570
+ when no view's query and no table's data files changed. Returns
571
+ how many were made from the data.
571
572
  """
572
573
  local = Path(local)
573
574
  described_path = local / layout.DATASET
@@ -582,18 +583,33 @@ def write_profiles(
582
583
  except Exception as exc: # noqa: BLE001 - stores raise their own types
583
584
  if not is_missing(exc):
584
585
  raise
586
+ views_now = _view_sql(described)
587
+ views_unchanged = False
588
+ if views_now and parent is not None and parent.dataset is not None and files_before == files_now:
589
+ # A view reads tables and other views, so it keeps its profile only when
590
+ # no table's data files and no view's query changed.
591
+ try:
592
+ held = json.loads(store.get(layout.version_dir(parent.id) + parent.dataset.path))
593
+ views_unchanged = _view_sql(held) == views_now
594
+ except Exception as exc: # noqa: BLE001 - stores raise their own types
595
+ if not is_missing(exc):
596
+ raise
585
597
  kept: dict[str, bytes] = {}
586
598
  pending = []
587
599
  for table in profiles.tables(described):
588
600
  key = f"{table['schema']}.{table['name']}"
589
601
  relative = profiles.path(table["schema"], table["name"])
590
602
  previous = None
591
- before, now = files_before.get(key), files_now.get(key, {})
592
- if before is not None and "columns" not in before:
593
- # A parent written before files.json listed the columns: compare the
594
- # rest, and leave the names and types to profiles.reusable.
595
- now = {name: value for name, value in now.items() if name != "columns"}
596
- if parent is not None and before is not None and before == now:
603
+ if table.get("kind") == "view":
604
+ unchanged = views_unchanged
605
+ else:
606
+ before, now = files_before.get(key), files_now.get(key, {})
607
+ if before is not None and "columns" not in before:
608
+ # A parent written before files.json listed the columns: compare the
609
+ # rest, and leave the names and types to profiles.reusable.
610
+ now = {name: value for name, value in now.items() if name != "columns"}
611
+ unchanged = before is not None and before == now
612
+ if parent is not None and unchanged:
597
613
  try:
598
614
  previous = store.get(layout.version_dir(parent.id) + relative)
599
615
  except Exception as exc: # noqa: BLE001
@@ -611,6 +627,55 @@ def write_profiles(
611
627
  return len(made)
612
628
 
613
629
 
630
+ def _view_sql(described: dict) -> dict[str, str | None]:
631
+ """Each view of a `dataset.json` (``schema.name``) and its query."""
632
+ return {
633
+ f"{t['schema']}.{t['name']}": t.get("sql") for t in described.get("tables", []) if t.get("kind") == "view"
634
+ }
635
+
636
+
637
+ def missing_profiles(store: Store, version: Version) -> list[dict]:
638
+ """The tables of ``version`` that get a profile and have none in the repository."""
639
+ if version.dataset is None:
640
+ return []
641
+ folder = layout.version_dir(version.id)
642
+ described = json.loads(store.get(folder + version.dataset.path))
643
+ there = set(store.list(folder + layout.PROFILES_DIR))
644
+ return [
645
+ table
646
+ for table in profiles.tables(described)
647
+ if folder + profiles.path(table["schema"], table["name"]) not in there
648
+ ]
649
+
650
+
651
+ def add_profiles(store: Store, version: Version, *, setup: Iterable[str] = ()) -> tuple[list[str], list[str]]:
652
+ """Profile the tables of a pushed version that have no profile, and put those profiles.
653
+
654
+ For a version made before its writer profiled tables, or whose profiling
655
+ failed. A profile already there is left as it is. Returns the paths put,
656
+ under the version, and the tables (``schema.name``) that could not be read.
657
+ Refused for a version past retention, which has no catalog to check out.
658
+ """
659
+ wanted = missing_profiles(store, version)
660
+ if not wanted:
661
+ return [], []
662
+ with tempfile.TemporaryDirectory(prefix=".fdl-profile-") as scratch:
663
+ working = Path(scratch) / "work.sqlite"
664
+ checkout(store, working, version=version, local_data=Path(scratch) / "data")
665
+ made = profiles.build(working, wanted, data_path=data_url(store), setup=setup)
666
+ for relative, content in made.items():
667
+ written = Path(scratch) / relative
668
+ written.parent.mkdir(parents=True, exist_ok=True)
669
+ written.write_bytes(content)
670
+ _put_once(store, layout.version_dir(version.id) + relative, written)
671
+ unread = [
672
+ f"{table['schema']}.{table['name']}"
673
+ for table in wanted
674
+ if profiles.path(table["schema"], table["name"]) not in made
675
+ ]
676
+ return sorted(made), unread
677
+
678
+
614
679
  def upload(store: Store, local: Path, version: Version) -> None:
615
680
  """Put a committed version's files, catalog first and the record last.
616
681
 
@@ -1,4 +1,4 @@
1
- """Profiles: a summary of each table, made from the data at commit."""
1
+ """Profiles: a summary of each table, made from the data at push."""
2
2
 
3
3
  from __future__ import annotations
4
4
 
@@ -201,3 +201,131 @@ def test_a_geometry_column_gets_its_extent_and_a_sample(repository) -> None:
201
201
  assert got["geometry"]["types"] == ["POINT"]
202
202
  assert len(got["geometry"]["sample"]["features"]) == 3
203
203
  assert [r.keys() for r in got["head"]][0] == {"id"}
204
+
205
+
206
+ def test_profile_adds_the_profiles_a_pushed_version_lacks(repository) -> None:
207
+ store, working = repository
208
+ write(working, store, "CREATE TABLE lake.main.towns AS SELECT i AS id FROM range(4) t(i)")
209
+ write(working, store, "CREATE TABLE lake.main.other AS SELECT 1 AS id")
210
+ commit(working, "-m", "m", "--author", "t")
211
+ head = repo.head(store)
212
+ kept = store.get(f"{layout.version_dir(head.id)}profiles/main/other.json")
213
+ # A version made before its writer profiled: no profile for towns.
214
+ (Path(store.url) / f"{layout.version_dir(head.id)}profiles/main/towns.json").unlink()
215
+
216
+ output = run("profile", store.url)
217
+ assert "put profiles/main/towns.json" in output
218
+ assert "Profiled 1 table(s)" in output
219
+ assert profile_of(store)["rows"] == 4
220
+ assert store.get(f"{layout.version_dir(head.id)}profiles/main/other.json") == kept
221
+ assert "Profiled 0 table(s)" in run("profile", store.url)
222
+
223
+
224
+ def test_push_no_profile_uploads_none(repository) -> None:
225
+ store, working = repository
226
+ write(working, store, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
227
+ run("commit", working, "-m", "m", "--author", "t")
228
+ run("push", working, "--no-profile")
229
+ assert store.list(f"{layout.version_dir(repo.head(store).id)}profiles/") == []
230
+ assert "Profiled 1 table(s)" in run("profile", store.url)
231
+ assert profile_of(store)["rows"] == 1
232
+
233
+
234
+ def test_push_skips_profiles_when_the_working_catalog_changed_after_the_commit(repository) -> None:
235
+ store, working = repository
236
+ write(working, store, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
237
+ run("commit", working, "-m", "m", "--author", "t")
238
+ write(working, store, "INSERT INTO lake.main.towns VALUES (2)")
239
+ result = runner.invoke(app, ["push", str(working)])
240
+ assert result.exit_code == 0, result.output
241
+ assert "changed after the commit" in result.output
242
+ assert store.list(f"{layout.version_dir(repo.head(store).id)}profiles/") == []
243
+
244
+
245
+ def test_push_goes_without_profiles_when_the_working_catalog_is_gone(repository) -> None:
246
+ store, working = repository
247
+ write(working, store, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
248
+ run("commit", working, "-m", "m", "--author", "t")
249
+ working.unlink()
250
+ result = runner.invoke(app, ["push", str(working)])
251
+ assert result.exit_code == 0, result.output
252
+ assert "were not profiled" in result.output
253
+ assert store.list(f"{layout.version_dir(repo.head(store).id)}profiles/") == []
254
+
255
+
256
+ def test_push_profiles_the_tables_a_stopped_push_left_without(repository) -> None:
257
+ store, working = repository
258
+ write(working, store, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
259
+ write(working, store, "CREATE TABLE lake.main.roads AS SELECT 1 AS id")
260
+ run("commit", working, "-m", "m", "--author", "t")
261
+ pending = next((working.parent / f"{working.name}.pending").iterdir())
262
+ partial = pending / "profiles" / "main" / "towns.json"
263
+ partial.parent.mkdir(parents=True)
264
+ partial.write_bytes(b'{"rows": 1, "columns": []}')
265
+ run("push", working)
266
+ assert profile_of(store, "roads")["rows"] == 1
267
+
268
+
269
+ def test_a_view_is_profiled(repository) -> None:
270
+ store, working = repository
271
+ write(working, store, "CREATE TABLE lake.main.towns AS SELECT i AS id FROM range(5) t(i)")
272
+ write(working, store, "CREATE VIEW lake.main.big AS SELECT id FROM lake.main.towns WHERE id >= 2")
273
+ commit(working, "-m", "m", "--author", "t")
274
+ got = profile_of(store, "big")
275
+ assert got["rows"] == 3
276
+ assert got["columns"][0]["min"] == 2
277
+
278
+
279
+ def _spy_builds(monkeypatch) -> list[str]:
280
+ built: list[str] = []
281
+ original = profiles.build
282
+
283
+ def spy(working, wanted, **kwargs):
284
+ wanted = list(wanted)
285
+ built.extend(t["name"] for t in wanted)
286
+ return original(working, wanted, **kwargs)
287
+
288
+ monkeypatch.setattr(profiles, "build", spy)
289
+ return built
290
+
291
+
292
+ def test_a_view_keeps_its_profile_only_while_no_table_changes(repository, monkeypatch) -> None:
293
+ store, working = repository
294
+ write(working, store, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
295
+ write(working, store, "CREATE VIEW lake.main.big AS SELECT id FROM lake.main.towns")
296
+ commit(working, "-m", "m", "--author", "t")
297
+ built = _spy_builds(monkeypatch)
298
+
299
+ run("commit", working, "-m", "same", "--author", "t", "--allow-empty")
300
+ run("push", working)
301
+ assert built == []
302
+
303
+ write(working, store, "INSERT INTO lake.main.towns VALUES (2)")
304
+ commit(working, "-m", "again", "--author", "t")
305
+ assert sorted(built) == ["big", "towns"]
306
+ assert profile_of(store, "big")["rows"] == 2
307
+
308
+
309
+ def test_a_view_over_a_view_is_profiled_again_when_that_view_changes(repository, monkeypatch) -> None:
310
+ store, working = repository
311
+ write(working, store, "CREATE TABLE lake.main.towns AS SELECT i AS id FROM range(5) t(i)")
312
+ write(working, store, "CREATE VIEW lake.main.big AS SELECT id FROM lake.main.towns WHERE id >= 2")
313
+ write(working, store, "CREATE VIEW lake.main.top AS SELECT id FROM lake.main.big")
314
+ commit(working, "-m", "m", "--author", "t")
315
+ assert profile_of(store, "top")["rows"] == 3
316
+
317
+ write(working, store, "CREATE OR REPLACE VIEW lake.main.big AS SELECT id FROM lake.main.towns WHERE id >= 4")
318
+ commit(working, "-m", "again", "--author", "t")
319
+ assert profile_of(store, "top")["rows"] == 1
320
+
321
+
322
+ def test_a_wide_table_is_measured_a_few_columns_at_a_time(repository, monkeypatch) -> None:
323
+ store, working = repository
324
+ monkeypatch.setattr(profiles, "COLUMNS_PER_QUERY", 2)
325
+ write(working, store, "CREATE TABLE lake.main.towns AS SELECT i AS a, i + 1 AS b, i + 2 AS c FROM range(3) t(i)")
326
+ commit(working, "-m", "m", "--author", "t")
327
+ assert [(c["name"], c["min"], c["max"]) for c in profile_of(store)["columns"]] == [
328
+ ("a", 0, 2),
329
+ ("b", 1, 3),
330
+ ("c", 2, 4),
331
+ ]
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes