fdl-cli 0.20.1__tar.gz → 0.21.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/CHANGELOG.md +28 -0
  2. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/PKG-INFO +1 -1
  3. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/reference/cli.md +3 -3
  4. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/spec/versions.md +2 -2
  5. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/reference/cli.md +3 -3
  6. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/spec/versions.md +2 -2
  7. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/pyproject.toml +1 -1
  8. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/src/fdl/cli.py +6 -6
  9. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/src/fdl/profiles.py +21 -4
  10. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/src/fdl/repo.py +74 -19
  11. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/src/fdl/stores.py +19 -0
  12. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/tests/test_http.py +36 -0
  13. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/tests/test_objects.py +34 -0
  14. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/tests/test_profiles.py +27 -0
  15. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/.github/workflows/docs.yml +0 -0
  16. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/.github/workflows/publish.yml +0 -0
  17. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/.github/workflows/release.yml +0 -0
  18. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/.gitignore +0 -0
  19. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/.node-version +0 -0
  20. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/.python-version +0 -0
  21. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/CLAUDE.md +0 -0
  22. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/CONTRIBUTING.md +0 -0
  23. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/LICENSE +0 -0
  24. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/README.md +0 -0
  25. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/blume.config.ts +0 -0
  26. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/concepts/meta.ts +0 -0
  27. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/concepts/repositories.md +0 -0
  28. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/concepts/versions-and-refs.md +0 -0
  29. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/concepts/working-catalogs.md +0 -0
  30. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/getting-started.md +0 -0
  31. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/guides/building-a-host.md +0 -0
  32. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/guides/meta.ts +0 -0
  33. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/guides/mirrors.md +0 -0
  34. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/guides/public-and-private.md +0 -0
  35. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/guides/reading-without-duckdb.md +0 -0
  36. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/guides/serving-over-http.md +0 -0
  37. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/guides/stores.md +0 -0
  38. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/index.md +0 -0
  39. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/internals/ducklake-formats.md +0 -0
  40. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/internals/from-write-to-read.md +0 -0
  41. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/internals/inside-a-repository.md +0 -0
  42. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/internals/meta.ts +0 -0
  43. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/concepts/meta.ts +0 -0
  44. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/concepts/repositories.md +0 -0
  45. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/concepts/versions-and-refs.md +0 -0
  46. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/concepts/working-catalogs.md +0 -0
  47. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/getting-started.md +0 -0
  48. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/guides/building-a-host.md +0 -0
  49. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/guides/meta.ts +0 -0
  50. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/guides/mirrors.md +0 -0
  51. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/guides/public-and-private.md +0 -0
  52. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/guides/reading-without-duckdb.md +0 -0
  53. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/guides/serving-over-http.md +0 -0
  54. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/guides/stores.md +0 -0
  55. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/index.md +0 -0
  56. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/internals/ducklake-formats.md +0 -0
  57. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/internals/from-write-to-read.md +0 -0
  58. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/internals/inside-a-repository.md +0 -0
  59. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/internals/meta.ts +0 -0
  60. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/reference/meta.ts +0 -0
  61. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/reference/python.md +0 -0
  62. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/spec/data-files.md +0 -0
  63. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/spec/http.md +0 -0
  64. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/spec/index.md +0 -0
  65. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/spec/layout.md +0 -0
  66. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/spec/meta.ts +0 -0
  67. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/spec/reading.md +0 -0
  68. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/spec/refs.md +0 -0
  69. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/ja/spec/writing.md +0 -0
  70. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/reference/meta.ts +0 -0
  71. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/reference/python.md +0 -0
  72. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/spec/data-files.md +0 -0
  73. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/spec/http.md +0 -0
  74. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/spec/index.md +0 -0
  75. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/spec/layout.md +0 -0
  76. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/spec/meta.ts +0 -0
  77. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/spec/reading.md +0 -0
  78. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/spec/refs.md +0 -0
  79. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/docs/spec/writing.md +0 -0
  80. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/package-lock.json +0 -0
  81. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/package.json +0 -0
  82. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/public/CNAME +0 -0
  83. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/src/fdl/__init__.py +0 -0
  84. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/src/fdl/catalog.py +0 -0
  85. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/src/fdl/dataset.py +0 -0
  86. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/src/fdl/layout.py +0 -0
  87. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/src/fdl/objects.py +0 -0
  88. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/src/fdl/py.typed +0 -0
  89. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/src/fdl/refs.py +0 -0
  90. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/src/fdl/store.py +0 -0
  91. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/src/fdl/version.py +0 -0
  92. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/tests/http_repo.py +0 -0
  93. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/tests/test_cli.py +0 -0
  94. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/tests/test_dataset.py +0 -0
  95. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/tests/test_repo.py +0 -0
  96. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/tests/test_stores.py +0 -0
  97. {fdl_cli-0.20.1 → fdl_cli-0.21.1}/uv.lock +0 -0
@@ -1,6 +1,34 @@
1
1
  # CHANGELOG
2
2
 
3
3
 
4
+ ## v0.21.1 (2026-11-08)
5
+
6
+ ### Bug Fixes
7
+
8
+ - Retry a data file upload when storage answers 5xx or 429
9
+ ([#43](https://github.com/queria-io/fdl/pull/43),
10
+ [`2fcaaf5`](https://github.com/queria-io/fdl/commit/2fcaaf56953237a4c7ae6c042d1b607e09752749))
11
+
12
+
13
+ ## v0.21.0 (2026-11-06)
14
+
15
+ ### Bug Fixes
16
+
17
+ - **withdraw**: Delete the assets only withdrawn versions reach
18
+ ([#41](https://github.com/queria-io/fdl/pull/41),
19
+ [`33a452e`](https://github.com/queria-io/fdl/commit/33a452e2de9ab0640a32a6550fd21851ba4dc038))
20
+
21
+ A version reaches the assets its tree.json lists and those its asset columns hold. Withdrawal now
22
+ deletes those no kept version reaches, before the data files, so a run that stops half way can
23
+ still work them out.
24
+
25
+ ### Features
26
+
27
+ - **profiles**: List every value of a column with up to 2,000 distinct values
28
+ ([#42](https://github.com/queria-io/fdl/pull/42),
29
+ [`50a9333`](https://github.com/queria-io/fdl/commit/50a9333fc7b1d2fae990d555ed1393d11d01328c))
30
+
31
+
4
32
  ## v0.20.1 (2026-11-06)
5
33
 
6
34
  ### Bug Fixes
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: fdl-cli
3
- Version: 0.20.1
3
+ Version: 0.21.1
4
4
  Summary: Versions of a DuckLake catalog on object storage, kept the way Git keeps commits
5
5
  Project-URL: Homepage, https://fdl.flo8s.com/
6
6
  Project-URL: Documentation, https://fdl.flo8s.com/
@@ -108,11 +108,11 @@ fdl tag <remote> <name> --delete
108
108
  fdl withdraw <remote> <version>... [--yes]
109
109
  ```
110
110
 
111
- 間違って公開した個人データなど、公開してはいけないデータを持つバージョンを取り下げます([取り下げ](/ja/spec/data-files#取り下げ))。先にそのデータを除いたバージョンをコミットし、該当するバージョンを指すブランチが無い状態にします。そのうえでこのコマンドが、該当するバージョンを指すタグ、そのカタログが参照し残すバージョンが読まないデータファイル、カタログを消し、`"reason": "withdrawn"` の `expired.json` を置きます。`version.json` は残します。保持期間を過ぎたバージョンを含めると、そのカタログはもう無いので、有効なバージョンのどれも読まないデータファイルを、猶予期間を待たずにすべて消します。
111
+ 間違って公開した個人データなど、公開してはいけないデータを持つバージョンを取り下げます([取り下げ](/ja/spec/data-files#取り下げ))。先にそのデータを除いたバージョンをコミットし、該当するバージョンを指すブランチが無い状態にします。そのうえでこのコマンドが、該当するバージョンを指すタグ、そのバージョンが参照し残すバージョンが参照しないデータファイルとアセット、カタログを消し、`"reason": "withdrawn"` の `expired.json` を置きます。`version.json` は残します。保持期間を過ぎたバージョンを含めると、そのカタログはもう無いので、有効なバージョンのどれも参照しないデータファイルとアセットを、猶予期間を待たずにすべて消します。
112
112
 
113
- アセットは、まだ消しません。データを持つものは `assets/` から手で消してください。
113
+ バージョンが参照するアセットは、`tree.json` に載っているものと、アセット列が持つ sha256 です。
114
114
 
115
- `--yes` を付けないと、消すものを表示するだけです。残すバージョンが読んでいるファイルは残すものとして表示します。それがそのデータを持つなら、そのバージョンも取り下げます。ブランチかリクエストが該当するバージョンを指しているあいだは断られます。
115
+ `--yes` を付けないと、消すものを表示するだけです。残すバージョンが参照しているファイルは残すものとして表示します。それがそのデータを持つなら、そのバージョンも取り下げます。ブランチかリクエストが該当するバージョンを指しているあいだは断られます。
116
116
 
117
117
  ## fdl expire
118
118
 
@@ -191,12 +191,12 @@ profile は、`ducklake.snapshot_id` の時点の 1 つのテーブルの要約
191
191
 
192
192
  - `rows` はテーブルの行数。`columns` にはカタログの順にすべての列を並べ、`name` と `type`(DuckLake の型名)は `dataset.json` と同じにする
193
193
  - 図形の列を除くすべての列が `null_count` を持つ。数値・テキスト・日付と時刻の列は `distinct_count` と `values` を持つ。大きなテーブルの `distinct_count` は推定値のことがある
194
- - `values` は、値(テキスト)とその値を持つ行の数を、多い順、同数なら値の順に並べる。列が持つ異なる値が 1,000 以下なら全部を並べ、そのとき `distinct_count` は正確な数になる。読み手は、`values` の長さが `distinct_count` と等しいことで全部の一覧だと判定する。それより多い列は上位 10 件だけを並べる。ホストは全部の一覧を、地域コードや名前などの値で検索できる
194
+ - `values` は、値(テキスト)とその値を持つ行の数を、多い順、同数なら値の順に並べる。列が持つ異なる値が 2,000 以下なら全部を並べ、そのとき `distinct_count` は正確な数になる。読み手は、`values` の長さが `distinct_count` と等しいことで全部の一覧だと判定する。それより多い列は上位 10 件だけを並べる。ホストは全部の一覧を、地域コードや名前などの値で検索できる
195
195
  - 数値の列は `min`・`max`・`mean`・`std`、四分位数の `q1`・`median`・`q3`、`min` から `max` までを等幅に区切った `histogram` を持つ。日付と時刻の列は `min` と `max` をテキストで持ち、日付とタイムスタンプは `span_days` も持つ。真偽値の列は `true_count` と `false_count` を持つ
196
196
  - `head` は、テーブルが返す先頭の行を JSON のオブジェクトで持つ。図形の列と、値を JSON で表せない列は入れない
197
197
  - `geometry` は、書き手が読めたときに、最初の図形の列を説明する。図形を持つ行の数、全体の `bbox`(列の座標で `[xmin, ymin, xmax, ymax]`)、含まれる図形の種類、地図を描くための、無作為に選んで簡略化した図形の GeoJSON `FeatureCollection` を持つ
198
198
  - 書き手は計算できないキーを省いてよい。読み手は知らないキーを無視する
199
- - 書き手は、データファイル(`files.json`)と列が親と同じテーブルには親の profile を使い回す。コミットで読むのは変わったテーブルだけになる
199
+ - 書き手は、データファイル(`files.json`)と列が親と同じテーブルには親の profile を使い回す。コミットで読むのは変わったテーブルだけになる。親の profile の `values` が書き手と異なる上限で書かれているときは、profile を作り直す
200
200
  - profile はバージョンの一部で、そのバージョンの中では変わらない。保持期間から外すときはカタログと一緒に消す
201
201
 
202
202
  ## 固めたカタログ
@@ -108,11 +108,11 @@ fdl tag <remote> <name> --delete
108
108
  fdl withdraw <remote> <version>... [--yes]
109
109
  ```
110
110
 
111
- Removes versions that hold data that must not be published, such as personal data published by mistake (see [Withdrawal](/spec/data-files#withdrawal)). First commit a version without the data, so that no branch points at an affected version. The command then deletes the tags that point at the versions, the data files their catalogs name that no kept version reads, and their catalogs, and puts `expired.json` with `"reason": "withdrawn"`. `version.json` stays. When one of the versions is already past retention, its catalog is gone, so every data file no live version reads is deleted at once, without waiting for the grace period.
111
+ Removes versions that hold data that must not be published, such as personal data published by mistake (see [Withdrawal](/spec/data-files#withdrawal)). First commit a version without the data, so that no branch points at an affected version. The command then deletes the tags that point at the versions, the data files and assets the versions reach that no kept version reaches, and their catalogs, and puts `expired.json` with `"reason": "withdrawn"`. `version.json` stays. When one of the versions is already past retention, its catalog is gone, so every data file and asset no live version reaches is deleted at once, without waiting for the grace period.
112
112
 
113
- Assets are not deleted yet: delete those that hold the data from `assets/` by hand.
113
+ A version reaches the assets its `tree.json` lists and those its asset columns hold.
114
114
 
115
- Without `--yes` it only prints what it would delete. Files a kept version still reads are listed as kept; if they hold the data, withdraw that version too. Refused while a branch or a request points at one of the versions.
115
+ Without `--yes` it only prints what it would delete. Files a kept version still reaches are listed as kept; if they hold the data, withdraw that version too. Refused while a branch or a request points at one of the versions.
116
116
 
117
117
  ## fdl expire
118
118
 
@@ -191,12 +191,12 @@ Each table's profile is at `versions/<ID>/profiles/<schema>/<table>.json`. A tab
191
191
 
192
192
  - `rows` is the table's row count. `columns` lists every column in the catalog's order, with `name` and `type` (DuckLake's type name) as in `dataset.json`
193
193
  - Every column but a geometry one has `null_count`. Numbers, text, dates and times have `distinct_count`, which may be estimated on a large table, and `values`
194
- - `values` lists each value, as text, with the number of rows that hold it, most common first and then by value. When the column holds at most 1,000 distinct values, it lists them all and `distinct_count` is exact: a reader tells a whole list by its length matching `distinct_count`. Otherwise it lists the 10 most common. A host can search a whole list by value, such as a town's code or name
194
+ - `values` lists each value, as text, with the number of rows that hold it, most common first and then by value. When the column holds at most 2,000 distinct values, it lists them all and `distinct_count` is exact: a reader tells a whole list by its length matching `distinct_count`. Otherwise it lists the 10 most common. A host can search a whole list by value, such as a town's code or name
195
195
  - Numbers have `min`, `max`, `mean`, `std`, the quartiles `q1`, `median` and `q3`, and a `histogram` of equal-width buckets between `min` and `max`. Dates and times have `min` and `max` as text, and dates and timestamps `span_days`. Booleans have `true_count` and `false_count`
196
196
  - `head` holds the first rows the table returns, as JSON objects, leaving out geometry and columns whose values have no JSON form
197
197
  - `geometry` describes the first geometry column, when the writer could read it: how many rows have a shape, the `bbox` of all of them (`[xmin, ymin, xmax, ymax]` in the column's coordinates), the geometry types present, and a GeoJSON `FeatureCollection` of a random sample of simplified shapes for drawing a map
198
198
  - A writer may leave out any key it cannot compute. A reader ignores keys it does not know
199
- - A writer reuses its parent's profile for a table whose data files (in `files.json`) and columns are the parent's, so a commit reads only the tables that changed
199
+ - A writer reuses its parent's profile for a table whose data files (in `files.json`) and columns are the parent's, so a commit reads only the tables that changed. It makes the profile again when the parent's `values` follow another limit than its own
200
200
  - Profiles are part of the version and do not change with it. They are deleted with the catalogs when the version is taken out of retention
201
201
 
202
202
  ## The frozen catalog
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "fdl-cli"
3
- version = "0.20.1"
3
+ version = "0.21.1"
4
4
  description = "Versions of a DuckLake catalog on object storage, kept the way Git keeps commits"
5
5
  readme = "README.md"
6
6
  license = "MIT"
@@ -294,14 +294,14 @@ def withdraw(
294
294
  """Remove versions that hold data that must not be published.
295
295
 
296
296
  Move every branch to a version without the data first. This deletes the
297
- tags that point at the versions, the data files no kept version reads,
298
- and the versions' catalogs, at once. Their version.json stays. A version
299
- already past retention has no catalog, so then every data file no live
300
- version reads is deleted.
297
+ tags that point at the versions, the data files and assets no kept version
298
+ reaches, and the versions' catalogs, at once. Their version.json stays. A
299
+ version already past retention has no catalog, so then every data file and
300
+ asset no live version reaches is deleted.
301
301
  """
302
302
  store = open_store(remote)
303
303
  try:
304
- plan = repo.plan_withdrawal(store, versions)
304
+ plan = repo.plan_withdrawal(store, versions, setup=_read_setup(store))
305
305
  for name in plan.tags:
306
306
  typer.echo(f"delete tag {name}")
307
307
  for key in plan.deleted:
@@ -309,7 +309,7 @@ def withdraw(
309
309
  for version_id in plan.versions:
310
310
  typer.echo(f"withdraw {version_id}")
311
311
  for key in plan.still_read:
312
- typer.echo(f"keep file {key} (a kept version reads it)")
312
+ typer.echo(f"keep file {key} (a kept version reaches it)")
313
313
  if not yes:
314
314
  typer.echo("Nothing was changed. Run again with --yes to do this.")
315
315
  return
@@ -25,8 +25,9 @@ GEOMETRY_SAMPLE = 500
25
25
  #: Vertex tolerance of the sampled shapes, in the column's units (about 11 m in degrees).
26
26
  SIMPLIFY_TOLERANCE = 0.0001
27
27
  HISTOGRAM_BUCKETS = 20
28
- #: A column with at most this many distinct values lists them all in `values`.
29
- ALL_VALUES = 1000
28
+ #: A column with at most this many distinct values lists them all in `values`: enough for
29
+ #: a column of codes or names that covers a whole country, such as its municipalities.
30
+ ALL_VALUES = 2000
30
31
  #: Otherwise `values` lists the most common.
31
32
  TOP_VALUES = 10
32
33
  #: Above this many rows, distinct values are estimated rather than counted.
@@ -112,14 +113,30 @@ def build(working: Path, wanted: Iterable[dict[str, Any]], *, data_path: str, se
112
113
 
113
114
 
114
115
  def reusable(previous: bytes, table: dict[str, Any]) -> bool:
115
- """Whether a parent's profile describes ``table`` as it is: the same columns and types."""
116
+ """Whether a parent's profile describes ``table`` as it is and as this writer would.
117
+
118
+ The same columns and types, and `values` listed by the current `ALL_VALUES`:
119
+ a profile written under another limit is made again rather than carried forward.
120
+ """
116
121
  try:
117
122
  held = json.loads(previous).get("columns", [])
118
123
  except ValueError:
119
124
  return False
120
- return [{"name": c.get("name"), "type": c.get("type")} for c in held] == columns_of(table)
125
+ if [{"name": c.get("name"), "type": c.get("type")} for c in held] != columns_of(table):
126
+ return False
127
+ return all(_listed_as_now(c) for c in held)
121
128
 
122
129
 
130
+ def _listed_as_now(column: dict[str, Any]) -> bool:
131
+ values, distinct = column.get("values"), column.get("distinct_count")
132
+ if values is None:
133
+ return True
134
+ if not isinstance(values, list) or not isinstance(distinct, int):
135
+ return False
136
+ if len(values) == distinct:
137
+ return distinct <= ALL_VALUES
138
+ return distinct > ALL_VALUES and len(values) <= TOP_VALUES
139
+
123
140
 
124
141
  def _load_spatial(con) -> bool:
125
142
  try:
@@ -622,18 +622,23 @@ class Withdrawal:
622
622
  versions: tuple[str, ...]
623
623
  #: Tags that point at the versions, deleted first.
624
624
  tags: tuple[str, ...]
625
- #: Data files the versions' catalogs name and no version kept reads.
625
+ #: Shared objects (data files and assets) the versions reach and no kept
626
+ #: version reaches, as store keys.
626
627
  deleted: tuple[str, ...]
627
- #: Data files the versions name that a kept version still reads. Kept: if
628
- #: they hold the data, the version that reads them has to be withdrawn too.
628
+ #: Shared objects the versions reach that a kept version still reaches. Kept:
629
+ #: if they hold the data, the version that reaches them has to be withdrawn too.
629
630
  still_read: tuple[str, ...]
630
631
 
631
632
 
632
- def plan_withdrawal(store: Store, version_ids: Iterable[str]) -> Withdrawal:
633
+ def plan_withdrawal(store: Store, version_ids: Iterable[str], *, setup: Iterable[str] = ()) -> Withdrawal:
633
634
  """Work out a withdrawal (see `withdraw`) without changing anything.
634
635
 
635
636
  Refused while a branch or a request still points at one of the versions:
636
637
  the first step, a new version without the data, is the writer's.
638
+
639
+ Shared objects are data files and assets. ``setup`` runs before a
640
+ catalog is attached to read its asset columns: a ``CREATE SECRET`` for
641
+ data files in private storage, say.
637
642
  """
638
643
  config = read_config(store)
639
644
  affected = tuple(dict.fromkeys(version_ids))
@@ -653,17 +658,28 @@ def plan_withdrawal(store: Store, version_ids: Iterable[str]) -> Withdrawal:
653
658
  )
654
659
  tags.append(name)
655
660
 
661
+ setup = list(setup)
662
+ # Store keys: data files under the data directory, assets under assets/.
656
663
  candidates: set[str] = set()
657
664
  kept: set[str] = set()
658
665
  catalogless = False
659
666
  with tempfile.TemporaryDirectory(prefix="fdl-") as tmp:
660
667
  for version_id in _version_ids(store):
661
668
  if version_id in affected:
669
+ version = read_version(store, version_id)
662
670
  local = _fetch_catalog(store, version_id, Path(tmp))
663
671
  if local is not None:
664
672
  # Every file the catalog has ever named: earlier snapshots
665
673
  # in it may hold the data too.
666
- candidates.update(catalog.data_files(local))
674
+ candidates.update(config.data_dir + key for key in catalog.data_files(local))
675
+ try:
676
+ candidates.update(_reached_assets(store, version, local, setup))
677
+ except objects.ObjectError:
678
+ # A run stopped half way may have deleted the data
679
+ # files that held its asset columns: then, as for a
680
+ # version without a catalog, every asset no live
681
+ # version reaches goes.
682
+ catalogless = True
667
683
  local.unlink()
668
684
  else:
669
685
  catalogless = True
@@ -673,40 +689,79 @@ def plan_withdrawal(store: Store, version_ids: Iterable[str]) -> Withdrawal:
673
689
  version = read_version(store, version_id)
674
690
  # files.json lists what the version reads, without its catalog.
675
691
  indexed = _indexed_files(store, version)
692
+ local = None
693
+ if indexed is None or _asset_columns(store, version):
694
+ local = _fetch_catalog(store, version_id, Path(tmp))
676
695
  if indexed is not None:
677
- kept.update(indexed)
678
- continue
679
- local = _fetch_catalog(store, version_id, Path(tmp))
696
+ kept.update(config.data_dir + key for key in indexed)
697
+ elif local is not None:
698
+ kept.update(config.data_dir + key for key in _read_files(local, version))
699
+ kept.update(_reached_assets(store, version, local, setup))
680
700
  if local is not None:
681
- kept.update(_read_files(local, version))
682
701
  local.unlink()
683
702
  if catalogless:
684
- # A version already past retention no longer says which files it
685
- # named. Those no live version reads are waiting for the grace period
686
- # anyway, so all of them go now.
687
- candidates.update(key[len(config.data_dir) :] for key in store.list(config.data_dir))
703
+ # A version already past retention no longer says which objects it
704
+ # reached. Those no live version reaches are waiting for the grace
705
+ # period anyway, so all of them go now.
706
+ candidates.update(store.list(config.data_dir))
707
+ candidates.update(store.list(layout.ASSETS_DIR))
688
708
  return Withdrawal(
689
709
  versions=affected,
690
710
  tags=tuple(tags),
691
- deleted=tuple(config.data_dir + key for key in sorted(candidates - kept)),
692
- still_read=tuple(config.data_dir + key for key in sorted(candidates & kept)),
711
+ deleted=tuple(sorted(candidates - kept)),
712
+ still_read=tuple(sorted(candidates & kept)),
693
713
  )
694
714
 
695
715
 
716
+ def _asset_columns(store: Store, version: Version) -> list[tuple[str, str, str]]:
717
+ """The asset columns the version's `dataset.json` marks."""
718
+ if version.dataset is None:
719
+ return []
720
+ try:
721
+ described = json.loads(store.get(layout.version_dir(version.id) + version.dataset.path))
722
+ except Exception as exc: # noqa: BLE001 - stores raise their own types
723
+ if is_missing(exc):
724
+ return []
725
+ raise
726
+ return objects.asset_columns(described)
727
+
728
+
729
+ def _reached_assets(store: Store, version: Version, local: Path | None, setup: list[str]) -> set[str]:
730
+ """The asset keys the version reaches: its `tree.json` and its asset columns."""
731
+ reached: set[str] = set()
732
+ if version.tree is not None:
733
+ try:
734
+ listed = json.loads(store.get(layout.version_dir(version.id) + version.tree.path))
735
+ except Exception as exc: # noqa: BLE001 - stores raise their own types
736
+ if not is_missing(exc):
737
+ raise
738
+ else:
739
+ reached.update(objects.asset_key(item["sha256"]) for item in listed["entries"])
740
+ columns = _asset_columns(store, version)
741
+ if columns and local is not None:
742
+ values = objects.asset_values(local, columns, data_path=data_url(store), setup=setup)
743
+ reached.update(objects.asset_key(sha256) for sha256 in values)
744
+ return reached
745
+
746
+
696
747
  def withdraw(store: Store, version_ids: Iterable[str], *, plan: Withdrawal | None = None) -> Withdrawal:
697
748
  """Remove versions that hold data that must not be published.
698
749
 
699
- Unlike retention it takes live versions and deletes their data files at
700
- once. In the spec's order, except that the data files go before the
750
+ Unlike retention it takes live versions and deletes their data files and
751
+ assets at once. In the spec's order: the shared objects go before the
701
752
  catalogs, so a run that stops half way can be run again while the
702
- catalogs still say which files they are. `version.json` stays.
753
+ catalogs still say which objects they are. `version.json` stays.
703
754
  """
704
755
  plan = plan or plan_withdrawal(store, version_ids)
705
756
  for name in plan.tags:
706
757
  refs.delete(store, name)
707
758
  if plan.tags:
708
759
  update_refs_index(store)
709
- store.delete(list(plan.deleted))
760
+ # Assets first: the data files are what says which assets the versions'
761
+ # rows reach, so a run that stops half way can still work them out.
762
+ assets = [key for key in plan.deleted if key.startswith(layout.ASSETS_DIR)]
763
+ store.delete(assets)
764
+ store.delete([key for key in plan.deleted if not key.startswith(layout.ASSETS_DIR)])
710
765
  for version_id in plan.versions:
711
766
  expire(store, version_id, reason="withdrawn")
712
767
  return plan
@@ -380,6 +380,25 @@ class HttpStore:
380
380
  action = action if action is None else {k: v for k, v in action.items() if k != "_until"}
381
381
  self._send_one(key, path, action, content_type)
382
382
  return
383
+ for attempt in range(self.STORAGE_ATTEMPTS):
384
+ try:
385
+ self._put_to_storage(key, path, action, content_type)
386
+ return
387
+ except StoreError as exc:
388
+ # Storage answers a busy moment with 5xx or 429, or drops the
389
+ # connection. The PUT writes the same bytes again, so it is
390
+ # safe to repeat.
391
+ transient = exc.status is None or exc.status == 429 or exc.status >= 500
392
+ if not transient or attempt == self.STORAGE_ATTEMPTS - 1:
393
+ raise
394
+ time.sleep(self.RETRY_DELAY * 2**attempt)
395
+
396
+ #: Tries of one PUT to storage, and the wait before the second (doubled
397
+ #: for each one after).
398
+ STORAGE_ATTEMPTS = 4
399
+ RETRY_DELAY = 1.0
400
+
401
+ def _put_to_storage(self, key: str, path: Path, action: dict, content_type: str | None) -> None:
383
402
  try:
384
403
  with path.open("rb") as body:
385
404
  request = urllib.request.Request(
@@ -2,6 +2,7 @@
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
+ import io
5
6
  import json
6
7
  import urllib.error
7
8
  import urllib.request
@@ -181,3 +182,38 @@ def test_a_rebuild_of_the_same_rows_over_http_is_nothing_to_commit(served, tmp_p
181
182
  assert "Nothing to commit" in result.output
182
183
  assert "could not tell" not in result.output
183
184
  assert len(list(repo.log(local))) == 1
185
+
186
+
187
+ def test_a_busy_storage_answer_is_tried_again(tmp_path: Path, monkeypatch) -> None:
188
+ answers = [500, 503, None]
189
+ sent = []
190
+
191
+ def urlopen(request):
192
+ sent.append(request.full_url)
193
+ code = answers.pop(0)
194
+ if code is not None:
195
+ raise urllib.error.HTTPError(request.full_url, code, "busy", {}, io.BytesIO(b"InternalError"))
196
+ return io.BytesIO()
197
+
198
+ monkeypatch.setattr(urllib.request, "urlopen", urlopen)
199
+ monkeypatch.setattr(HttpStore, "RETRY_DELAY", 0)
200
+ path = tmp_path / "a.parquet"
201
+ path.write_bytes(b"ab")
202
+ HttpStore("https://example.invalid/repo/")._send_one("data/a.parquet", path, {"href": "https://storage.invalid/a"}, None)
203
+ assert sent == ["https://storage.invalid/a"] * 3
204
+
205
+
206
+ def test_a_refused_storage_answer_is_not_tried_again(tmp_path: Path, monkeypatch) -> None:
207
+ sent = []
208
+
209
+ def urlopen(request):
210
+ sent.append(request.full_url)
211
+ raise urllib.error.HTTPError(request.full_url, 403, "denied", {}, io.BytesIO(b""))
212
+
213
+ monkeypatch.setattr(urllib.request, "urlopen", urlopen)
214
+ monkeypatch.setattr(HttpStore, "RETRY_DELAY", 0)
215
+ path = tmp_path / "a.parquet"
216
+ path.write_bytes(b"ab")
217
+ with pytest.raises(StoreError):
218
+ HttpStore("https://example.invalid/repo/")._send_one("data/a.parquet", path, {"href": "https://storage.invalid/a"}, None)
219
+ assert len(sent) == 1
@@ -175,3 +175,37 @@ def test_media_type_does_not_depend_on_the_machine(path: str, expected: str) ->
175
175
  def test_asset_columns_that_cannot_be_read_are_an_object_error(tmp_path: Path) -> None:
176
176
  with pytest.raises(objects.ObjectError, match="could not read the asset columns"):
177
177
  objects.asset_values(tmp_path / "missing.ducklake", [("main", "t", "c")], data_path=str(tmp_path) + "/")
178
+
179
+
180
+ def test_withdrawal_deletes_the_assets_only_the_withdrawn_versions_reach(tmp_path: Path, repository) -> None:
181
+ store, working = repository
182
+ folder = tmp_path / "files"
183
+ folder.mkdir()
184
+ (folder / "kept.pdf").write_bytes(b"%PDF-kept")
185
+ (folder / "leak.pdf").write_bytes(b"%PDF-leak")
186
+ (tmp_path / "files.csv").write_text(run("assets", folder))
187
+ (tmp_path / "README.md").write_text("# Reports\n")
188
+ (tmp_path / "dataset.yml").write_text(
189
+ "name: reports\ntitle: Reports\ntables:\n"
190
+ " - schema: main\n name: reports\n fields:\n - name: sha256\n asset: true\n"
191
+ )
192
+ write(
193
+ working,
194
+ store,
195
+ f"CREATE TABLE lake.main.reports AS SELECT path, sha256 FROM read_csv('{tmp_path / 'files.csv'}', all_varchar = true)",
196
+ )
197
+ run("commit", working, "-m", "leak", "--author", "t", "--assets", folder)
198
+ leaked = repo.head(store, "refs/heads/main")
199
+ write(working, store, "DELETE FROM lake.main.reports WHERE path = 'leak.pdf'")
200
+ run("commit", working, "-m", "clean", "--author", "t")
201
+ rows = {row["path"]: row["sha256"] for row in csv.DictReader(io.StringIO(run("assets", folder)))}
202
+ readme = objects.asset_key(objects.sha256_of(tmp_path / "README.md"))
203
+
204
+ plan = repo.plan_withdrawal(store, [leaked.id])
205
+ assert objects.asset_key(rows["leak.pdf"]) in plan.deleted
206
+ assert {objects.asset_key(rows["kept.pdf"]), readme} <= set(plan.still_read)
207
+
208
+ run("withdraw", store.root, leaked.id, "--yes")
209
+ assert store.head(objects.asset_key(rows["leak.pdf"])) is None
210
+ assert store.head(objects.asset_key(rows["kept.pdf"])) is not None
211
+ assert store.head(readme) is not None
@@ -76,6 +76,33 @@ def test_many_values_list_only_the_most_common(repository) -> None:
76
76
  assert code["distinct_count"] > profiles.ALL_VALUES
77
77
 
78
78
 
79
+ def test_a_column_at_the_limit_lists_every_value(repository) -> None:
80
+ store, working = repository
81
+ write(working, store, f"CREATE TABLE lake.main.towns AS SELECT 'v' || i AS code FROM range({profiles.ALL_VALUES}) t(i)")
82
+ run("commit", working, "-m", "m", "--author", "t")
83
+ code = profile_of(store)["columns"][0]
84
+ assert len(code["values"]) == code["distinct_count"] == profiles.ALL_VALUES
85
+
86
+
87
+ @pytest.mark.parametrize(
88
+ ("values", "distinct", "kept"),
89
+ [
90
+ (3, 3, True),
91
+ (profiles.TOP_VALUES, profiles.ALL_VALUES + 1, True),
92
+ # Written under a lower limit: a short list of a column now listed whole.
93
+ (profiles.TOP_VALUES, profiles.ALL_VALUES - 1, False),
94
+ # Written under a higher limit: a whole list longer than now allowed.
95
+ (profiles.ALL_VALUES + 1, profiles.ALL_VALUES + 1, False),
96
+ ],
97
+ )
98
+ def test_a_profile_written_under_another_limit_is_made_again(values: int, distinct: int, kept: bool) -> None:
99
+ table = {"fields": [{"name": "code", "type": "VARCHAR"}]}
100
+ column = {"name": "code", "type": "VARCHAR", "distinct_count": distinct}
101
+ column["values"] = [{"value": f"v{i}", "count": 1} for i in range(values)]
102
+ previous = json.dumps({"rows": distinct, "columns": [column]}).encode()
103
+ assert profiles.reusable(previous, table) is kept
104
+
105
+
79
106
  def test_an_unchanged_table_keeps_its_parents_profile(repository, monkeypatch) -> None:
80
107
  store, working = repository
81
108
  write(working, store, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes