fdl-cli 0.20.1__tar.gz → 0.21.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/CHANGELOG.md +19 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/PKG-INFO +1 -1
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/reference/cli.md +3 -3
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/spec/versions.md +2 -2
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/reference/cli.md +3 -3
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/spec/versions.md +2 -2
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/pyproject.toml +1 -1
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/src/fdl/cli.py +6 -6
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/src/fdl/profiles.py +21 -4
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/src/fdl/repo.py +74 -19
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/tests/test_objects.py +34 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/tests/test_profiles.py +27 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/.github/workflows/docs.yml +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/.github/workflows/publish.yml +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/.github/workflows/release.yml +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/.gitignore +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/.node-version +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/.python-version +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/CLAUDE.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/CONTRIBUTING.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/LICENSE +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/README.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/blume.config.ts +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/concepts/meta.ts +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/concepts/repositories.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/concepts/versions-and-refs.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/concepts/working-catalogs.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/getting-started.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/guides/building-a-host.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/guides/meta.ts +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/guides/mirrors.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/guides/public-and-private.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/guides/reading-without-duckdb.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/guides/serving-over-http.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/guides/stores.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/index.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/internals/from-write-to-read.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/internals/inside-a-repository.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/internals/meta.ts +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/concepts/meta.ts +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/concepts/repositories.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/concepts/versions-and-refs.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/concepts/working-catalogs.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/getting-started.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/guides/building-a-host.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/guides/meta.ts +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/guides/mirrors.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/guides/public-and-private.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/guides/reading-without-duckdb.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/guides/serving-over-http.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/guides/stores.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/index.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/internals/from-write-to-read.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/internals/inside-a-repository.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/internals/meta.ts +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/reference/meta.ts +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/reference/python.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/spec/data-files.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/spec/http.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/spec/index.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/spec/layout.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/spec/meta.ts +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/spec/reading.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/spec/refs.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/ja/spec/writing.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/reference/meta.ts +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/reference/python.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/spec/data-files.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/spec/http.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/spec/index.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/spec/layout.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/spec/meta.ts +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/spec/reading.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/spec/refs.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/docs/spec/writing.md +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/package-lock.json +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/package.json +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/public/CNAME +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/src/fdl/__init__.py +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/src/fdl/catalog.py +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/src/fdl/dataset.py +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/src/fdl/layout.py +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/src/fdl/objects.py +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/src/fdl/py.typed +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/src/fdl/refs.py +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/src/fdl/store.py +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/src/fdl/stores.py +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/src/fdl/version.py +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/tests/http_repo.py +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/tests/test_cli.py +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/tests/test_dataset.py +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/tests/test_http.py +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/tests/test_repo.py +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/tests/test_stores.py +0 -0
- {fdl_cli-0.20.1 → fdl_cli-0.21.0}/uv.lock +0 -0
|
@@ -1,6 +1,25 @@
|
|
|
1
1
|
# CHANGELOG
|
|
2
2
|
|
|
3
3
|
|
|
4
|
+
## v0.21.0 (2026-11-06)
|
|
5
|
+
|
|
6
|
+
### Bug Fixes
|
|
7
|
+
|
|
8
|
+
- **withdraw**: Delete the assets only withdrawn versions reach
|
|
9
|
+
([#41](https://github.com/queria-io/fdl/pull/41),
|
|
10
|
+
[`33a452e`](https://github.com/queria-io/fdl/commit/33a452e2de9ab0640a32a6550fd21851ba4dc038))
|
|
11
|
+
|
|
12
|
+
A version reaches the assets its tree.json lists and those its asset columns hold. Withdrawal now
|
|
13
|
+
deletes those no kept version reaches, before the data files, so a run that stops half way can
|
|
14
|
+
still work them out.
|
|
15
|
+
|
|
16
|
+
### Features
|
|
17
|
+
|
|
18
|
+
- **profiles**: List every value of a column with up to 2,000 distinct values
|
|
19
|
+
([#42](https://github.com/queria-io/fdl/pull/42),
|
|
20
|
+
[`50a9333`](https://github.com/queria-io/fdl/commit/50a9333fc7b1d2fae990d555ed1393d11d01328c))
|
|
21
|
+
|
|
22
|
+
|
|
4
23
|
## v0.20.1 (2026-11-06)
|
|
5
24
|
|
|
6
25
|
### Bug Fixes
|
|
@@ -108,11 +108,11 @@ fdl tag <remote> <name> --delete
|
|
|
108
108
|
fdl withdraw <remote> <version>... [--yes]
|
|
109
109
|
```
|
|
110
110
|
|
|
111
|
-
間違って公開した個人データなど、公開してはいけないデータを持つバージョンを取り下げます([取り下げ](/ja/spec/data-files#取り下げ)
|
|
111
|
+
間違って公開した個人データなど、公開してはいけないデータを持つバージョンを取り下げます([取り下げ](/ja/spec/data-files#取り下げ))。先にそのデータを除いたバージョンをコミットし、該当するバージョンを指すブランチが無い状態にします。そのうえでこのコマンドが、該当するバージョンを指すタグ、そのバージョンが参照し残すバージョンが参照しないデータファイルとアセット、カタログを消し、`"reason": "withdrawn"` の `expired.json` を置きます。`version.json` は残します。保持期間を過ぎたバージョンを含めると、そのカタログはもう無いので、有効なバージョンのどれも参照しないデータファイルとアセットを、猶予期間を待たずにすべて消します。
|
|
112
112
|
|
|
113
|
-
|
|
113
|
+
バージョンが参照するアセットは、`tree.json` に載っているものと、アセット列が持つ sha256 です。
|
|
114
114
|
|
|
115
|
-
`--yes`
|
|
115
|
+
`--yes` を付けないと、消すものを表示するだけです。残すバージョンが参照しているファイルは残すものとして表示します。それがそのデータを持つなら、そのバージョンも取り下げます。ブランチかリクエストが該当するバージョンを指しているあいだは断られます。
|
|
116
116
|
|
|
117
117
|
## fdl expire
|
|
118
118
|
|
|
@@ -191,12 +191,12 @@ profile は、`ducklake.snapshot_id` の時点の 1 つのテーブルの要約
|
|
|
191
191
|
|
|
192
192
|
- `rows` はテーブルの行数。`columns` にはカタログの順にすべての列を並べ、`name` と `type`(DuckLake の型名)は `dataset.json` と同じにする
|
|
193
193
|
- 図形の列を除くすべての列が `null_count` を持つ。数値・テキスト・日付と時刻の列は `distinct_count` と `values` を持つ。大きなテーブルの `distinct_count` は推定値のことがある
|
|
194
|
-
- `values` は、値(テキスト)とその値を持つ行の数を、多い順、同数なら値の順に並べる。列が持つ異なる値が
|
|
194
|
+
- `values` は、値(テキスト)とその値を持つ行の数を、多い順、同数なら値の順に並べる。列が持つ異なる値が 2,000 以下なら全部を並べ、そのとき `distinct_count` は正確な数になる。読み手は、`values` の長さが `distinct_count` と等しいことで全部の一覧だと判定する。それより多い列は上位 10 件だけを並べる。ホストは全部の一覧を、地域コードや名前などの値で検索できる
|
|
195
195
|
- 数値の列は `min`・`max`・`mean`・`std`、四分位数の `q1`・`median`・`q3`、`min` から `max` までを等幅に区切った `histogram` を持つ。日付と時刻の列は `min` と `max` をテキストで持ち、日付とタイムスタンプは `span_days` も持つ。真偽値の列は `true_count` と `false_count` を持つ
|
|
196
196
|
- `head` は、テーブルが返す先頭の行を JSON のオブジェクトで持つ。図形の列と、値を JSON で表せない列は入れない
|
|
197
197
|
- `geometry` は、書き手が読めたときに、最初の図形の列を説明する。図形を持つ行の数、全体の `bbox`(列の座標で `[xmin, ymin, xmax, ymax]`)、含まれる図形の種類、地図を描くための、無作為に選んで簡略化した図形の GeoJSON `FeatureCollection` を持つ
|
|
198
198
|
- 書き手は計算できないキーを省いてよい。読み手は知らないキーを無視する
|
|
199
|
-
- 書き手は、データファイル(`files.json`)と列が親と同じテーブルには親の profile
|
|
199
|
+
- 書き手は、データファイル(`files.json`)と列が親と同じテーブルには親の profile を使い回す。コミットで読むのは変わったテーブルだけになる。親の profile の `values` が書き手と異なる上限で書かれているときは、profile を作り直す
|
|
200
200
|
- profile はバージョンの一部で、そのバージョンの中では変わらない。保持期間から外すときはカタログと一緒に消す
|
|
201
201
|
|
|
202
202
|
## 固めたカタログ
|
|
@@ -108,11 +108,11 @@ fdl tag <remote> <name> --delete
|
|
|
108
108
|
fdl withdraw <remote> <version>... [--yes]
|
|
109
109
|
```
|
|
110
110
|
|
|
111
|
-
Removes versions that hold data that must not be published, such as personal data published by mistake (see [Withdrawal](/spec/data-files#withdrawal)). First commit a version without the data, so that no branch points at an affected version. The command then deletes the tags that point at the versions, the data files
|
|
111
|
+
Removes versions that hold data that must not be published, such as personal data published by mistake (see [Withdrawal](/spec/data-files#withdrawal)). First commit a version without the data, so that no branch points at an affected version. The command then deletes the tags that point at the versions, the data files and assets the versions reach that no kept version reaches, and their catalogs, and puts `expired.json` with `"reason": "withdrawn"`. `version.json` stays. When one of the versions is already past retention, its catalog is gone, so every data file and asset no live version reaches is deleted at once, without waiting for the grace period.
|
|
112
112
|
|
|
113
|
-
|
|
113
|
+
A version reaches the assets its `tree.json` lists and those its asset columns hold.
|
|
114
114
|
|
|
115
|
-
Without `--yes` it only prints what it would delete. Files a kept version still
|
|
115
|
+
Without `--yes` it only prints what it would delete. Files a kept version still reaches are listed as kept; if they hold the data, withdraw that version too. Refused while a branch or a request points at one of the versions.
|
|
116
116
|
|
|
117
117
|
## fdl expire
|
|
118
118
|
|
|
@@ -191,12 +191,12 @@ Each table's profile is at `versions/<ID>/profiles/<schema>/<table>.json`. A tab
|
|
|
191
191
|
|
|
192
192
|
- `rows` is the table's row count. `columns` lists every column in the catalog's order, with `name` and `type` (DuckLake's type name) as in `dataset.json`
|
|
193
193
|
- Every column but a geometry one has `null_count`. Numbers, text, dates and times have `distinct_count`, which may be estimated on a large table, and `values`
|
|
194
|
-
- `values` lists each value, as text, with the number of rows that hold it, most common first and then by value. When the column holds at most
|
|
194
|
+
- `values` lists each value, as text, with the number of rows that hold it, most common first and then by value. When the column holds at most 2,000 distinct values, it lists them all and `distinct_count` is exact: a reader tells a whole list by its length matching `distinct_count`. Otherwise it lists the 10 most common. A host can search a whole list by value, such as a town's code or name
|
|
195
195
|
- Numbers have `min`, `max`, `mean`, `std`, the quartiles `q1`, `median` and `q3`, and a `histogram` of equal-width buckets between `min` and `max`. Dates and times have `min` and `max` as text, and dates and timestamps `span_days`. Booleans have `true_count` and `false_count`
|
|
196
196
|
- `head` holds the first rows the table returns, as JSON objects, leaving out geometry and columns whose values have no JSON form
|
|
197
197
|
- `geometry` describes the first geometry column, when the writer could read it: how many rows have a shape, the `bbox` of all of them (`[xmin, ymin, xmax, ymax]` in the column's coordinates), the geometry types present, and a GeoJSON `FeatureCollection` of a random sample of simplified shapes for drawing a map
|
|
198
198
|
- A writer may leave out any key it cannot compute. A reader ignores keys it does not know
|
|
199
|
-
- A writer reuses its parent's profile for a table whose data files (in `files.json`) and columns are the parent's, so a commit reads only the tables that changed
|
|
199
|
+
- A writer reuses its parent's profile for a table whose data files (in `files.json`) and columns are the parent's, so a commit reads only the tables that changed. It makes the profile again when the parent's `values` follow another limit than its own
|
|
200
200
|
- Profiles are part of the version and do not change with it. They are deleted with the catalogs when the version is taken out of retention
|
|
201
201
|
|
|
202
202
|
## The frozen catalog
|
|
@@ -294,14 +294,14 @@ def withdraw(
|
|
|
294
294
|
"""Remove versions that hold data that must not be published.
|
|
295
295
|
|
|
296
296
|
Move every branch to a version without the data first. This deletes the
|
|
297
|
-
tags that point at the versions, the data files no kept version
|
|
298
|
-
and the versions' catalogs, at once. Their version.json stays. A
|
|
299
|
-
already past retention has no catalog, so then every data file
|
|
300
|
-
version
|
|
297
|
+
tags that point at the versions, the data files and assets no kept version
|
|
298
|
+
reaches, and the versions' catalogs, at once. Their version.json stays. A
|
|
299
|
+
version already past retention has no catalog, so then every data file and
|
|
300
|
+
asset no live version reaches is deleted.
|
|
301
301
|
"""
|
|
302
302
|
store = open_store(remote)
|
|
303
303
|
try:
|
|
304
|
-
plan = repo.plan_withdrawal(store, versions)
|
|
304
|
+
plan = repo.plan_withdrawal(store, versions, setup=_read_setup(store))
|
|
305
305
|
for name in plan.tags:
|
|
306
306
|
typer.echo(f"delete tag {name}")
|
|
307
307
|
for key in plan.deleted:
|
|
@@ -309,7 +309,7 @@ def withdraw(
|
|
|
309
309
|
for version_id in plan.versions:
|
|
310
310
|
typer.echo(f"withdraw {version_id}")
|
|
311
311
|
for key in plan.still_read:
|
|
312
|
-
typer.echo(f"keep file {key} (a kept version
|
|
312
|
+
typer.echo(f"keep file {key} (a kept version reaches it)")
|
|
313
313
|
if not yes:
|
|
314
314
|
typer.echo("Nothing was changed. Run again with --yes to do this.")
|
|
315
315
|
return
|
|
@@ -25,8 +25,9 @@ GEOMETRY_SAMPLE = 500
|
|
|
25
25
|
#: Vertex tolerance of the sampled shapes, in the column's units (about 11 m in degrees).
|
|
26
26
|
SIMPLIFY_TOLERANCE = 0.0001
|
|
27
27
|
HISTOGRAM_BUCKETS = 20
|
|
28
|
-
#: A column with at most this many distinct values lists them all in `values
|
|
29
|
-
|
|
28
|
+
#: A column with at most this many distinct values lists them all in `values`: enough for
|
|
29
|
+
#: a column of codes or names that covers a whole country, such as its municipalities.
|
|
30
|
+
ALL_VALUES = 2000
|
|
30
31
|
#: Otherwise `values` lists the most common.
|
|
31
32
|
TOP_VALUES = 10
|
|
32
33
|
#: Above this many rows, distinct values are estimated rather than counted.
|
|
@@ -112,14 +113,30 @@ def build(working: Path, wanted: Iterable[dict[str, Any]], *, data_path: str, se
|
|
|
112
113
|
|
|
113
114
|
|
|
114
115
|
def reusable(previous: bytes, table: dict[str, Any]) -> bool:
|
|
115
|
-
"""Whether a parent's profile describes ``table`` as it is
|
|
116
|
+
"""Whether a parent's profile describes ``table`` as it is and as this writer would.
|
|
117
|
+
|
|
118
|
+
The same columns and types, and `values` listed by the current `ALL_VALUES`:
|
|
119
|
+
a profile written under another limit is made again rather than carried forward.
|
|
120
|
+
"""
|
|
116
121
|
try:
|
|
117
122
|
held = json.loads(previous).get("columns", [])
|
|
118
123
|
except ValueError:
|
|
119
124
|
return False
|
|
120
|
-
|
|
125
|
+
if [{"name": c.get("name"), "type": c.get("type")} for c in held] != columns_of(table):
|
|
126
|
+
return False
|
|
127
|
+
return all(_listed_as_now(c) for c in held)
|
|
121
128
|
|
|
122
129
|
|
|
130
|
+
def _listed_as_now(column: dict[str, Any]) -> bool:
|
|
131
|
+
values, distinct = column.get("values"), column.get("distinct_count")
|
|
132
|
+
if values is None:
|
|
133
|
+
return True
|
|
134
|
+
if not isinstance(values, list) or not isinstance(distinct, int):
|
|
135
|
+
return False
|
|
136
|
+
if len(values) == distinct:
|
|
137
|
+
return distinct <= ALL_VALUES
|
|
138
|
+
return distinct > ALL_VALUES and len(values) <= TOP_VALUES
|
|
139
|
+
|
|
123
140
|
|
|
124
141
|
def _load_spatial(con) -> bool:
|
|
125
142
|
try:
|
|
@@ -622,18 +622,23 @@ class Withdrawal:
|
|
|
622
622
|
versions: tuple[str, ...]
|
|
623
623
|
#: Tags that point at the versions, deleted first.
|
|
624
624
|
tags: tuple[str, ...]
|
|
625
|
-
#:
|
|
625
|
+
#: Shared objects (data files and assets) the versions reach and no kept
|
|
626
|
+
#: version reaches, as store keys.
|
|
626
627
|
deleted: tuple[str, ...]
|
|
627
|
-
#:
|
|
628
|
-
#: they hold the data, the version that
|
|
628
|
+
#: Shared objects the versions reach that a kept version still reaches. Kept:
|
|
629
|
+
#: if they hold the data, the version that reaches them has to be withdrawn too.
|
|
629
630
|
still_read: tuple[str, ...]
|
|
630
631
|
|
|
631
632
|
|
|
632
|
-
def plan_withdrawal(store: Store, version_ids: Iterable[str]) -> Withdrawal:
|
|
633
|
+
def plan_withdrawal(store: Store, version_ids: Iterable[str], *, setup: Iterable[str] = ()) -> Withdrawal:
|
|
633
634
|
"""Work out a withdrawal (see `withdraw`) without changing anything.
|
|
634
635
|
|
|
635
636
|
Refused while a branch or a request still points at one of the versions:
|
|
636
637
|
the first step, a new version without the data, is the writer's.
|
|
638
|
+
|
|
639
|
+
Shared objects are data files and assets. ``setup`` runs before a
|
|
640
|
+
catalog is attached to read its asset columns: a ``CREATE SECRET`` for
|
|
641
|
+
data files in private storage, say.
|
|
637
642
|
"""
|
|
638
643
|
config = read_config(store)
|
|
639
644
|
affected = tuple(dict.fromkeys(version_ids))
|
|
@@ -653,17 +658,28 @@ def plan_withdrawal(store: Store, version_ids: Iterable[str]) -> Withdrawal:
|
|
|
653
658
|
)
|
|
654
659
|
tags.append(name)
|
|
655
660
|
|
|
661
|
+
setup = list(setup)
|
|
662
|
+
# Store keys: data files under the data directory, assets under assets/.
|
|
656
663
|
candidates: set[str] = set()
|
|
657
664
|
kept: set[str] = set()
|
|
658
665
|
catalogless = False
|
|
659
666
|
with tempfile.TemporaryDirectory(prefix="fdl-") as tmp:
|
|
660
667
|
for version_id in _version_ids(store):
|
|
661
668
|
if version_id in affected:
|
|
669
|
+
version = read_version(store, version_id)
|
|
662
670
|
local = _fetch_catalog(store, version_id, Path(tmp))
|
|
663
671
|
if local is not None:
|
|
664
672
|
# Every file the catalog has ever named: earlier snapshots
|
|
665
673
|
# in it may hold the data too.
|
|
666
|
-
candidates.update(catalog.data_files(local))
|
|
674
|
+
candidates.update(config.data_dir + key for key in catalog.data_files(local))
|
|
675
|
+
try:
|
|
676
|
+
candidates.update(_reached_assets(store, version, local, setup))
|
|
677
|
+
except objects.ObjectError:
|
|
678
|
+
# A run stopped half way may have deleted the data
|
|
679
|
+
# files that held its asset columns: then, as for a
|
|
680
|
+
# version without a catalog, every asset no live
|
|
681
|
+
# version reaches goes.
|
|
682
|
+
catalogless = True
|
|
667
683
|
local.unlink()
|
|
668
684
|
else:
|
|
669
685
|
catalogless = True
|
|
@@ -673,40 +689,79 @@ def plan_withdrawal(store: Store, version_ids: Iterable[str]) -> Withdrawal:
|
|
|
673
689
|
version = read_version(store, version_id)
|
|
674
690
|
# files.json lists what the version reads, without its catalog.
|
|
675
691
|
indexed = _indexed_files(store, version)
|
|
692
|
+
local = None
|
|
693
|
+
if indexed is None or _asset_columns(store, version):
|
|
694
|
+
local = _fetch_catalog(store, version_id, Path(tmp))
|
|
676
695
|
if indexed is not None:
|
|
677
|
-
kept.update(indexed)
|
|
678
|
-
|
|
679
|
-
|
|
696
|
+
kept.update(config.data_dir + key for key in indexed)
|
|
697
|
+
elif local is not None:
|
|
698
|
+
kept.update(config.data_dir + key for key in _read_files(local, version))
|
|
699
|
+
kept.update(_reached_assets(store, version, local, setup))
|
|
680
700
|
if local is not None:
|
|
681
|
-
kept.update(_read_files(local, version))
|
|
682
701
|
local.unlink()
|
|
683
702
|
if catalogless:
|
|
684
|
-
# A version already past retention no longer says which
|
|
685
|
-
#
|
|
686
|
-
# anyway, so all of them go now.
|
|
687
|
-
candidates.update(
|
|
703
|
+
# A version already past retention no longer says which objects it
|
|
704
|
+
# reached. Those no live version reaches are waiting for the grace
|
|
705
|
+
# period anyway, so all of them go now.
|
|
706
|
+
candidates.update(store.list(config.data_dir))
|
|
707
|
+
candidates.update(store.list(layout.ASSETS_DIR))
|
|
688
708
|
return Withdrawal(
|
|
689
709
|
versions=affected,
|
|
690
710
|
tags=tuple(tags),
|
|
691
|
-
deleted=tuple(
|
|
692
|
-
still_read=tuple(
|
|
711
|
+
deleted=tuple(sorted(candidates - kept)),
|
|
712
|
+
still_read=tuple(sorted(candidates & kept)),
|
|
693
713
|
)
|
|
694
714
|
|
|
695
715
|
|
|
716
|
+
def _asset_columns(store: Store, version: Version) -> list[tuple[str, str, str]]:
|
|
717
|
+
"""The asset columns the version's `dataset.json` marks."""
|
|
718
|
+
if version.dataset is None:
|
|
719
|
+
return []
|
|
720
|
+
try:
|
|
721
|
+
described = json.loads(store.get(layout.version_dir(version.id) + version.dataset.path))
|
|
722
|
+
except Exception as exc: # noqa: BLE001 - stores raise their own types
|
|
723
|
+
if is_missing(exc):
|
|
724
|
+
return []
|
|
725
|
+
raise
|
|
726
|
+
return objects.asset_columns(described)
|
|
727
|
+
|
|
728
|
+
|
|
729
|
+
def _reached_assets(store: Store, version: Version, local: Path | None, setup: list[str]) -> set[str]:
|
|
730
|
+
"""The asset keys the version reaches: its `tree.json` and its asset columns."""
|
|
731
|
+
reached: set[str] = set()
|
|
732
|
+
if version.tree is not None:
|
|
733
|
+
try:
|
|
734
|
+
listed = json.loads(store.get(layout.version_dir(version.id) + version.tree.path))
|
|
735
|
+
except Exception as exc: # noqa: BLE001 - stores raise their own types
|
|
736
|
+
if not is_missing(exc):
|
|
737
|
+
raise
|
|
738
|
+
else:
|
|
739
|
+
reached.update(objects.asset_key(item["sha256"]) for item in listed["entries"])
|
|
740
|
+
columns = _asset_columns(store, version)
|
|
741
|
+
if columns and local is not None:
|
|
742
|
+
values = objects.asset_values(local, columns, data_path=data_url(store), setup=setup)
|
|
743
|
+
reached.update(objects.asset_key(sha256) for sha256 in values)
|
|
744
|
+
return reached
|
|
745
|
+
|
|
746
|
+
|
|
696
747
|
def withdraw(store: Store, version_ids: Iterable[str], *, plan: Withdrawal | None = None) -> Withdrawal:
|
|
697
748
|
"""Remove versions that hold data that must not be published.
|
|
698
749
|
|
|
699
|
-
Unlike retention it takes live versions and deletes their data files
|
|
700
|
-
once. In the spec's order
|
|
750
|
+
Unlike retention it takes live versions and deletes their data files and
|
|
751
|
+
assets at once. In the spec's order: the shared objects go before the
|
|
701
752
|
catalogs, so a run that stops half way can be run again while the
|
|
702
|
-
catalogs still say which
|
|
753
|
+
catalogs still say which objects they are. `version.json` stays.
|
|
703
754
|
"""
|
|
704
755
|
plan = plan or plan_withdrawal(store, version_ids)
|
|
705
756
|
for name in plan.tags:
|
|
706
757
|
refs.delete(store, name)
|
|
707
758
|
if plan.tags:
|
|
708
759
|
update_refs_index(store)
|
|
709
|
-
|
|
760
|
+
# Assets first: the data files are what says which assets the versions'
|
|
761
|
+
# rows reach, so a run that stops half way can still work them out.
|
|
762
|
+
assets = [key for key in plan.deleted if key.startswith(layout.ASSETS_DIR)]
|
|
763
|
+
store.delete(assets)
|
|
764
|
+
store.delete([key for key in plan.deleted if not key.startswith(layout.ASSETS_DIR)])
|
|
710
765
|
for version_id in plan.versions:
|
|
711
766
|
expire(store, version_id, reason="withdrawn")
|
|
712
767
|
return plan
|
|
@@ -175,3 +175,37 @@ def test_media_type_does_not_depend_on_the_machine(path: str, expected: str) ->
|
|
|
175
175
|
def test_asset_columns_that_cannot_be_read_are_an_object_error(tmp_path: Path) -> None:
|
|
176
176
|
with pytest.raises(objects.ObjectError, match="could not read the asset columns"):
|
|
177
177
|
objects.asset_values(tmp_path / "missing.ducklake", [("main", "t", "c")], data_path=str(tmp_path) + "/")
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def test_withdrawal_deletes_the_assets_only_the_withdrawn_versions_reach(tmp_path: Path, repository) -> None:
|
|
181
|
+
store, working = repository
|
|
182
|
+
folder = tmp_path / "files"
|
|
183
|
+
folder.mkdir()
|
|
184
|
+
(folder / "kept.pdf").write_bytes(b"%PDF-kept")
|
|
185
|
+
(folder / "leak.pdf").write_bytes(b"%PDF-leak")
|
|
186
|
+
(tmp_path / "files.csv").write_text(run("assets", folder))
|
|
187
|
+
(tmp_path / "README.md").write_text("# Reports\n")
|
|
188
|
+
(tmp_path / "dataset.yml").write_text(
|
|
189
|
+
"name: reports\ntitle: Reports\ntables:\n"
|
|
190
|
+
" - schema: main\n name: reports\n fields:\n - name: sha256\n asset: true\n"
|
|
191
|
+
)
|
|
192
|
+
write(
|
|
193
|
+
working,
|
|
194
|
+
store,
|
|
195
|
+
f"CREATE TABLE lake.main.reports AS SELECT path, sha256 FROM read_csv('{tmp_path / 'files.csv'}', all_varchar = true)",
|
|
196
|
+
)
|
|
197
|
+
run("commit", working, "-m", "leak", "--author", "t", "--assets", folder)
|
|
198
|
+
leaked = repo.head(store, "refs/heads/main")
|
|
199
|
+
write(working, store, "DELETE FROM lake.main.reports WHERE path = 'leak.pdf'")
|
|
200
|
+
run("commit", working, "-m", "clean", "--author", "t")
|
|
201
|
+
rows = {row["path"]: row["sha256"] for row in csv.DictReader(io.StringIO(run("assets", folder)))}
|
|
202
|
+
readme = objects.asset_key(objects.sha256_of(tmp_path / "README.md"))
|
|
203
|
+
|
|
204
|
+
plan = repo.plan_withdrawal(store, [leaked.id])
|
|
205
|
+
assert objects.asset_key(rows["leak.pdf"]) in plan.deleted
|
|
206
|
+
assert {objects.asset_key(rows["kept.pdf"]), readme} <= set(plan.still_read)
|
|
207
|
+
|
|
208
|
+
run("withdraw", store.root, leaked.id, "--yes")
|
|
209
|
+
assert store.head(objects.asset_key(rows["leak.pdf"])) is None
|
|
210
|
+
assert store.head(objects.asset_key(rows["kept.pdf"])) is not None
|
|
211
|
+
assert store.head(readme) is not None
|
|
@@ -76,6 +76,33 @@ def test_many_values_list_only_the_most_common(repository) -> None:
|
|
|
76
76
|
assert code["distinct_count"] > profiles.ALL_VALUES
|
|
77
77
|
|
|
78
78
|
|
|
79
|
+
def test_a_column_at_the_limit_lists_every_value(repository) -> None:
|
|
80
|
+
store, working = repository
|
|
81
|
+
write(working, store, f"CREATE TABLE lake.main.towns AS SELECT 'v' || i AS code FROM range({profiles.ALL_VALUES}) t(i)")
|
|
82
|
+
run("commit", working, "-m", "m", "--author", "t")
|
|
83
|
+
code = profile_of(store)["columns"][0]
|
|
84
|
+
assert len(code["values"]) == code["distinct_count"] == profiles.ALL_VALUES
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@pytest.mark.parametrize(
|
|
88
|
+
("values", "distinct", "kept"),
|
|
89
|
+
[
|
|
90
|
+
(3, 3, True),
|
|
91
|
+
(profiles.TOP_VALUES, profiles.ALL_VALUES + 1, True),
|
|
92
|
+
# Written under a lower limit: a short list of a column now listed whole.
|
|
93
|
+
(profiles.TOP_VALUES, profiles.ALL_VALUES - 1, False),
|
|
94
|
+
# Written under a higher limit: a whole list longer than now allowed.
|
|
95
|
+
(profiles.ALL_VALUES + 1, profiles.ALL_VALUES + 1, False),
|
|
96
|
+
],
|
|
97
|
+
)
|
|
98
|
+
def test_a_profile_written_under_another_limit_is_made_again(values: int, distinct: int, kept: bool) -> None:
|
|
99
|
+
table = {"fields": [{"name": "code", "type": "VARCHAR"}]}
|
|
100
|
+
column = {"name": "code", "type": "VARCHAR", "distinct_count": distinct}
|
|
101
|
+
column["values"] = [{"value": f"v{i}", "count": 1} for i in range(values)]
|
|
102
|
+
previous = json.dumps({"rows": distinct, "columns": [column]}).encode()
|
|
103
|
+
assert profiles.reusable(previous, table) is kept
|
|
104
|
+
|
|
105
|
+
|
|
79
106
|
def test_an_unchanged_table_keeps_its_parents_profile(repository, monkeypatch) -> None:
|
|
80
107
|
store, working = repository
|
|
81
108
|
write(working, store, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|