fdl-cli 0.20.0__tar.gz → 0.21.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/CHANGELOG.md +28 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/PKG-INFO +1 -1
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/reference/cli.md +3 -3
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/spec/data-files.md +1 -1
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/spec/versions.md +2 -2
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/reference/cli.md +3 -3
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/spec/data-files.md +1 -1
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/spec/versions.md +2 -2
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/pyproject.toml +1 -1
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/cli.py +6 -4
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/profiles.py +21 -4
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/repo.py +78 -15
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/tests/test_http.py +2 -1
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/tests/test_objects.py +34 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/tests/test_profiles.py +27 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/tests/test_repo.py +17 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/.github/workflows/docs.yml +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/.github/workflows/publish.yml +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/.github/workflows/release.yml +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/.gitignore +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/.node-version +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/.python-version +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/CLAUDE.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/CONTRIBUTING.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/LICENSE +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/README.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/blume.config.ts +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/concepts/meta.ts +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/concepts/repositories.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/concepts/versions-and-refs.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/concepts/working-catalogs.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/getting-started.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/guides/building-a-host.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/guides/meta.ts +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/guides/mirrors.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/guides/public-and-private.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/guides/reading-without-duckdb.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/guides/serving-over-http.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/guides/stores.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/index.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/internals/from-write-to-read.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/internals/inside-a-repository.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/internals/meta.ts +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/concepts/meta.ts +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/concepts/repositories.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/concepts/versions-and-refs.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/concepts/working-catalogs.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/getting-started.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/guides/building-a-host.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/guides/meta.ts +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/guides/mirrors.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/guides/public-and-private.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/guides/reading-without-duckdb.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/guides/serving-over-http.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/guides/stores.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/index.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/internals/from-write-to-read.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/internals/inside-a-repository.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/internals/meta.ts +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/reference/meta.ts +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/reference/python.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/spec/http.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/spec/index.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/spec/layout.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/spec/meta.ts +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/spec/reading.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/spec/refs.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/ja/spec/writing.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/reference/meta.ts +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/reference/python.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/spec/http.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/spec/index.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/spec/layout.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/spec/meta.ts +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/spec/reading.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/spec/refs.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/docs/spec/writing.md +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/package-lock.json +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/package.json +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/public/CNAME +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/__init__.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/catalog.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/dataset.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/layout.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/objects.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/py.typed +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/refs.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/store.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/stores.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/src/fdl/version.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/tests/http_repo.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/tests/test_cli.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/tests/test_dataset.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/tests/test_stores.py +0 -0
- {fdl_cli-0.20.0 → fdl_cli-0.21.0}/uv.lock +0 -0
|
@@ -1,6 +1,34 @@
|
|
|
1
1
|
# CHANGELOG
|
|
2
2
|
|
|
3
3
|
|
|
4
|
+
## v0.21.0 (2026-11-06)
|
|
5
|
+
|
|
6
|
+
### Bug Fixes
|
|
7
|
+
|
|
8
|
+
- **withdraw**: Delete the assets only withdrawn versions reach
|
|
9
|
+
([#41](https://github.com/queria-io/fdl/pull/41),
|
|
10
|
+
[`33a452e`](https://github.com/queria-io/fdl/commit/33a452e2de9ab0640a32a6550fd21851ba4dc038))
|
|
11
|
+
|
|
12
|
+
A version reaches the assets its tree.json lists and those its asset columns hold. Withdrawal now
|
|
13
|
+
deletes those no kept version reaches, before the data files, so a run that stops half way can
|
|
14
|
+
still work them out.
|
|
15
|
+
|
|
16
|
+
### Features
|
|
17
|
+
|
|
18
|
+
- **profiles**: List every value of a column with up to 2,000 distinct values
|
|
19
|
+
([#42](https://github.com/queria-io/fdl/pull/42),
|
|
20
|
+
[`50a9333`](https://github.com/queria-io/fdl/commit/50a9333fc7b1d2fae990d555ed1393d11d01328c))
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
## v0.20.1 (2026-11-06)
|
|
24
|
+
|
|
25
|
+
### Bug Fixes
|
|
26
|
+
|
|
27
|
+
- Withdraw a version already past retention by deleting unread files at once
|
|
28
|
+
([#40](https://github.com/queria-io/fdl/pull/40),
|
|
29
|
+
[`d9e3788`](https://github.com/queria-io/fdl/commit/d9e3788b6ffb429a9d301491a5e2a73eca8d891f))
|
|
30
|
+
|
|
31
|
+
|
|
4
32
|
## v0.20.0 (2026-11-06)
|
|
5
33
|
|
|
6
34
|
### Documentation
|
|
@@ -108,11 +108,11 @@ fdl tag <remote> <name> --delete
|
|
|
108
108
|
fdl withdraw <remote> <version>... [--yes]
|
|
109
109
|
```
|
|
110
110
|
|
|
111
|
-
間違って公開した個人データなど、公開してはいけないデータを持つバージョンを取り下げます([取り下げ](/ja/spec/data-files#取り下げ)
|
|
111
|
+
間違って公開した個人データなど、公開してはいけないデータを持つバージョンを取り下げます([取り下げ](/ja/spec/data-files#取り下げ))。先にそのデータを除いたバージョンをコミットし、該当するバージョンを指すブランチが無い状態にします。そのうえでこのコマンドが、該当するバージョンを指すタグ、そのバージョンが参照し残すバージョンが参照しないデータファイルとアセット、カタログを消し、`"reason": "withdrawn"` の `expired.json` を置きます。`version.json` は残します。保持期間を過ぎたバージョンを含めると、そのカタログはもう無いので、有効なバージョンのどれも参照しないデータファイルとアセットを、猶予期間を待たずにすべて消します。
|
|
112
112
|
|
|
113
|
-
|
|
113
|
+
バージョンが参照するアセットは、`tree.json` に載っているものと、アセット列が持つ sha256 です。
|
|
114
114
|
|
|
115
|
-
`--yes`
|
|
115
|
+
`--yes` を付けないと、消すものを表示するだけです。残すバージョンが参照しているファイルは残すものとして表示します。それがそのデータを持つなら、そのバージョンも取り下げます。ブランチかリクエストが該当するバージョンを指しているあいだは断られます。
|
|
116
116
|
|
|
117
117
|
## fdl expire
|
|
118
118
|
|
|
@@ -55,7 +55,7 @@ DuckLake 自身の片付け(`ducklake_cleanup_old_files`、`ducklake_expire_sn
|
|
|
55
55
|
|
|
56
56
|
1. そのデータを除いた新しいバージョンを作り、該当するバージョンを指しているブランチをすべてそこへ動かす。新しいバージョンの親はいまのバージョンなので、前へ動かすことになる
|
|
57
57
|
2. 該当するバージョンを指すタグを消す
|
|
58
|
-
3.
|
|
58
|
+
3. 個人情報を持つデータファイルや、それが写ったスクリーンショットなど、そのデータを持つ共有オブジェクトをすぐに消す。該当するバージョンから到達し、残すバージョンのどれからも到達しないものが対象。そのオブジェクトに到達するバージョンは、すべて該当するバージョンになる。該当するバージョンに保持期間を過ぎたものがあると、カタログが無いので、それがどのオブジェクトに到達していたかが分からない。その場合は、有効なバージョンのどれからも到達しない共有オブジェクトもすべてすぐに消す
|
|
59
59
|
4. 該当するバージョンそれぞれに `"reason": "withdrawn"` の `expired.json` を置き、カタログを消す
|
|
60
60
|
|
|
61
61
|
共有オブジェクトをカタログより先に消すのは、途中で止まった取り下げをやり直せるようにするためです。カタログが残っていれば、どのファイルがそのデータを持つかが分かります。
|
|
@@ -191,12 +191,12 @@ profile は、`ducklake.snapshot_id` の時点の 1 つのテーブルの要約
|
|
|
191
191
|
|
|
192
192
|
- `rows` はテーブルの行数。`columns` にはカタログの順にすべての列を並べ、`name` と `type`(DuckLake の型名)は `dataset.json` と同じにする
|
|
193
193
|
- 図形の列を除くすべての列が `null_count` を持つ。数値・テキスト・日付と時刻の列は `distinct_count` と `values` を持つ。大きなテーブルの `distinct_count` は推定値のことがある
|
|
194
|
-
- `values` は、値(テキスト)とその値を持つ行の数を、多い順、同数なら値の順に並べる。列が持つ異なる値が
|
|
194
|
+
- `values` は、値(テキスト)とその値を持つ行の数を、多い順、同数なら値の順に並べる。列が持つ異なる値が 2,000 以下なら全部を並べ、そのとき `distinct_count` は正確な数になる。読み手は、`values` の長さが `distinct_count` と等しいことで全部の一覧だと判定する。それより多い列は上位 10 件だけを並べる。ホストは全部の一覧を、地域コードや名前などの値で検索できる
|
|
195
195
|
- 数値の列は `min`・`max`・`mean`・`std`、四分位数の `q1`・`median`・`q3`、`min` から `max` までを等幅に区切った `histogram` を持つ。日付と時刻の列は `min` と `max` をテキストで持ち、日付とタイムスタンプは `span_days` も持つ。真偽値の列は `true_count` と `false_count` を持つ
|
|
196
196
|
- `head` は、テーブルが返す先頭の行を JSON のオブジェクトで持つ。図形の列と、値を JSON で表せない列は入れない
|
|
197
197
|
- `geometry` は、書き手が読めたときに、最初の図形の列を説明する。図形を持つ行の数、全体の `bbox`(列の座標で `[xmin, ymin, xmax, ymax]`)、含まれる図形の種類、地図を描くための、無作為に選んで簡略化した図形の GeoJSON `FeatureCollection` を持つ
|
|
198
198
|
- 書き手は計算できないキーを省いてよい。読み手は知らないキーを無視する
|
|
199
|
-
- 書き手は、データファイル(`files.json`)と列が親と同じテーブルには親の profile
|
|
199
|
+
- 書き手は、データファイル(`files.json`)と列が親と同じテーブルには親の profile を使い回す。コミットで読むのは変わったテーブルだけになる。親の profile の `values` が書き手と異なる上限で書かれているときは、profile を作り直す
|
|
200
200
|
- profile はバージョンの一部で、そのバージョンの中では変わらない。保持期間から外すときはカタログと一緒に消す
|
|
201
201
|
|
|
202
202
|
## 固めたカタログ
|
|
@@ -108,11 +108,11 @@ fdl tag <remote> <name> --delete
|
|
|
108
108
|
fdl withdraw <remote> <version>... [--yes]
|
|
109
109
|
```
|
|
110
110
|
|
|
111
|
-
Removes versions that hold data that must not be published, such as personal data published by mistake (see [Withdrawal](/spec/data-files#withdrawal)). First commit a version without the data, so that no branch points at an affected version. The command then deletes the tags that point at the versions, the data files
|
|
111
|
+
Removes versions that hold data that must not be published, such as personal data published by mistake (see [Withdrawal](/spec/data-files#withdrawal)). First commit a version without the data, so that no branch points at an affected version. The command then deletes the tags that point at the versions, the data files and assets the versions reach that no kept version reaches, and their catalogs, and puts `expired.json` with `"reason": "withdrawn"`. `version.json` stays. When one of the versions is already past retention, its catalog is gone, so every data file and asset no live version reaches is deleted at once, without waiting for the grace period.
|
|
112
112
|
|
|
113
|
-
|
|
113
|
+
A version reaches the assets its `tree.json` lists and those its asset columns hold.
|
|
114
114
|
|
|
115
|
-
Without `--yes` it only prints what it would delete. Files a kept version still
|
|
115
|
+
Without `--yes` it only prints what it would delete. Files a kept version still reaches are listed as kept; if they hold the data, withdraw that version too. Refused while a branch or a request points at one of the versions.
|
|
116
116
|
|
|
117
117
|
## fdl expire
|
|
118
118
|
|
|
@@ -55,7 +55,7 @@ Withdrawal removes data that must not be published, such as personal data publis
|
|
|
55
55
|
|
|
56
56
|
1. Make a new version without the data and move every branch that holds an affected version to it. This is a forward move: the new version's parent is the current one
|
|
57
57
|
2. Delete the tags that point at affected versions
|
|
58
|
-
3. Delete the shared objects that hold the data, at once, such as a data file with personal data or a screenshot showing it: those an affected version reaches and no version kept reaches. Every version that reaches such an object is an affected version
|
|
58
|
+
3. Delete the shared objects that hold the data, at once, such as a data file with personal data or a screenshot showing it: those an affected version reaches and no version kept reaches. Every version that reaches such an object is an affected version. An affected version already past retention has no catalog to say which objects it reached, so when there is one, every shared object no live version reaches is deleted at once too
|
|
59
59
|
4. For every affected version, put `expired.json` with `"reason": "withdrawn"` and delete its catalogs
|
|
60
60
|
|
|
61
61
|
The shared objects go before the catalogs, so a withdrawal that stops half way can be run again: the catalogs still say which files hold the data.
|
|
@@ -191,12 +191,12 @@ Each table's profile is at `versions/<ID>/profiles/<schema>/<table>.json`. A tab
|
|
|
191
191
|
|
|
192
192
|
- `rows` is the table's row count. `columns` lists every column in the catalog's order, with `name` and `type` (DuckLake's type name) as in `dataset.json`
|
|
193
193
|
- Every column but a geometry one has `null_count`. Numbers, text, dates and times have `distinct_count`, which may be estimated on a large table, and `values`
|
|
194
|
-
- `values` lists each value, as text, with the number of rows that hold it, most common first and then by value. When the column holds at most
|
|
194
|
+
- `values` lists each value, as text, with the number of rows that hold it, most common first and then by value. When the column holds at most 2,000 distinct values, it lists them all and `distinct_count` is exact: a reader tells a whole list by its length matching `distinct_count`. Otherwise it lists the 10 most common. A host can search a whole list by value, such as a town's code or name
|
|
195
195
|
- Numbers have `min`, `max`, `mean`, `std`, the quartiles `q1`, `median` and `q3`, and a `histogram` of equal-width buckets between `min` and `max`. Dates and times have `min` and `max` as text, and dates and timestamps `span_days`. Booleans have `true_count` and `false_count`
|
|
196
196
|
- `head` holds the first rows the table returns, as JSON objects, leaving out geometry and columns whose values have no JSON form
|
|
197
197
|
- `geometry` describes the first geometry column, when the writer could read it: how many rows have a shape, the `bbox` of all of them (`[xmin, ymin, xmax, ymax]` in the column's coordinates), the geometry types present, and a GeoJSON `FeatureCollection` of a random sample of simplified shapes for drawing a map
|
|
198
198
|
- A writer may leave out any key it cannot compute. A reader ignores keys it does not know
|
|
199
|
-
- A writer reuses its parent's profile for a table whose data files (in `files.json`) and columns are the parent's, so a commit reads only the tables that changed
|
|
199
|
+
- A writer reuses its parent's profile for a table whose data files (in `files.json`) and columns are the parent's, so a commit reads only the tables that changed. It makes the profile again when the parent's `values` follow another limit than its own
|
|
200
200
|
- Profiles are part of the version and do not change with it. They are deleted with the catalogs when the version is taken out of retention
|
|
201
201
|
|
|
202
202
|
## The frozen catalog
|
|
@@ -294,12 +294,14 @@ def withdraw(
|
|
|
294
294
|
"""Remove versions that hold data that must not be published.
|
|
295
295
|
|
|
296
296
|
Move every branch to a version without the data first. This deletes the
|
|
297
|
-
tags that point at the versions, the data files no kept version
|
|
298
|
-
and the versions' catalogs, at once. Their version.json stays.
|
|
297
|
+
tags that point at the versions, the data files and assets no kept version
|
|
298
|
+
reaches, and the versions' catalogs, at once. Their version.json stays. A
|
|
299
|
+
version already past retention has no catalog, so then every data file and
|
|
300
|
+
asset no live version reaches is deleted.
|
|
299
301
|
"""
|
|
300
302
|
store = open_store(remote)
|
|
301
303
|
try:
|
|
302
|
-
plan = repo.plan_withdrawal(store, versions)
|
|
304
|
+
plan = repo.plan_withdrawal(store, versions, setup=_read_setup(store))
|
|
303
305
|
for name in plan.tags:
|
|
304
306
|
typer.echo(f"delete tag {name}")
|
|
305
307
|
for key in plan.deleted:
|
|
@@ -307,7 +309,7 @@ def withdraw(
|
|
|
307
309
|
for version_id in plan.versions:
|
|
308
310
|
typer.echo(f"withdraw {version_id}")
|
|
309
311
|
for key in plan.still_read:
|
|
310
|
-
typer.echo(f"keep file {key} (a kept version
|
|
312
|
+
typer.echo(f"keep file {key} (a kept version reaches it)")
|
|
311
313
|
if not yes:
|
|
312
314
|
typer.echo("Nothing was changed. Run again with --yes to do this.")
|
|
313
315
|
return
|
|
@@ -25,8 +25,9 @@ GEOMETRY_SAMPLE = 500
|
|
|
25
25
|
#: Vertex tolerance of the sampled shapes, in the column's units (about 11 m in degrees).
|
|
26
26
|
SIMPLIFY_TOLERANCE = 0.0001
|
|
27
27
|
HISTOGRAM_BUCKETS = 20
|
|
28
|
-
#: A column with at most this many distinct values lists them all in `values
|
|
29
|
-
|
|
28
|
+
#: A column with at most this many distinct values lists them all in `values`: enough for
|
|
29
|
+
#: a column of codes or names that covers a whole country, such as its municipalities.
|
|
30
|
+
ALL_VALUES = 2000
|
|
30
31
|
#: Otherwise `values` lists the most common.
|
|
31
32
|
TOP_VALUES = 10
|
|
32
33
|
#: Above this many rows, distinct values are estimated rather than counted.
|
|
@@ -112,14 +113,30 @@ def build(working: Path, wanted: Iterable[dict[str, Any]], *, data_path: str, se
|
|
|
112
113
|
|
|
113
114
|
|
|
114
115
|
def reusable(previous: bytes, table: dict[str, Any]) -> bool:
|
|
115
|
-
"""Whether a parent's profile describes ``table`` as it is
|
|
116
|
+
"""Whether a parent's profile describes ``table`` as it is and as this writer would.
|
|
117
|
+
|
|
118
|
+
The same columns and types, and `values` listed by the current `ALL_VALUES`:
|
|
119
|
+
a profile written under another limit is made again rather than carried forward.
|
|
120
|
+
"""
|
|
116
121
|
try:
|
|
117
122
|
held = json.loads(previous).get("columns", [])
|
|
118
123
|
except ValueError:
|
|
119
124
|
return False
|
|
120
|
-
|
|
125
|
+
if [{"name": c.get("name"), "type": c.get("type")} for c in held] != columns_of(table):
|
|
126
|
+
return False
|
|
127
|
+
return all(_listed_as_now(c) for c in held)
|
|
121
128
|
|
|
122
129
|
|
|
130
|
+
def _listed_as_now(column: dict[str, Any]) -> bool:
|
|
131
|
+
values, distinct = column.get("values"), column.get("distinct_count")
|
|
132
|
+
if values is None:
|
|
133
|
+
return True
|
|
134
|
+
if not isinstance(values, list) or not isinstance(distinct, int):
|
|
135
|
+
return False
|
|
136
|
+
if len(values) == distinct:
|
|
137
|
+
return distinct <= ALL_VALUES
|
|
138
|
+
return distinct > ALL_VALUES and len(values) <= TOP_VALUES
|
|
139
|
+
|
|
123
140
|
|
|
124
141
|
def _load_spatial(con) -> bool:
|
|
125
142
|
try:
|
|
@@ -622,18 +622,23 @@ class Withdrawal:
|
|
|
622
622
|
versions: tuple[str, ...]
|
|
623
623
|
#: Tags that point at the versions, deleted first.
|
|
624
624
|
tags: tuple[str, ...]
|
|
625
|
-
#:
|
|
625
|
+
#: Shared objects (data files and assets) the versions reach and no kept
|
|
626
|
+
#: version reaches, as store keys.
|
|
626
627
|
deleted: tuple[str, ...]
|
|
627
|
-
#:
|
|
628
|
-
#: they hold the data, the version that
|
|
628
|
+
#: Shared objects the versions reach that a kept version still reaches. Kept:
|
|
629
|
+
#: if they hold the data, the version that reaches them has to be withdrawn too.
|
|
629
630
|
still_read: tuple[str, ...]
|
|
630
631
|
|
|
631
632
|
|
|
632
|
-
def plan_withdrawal(store: Store, version_ids: Iterable[str]) -> Withdrawal:
|
|
633
|
+
def plan_withdrawal(store: Store, version_ids: Iterable[str], *, setup: Iterable[str] = ()) -> Withdrawal:
|
|
633
634
|
"""Work out a withdrawal (see `withdraw`) without changing anything.
|
|
634
635
|
|
|
635
636
|
Refused while a branch or a request still points at one of the versions:
|
|
636
637
|
the first step, a new version without the data, is the writer's.
|
|
638
|
+
|
|
639
|
+
Shared objects are data files and assets. ``setup`` runs before a
|
|
640
|
+
catalog is attached to read its asset columns: a ``CREATE SECRET`` for
|
|
641
|
+
data files in private storage, say.
|
|
637
642
|
"""
|
|
638
643
|
config = read_config(store)
|
|
639
644
|
affected = tuple(dict.fromkeys(version_ids))
|
|
@@ -653,52 +658,110 @@ def plan_withdrawal(store: Store, version_ids: Iterable[str]) -> Withdrawal:
|
|
|
653
658
|
)
|
|
654
659
|
tags.append(name)
|
|
655
660
|
|
|
661
|
+
setup = list(setup)
|
|
662
|
+
# Store keys: data files under the data directory, assets under assets/.
|
|
656
663
|
candidates: set[str] = set()
|
|
657
664
|
kept: set[str] = set()
|
|
665
|
+
catalogless = False
|
|
658
666
|
with tempfile.TemporaryDirectory(prefix="fdl-") as tmp:
|
|
659
667
|
for version_id in _version_ids(store):
|
|
660
668
|
if version_id in affected:
|
|
669
|
+
version = read_version(store, version_id)
|
|
661
670
|
local = _fetch_catalog(store, version_id, Path(tmp))
|
|
662
671
|
if local is not None:
|
|
663
672
|
# Every file the catalog has ever named: earlier snapshots
|
|
664
673
|
# in it may hold the data too.
|
|
665
|
-
candidates.update(catalog.data_files(local))
|
|
674
|
+
candidates.update(config.data_dir + key for key in catalog.data_files(local))
|
|
675
|
+
try:
|
|
676
|
+
candidates.update(_reached_assets(store, version, local, setup))
|
|
677
|
+
except objects.ObjectError:
|
|
678
|
+
# A run stopped half way may have deleted the data
|
|
679
|
+
# files that held its asset columns: then, as for a
|
|
680
|
+
# version without a catalog, every asset no live
|
|
681
|
+
# version reaches goes.
|
|
682
|
+
catalogless = True
|
|
666
683
|
local.unlink()
|
|
684
|
+
else:
|
|
685
|
+
catalogless = True
|
|
667
686
|
continue
|
|
668
687
|
if is_expired(store, version_id):
|
|
669
688
|
continue
|
|
670
689
|
version = read_version(store, version_id)
|
|
671
690
|
# files.json lists what the version reads, without its catalog.
|
|
672
691
|
indexed = _indexed_files(store, version)
|
|
692
|
+
local = None
|
|
693
|
+
if indexed is None or _asset_columns(store, version):
|
|
694
|
+
local = _fetch_catalog(store, version_id, Path(tmp))
|
|
673
695
|
if indexed is not None:
|
|
674
|
-
kept.update(indexed)
|
|
675
|
-
|
|
676
|
-
|
|
696
|
+
kept.update(config.data_dir + key for key in indexed)
|
|
697
|
+
elif local is not None:
|
|
698
|
+
kept.update(config.data_dir + key for key in _read_files(local, version))
|
|
699
|
+
kept.update(_reached_assets(store, version, local, setup))
|
|
677
700
|
if local is not None:
|
|
678
|
-
kept.update(_read_files(local, version))
|
|
679
701
|
local.unlink()
|
|
702
|
+
if catalogless:
|
|
703
|
+
# A version already past retention no longer says which objects it
|
|
704
|
+
# reached. Those no live version reaches are waiting for the grace
|
|
705
|
+
# period anyway, so all of them go now.
|
|
706
|
+
candidates.update(store.list(config.data_dir))
|
|
707
|
+
candidates.update(store.list(layout.ASSETS_DIR))
|
|
680
708
|
return Withdrawal(
|
|
681
709
|
versions=affected,
|
|
682
710
|
tags=tuple(tags),
|
|
683
|
-
deleted=tuple(
|
|
684
|
-
still_read=tuple(
|
|
711
|
+
deleted=tuple(sorted(candidates - kept)),
|
|
712
|
+
still_read=tuple(sorted(candidates & kept)),
|
|
685
713
|
)
|
|
686
714
|
|
|
687
715
|
|
|
716
|
+
def _asset_columns(store: Store, version: Version) -> list[tuple[str, str, str]]:
|
|
717
|
+
"""The asset columns the version's `dataset.json` marks."""
|
|
718
|
+
if version.dataset is None:
|
|
719
|
+
return []
|
|
720
|
+
try:
|
|
721
|
+
described = json.loads(store.get(layout.version_dir(version.id) + version.dataset.path))
|
|
722
|
+
except Exception as exc: # noqa: BLE001 - stores raise their own types
|
|
723
|
+
if is_missing(exc):
|
|
724
|
+
return []
|
|
725
|
+
raise
|
|
726
|
+
return objects.asset_columns(described)
|
|
727
|
+
|
|
728
|
+
|
|
729
|
+
def _reached_assets(store: Store, version: Version, local: Path | None, setup: list[str]) -> set[str]:
|
|
730
|
+
"""The asset keys the version reaches: its `tree.json` and its asset columns."""
|
|
731
|
+
reached: set[str] = set()
|
|
732
|
+
if version.tree is not None:
|
|
733
|
+
try:
|
|
734
|
+
listed = json.loads(store.get(layout.version_dir(version.id) + version.tree.path))
|
|
735
|
+
except Exception as exc: # noqa: BLE001 - stores raise their own types
|
|
736
|
+
if not is_missing(exc):
|
|
737
|
+
raise
|
|
738
|
+
else:
|
|
739
|
+
reached.update(objects.asset_key(item["sha256"]) for item in listed["entries"])
|
|
740
|
+
columns = _asset_columns(store, version)
|
|
741
|
+
if columns and local is not None:
|
|
742
|
+
values = objects.asset_values(local, columns, data_path=data_url(store), setup=setup)
|
|
743
|
+
reached.update(objects.asset_key(sha256) for sha256 in values)
|
|
744
|
+
return reached
|
|
745
|
+
|
|
746
|
+
|
|
688
747
|
def withdraw(store: Store, version_ids: Iterable[str], *, plan: Withdrawal | None = None) -> Withdrawal:
|
|
689
748
|
"""Remove versions that hold data that must not be published.
|
|
690
749
|
|
|
691
|
-
Unlike retention it takes live versions and deletes their data files
|
|
692
|
-
once. In the spec's order
|
|
750
|
+
Unlike retention it takes live versions and deletes their data files and
|
|
751
|
+
assets at once. In the spec's order: the shared objects go before the
|
|
693
752
|
catalogs, so a run that stops half way can be run again while the
|
|
694
|
-
catalogs still say which
|
|
753
|
+
catalogs still say which objects they are. `version.json` stays.
|
|
695
754
|
"""
|
|
696
755
|
plan = plan or plan_withdrawal(store, version_ids)
|
|
697
756
|
for name in plan.tags:
|
|
698
757
|
refs.delete(store, name)
|
|
699
758
|
if plan.tags:
|
|
700
759
|
update_refs_index(store)
|
|
701
|
-
|
|
760
|
+
# Assets first: the data files are what says which assets the versions'
|
|
761
|
+
# rows reach, so a run that stops half way can still work them out.
|
|
762
|
+
assets = [key for key in plan.deleted if key.startswith(layout.ASSETS_DIR)]
|
|
763
|
+
store.delete(assets)
|
|
764
|
+
store.delete([key for key in plan.deleted if not key.startswith(layout.ASSETS_DIR)])
|
|
702
765
|
for version_id in plan.versions:
|
|
703
766
|
expire(store, version_id, reason="withdrawn")
|
|
704
767
|
return plan
|
|
@@ -156,7 +156,8 @@ def test_a_file_the_host_will_not_sign_goes_through_the_host(tmp_path: Path) ->
|
|
|
156
156
|
|
|
157
157
|
assert local.get("data/small.parquet") == b"ab" and local.get("data/large.parquet") == b"abcdef"
|
|
158
158
|
puts = [key for method, key in server.requests if method == "PUT"]
|
|
159
|
-
|
|
159
|
+
# Uploads run at once, so they may arrive in any order.
|
|
160
|
+
assert sorted(puts) == ["data/large.parquet", "storage/data/small.parquet"]
|
|
160
161
|
|
|
161
162
|
|
|
162
163
|
def test_a_rebuild_of_the_same_rows_over_http_is_nothing_to_commit(served, tmp_path: Path) -> None:
|
|
@@ -175,3 +175,37 @@ def test_media_type_does_not_depend_on_the_machine(path: str, expected: str) ->
|
|
|
175
175
|
def test_asset_columns_that_cannot_be_read_are_an_object_error(tmp_path: Path) -> None:
|
|
176
176
|
with pytest.raises(objects.ObjectError, match="could not read the asset columns"):
|
|
177
177
|
objects.asset_values(tmp_path / "missing.ducklake", [("main", "t", "c")], data_path=str(tmp_path) + "/")
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def test_withdrawal_deletes_the_assets_only_the_withdrawn_versions_reach(tmp_path: Path, repository) -> None:
|
|
181
|
+
store, working = repository
|
|
182
|
+
folder = tmp_path / "files"
|
|
183
|
+
folder.mkdir()
|
|
184
|
+
(folder / "kept.pdf").write_bytes(b"%PDF-kept")
|
|
185
|
+
(folder / "leak.pdf").write_bytes(b"%PDF-leak")
|
|
186
|
+
(tmp_path / "files.csv").write_text(run("assets", folder))
|
|
187
|
+
(tmp_path / "README.md").write_text("# Reports\n")
|
|
188
|
+
(tmp_path / "dataset.yml").write_text(
|
|
189
|
+
"name: reports\ntitle: Reports\ntables:\n"
|
|
190
|
+
" - schema: main\n name: reports\n fields:\n - name: sha256\n asset: true\n"
|
|
191
|
+
)
|
|
192
|
+
write(
|
|
193
|
+
working,
|
|
194
|
+
store,
|
|
195
|
+
f"CREATE TABLE lake.main.reports AS SELECT path, sha256 FROM read_csv('{tmp_path / 'files.csv'}', all_varchar = true)",
|
|
196
|
+
)
|
|
197
|
+
run("commit", working, "-m", "leak", "--author", "t", "--assets", folder)
|
|
198
|
+
leaked = repo.head(store, "refs/heads/main")
|
|
199
|
+
write(working, store, "DELETE FROM lake.main.reports WHERE path = 'leak.pdf'")
|
|
200
|
+
run("commit", working, "-m", "clean", "--author", "t")
|
|
201
|
+
rows = {row["path"]: row["sha256"] for row in csv.DictReader(io.StringIO(run("assets", folder)))}
|
|
202
|
+
readme = objects.asset_key(objects.sha256_of(tmp_path / "README.md"))
|
|
203
|
+
|
|
204
|
+
plan = repo.plan_withdrawal(store, [leaked.id])
|
|
205
|
+
assert objects.asset_key(rows["leak.pdf"]) in plan.deleted
|
|
206
|
+
assert {objects.asset_key(rows["kept.pdf"]), readme} <= set(plan.still_read)
|
|
207
|
+
|
|
208
|
+
run("withdraw", store.root, leaked.id, "--yes")
|
|
209
|
+
assert store.head(objects.asset_key(rows["leak.pdf"])) is None
|
|
210
|
+
assert store.head(objects.asset_key(rows["kept.pdf"])) is not None
|
|
211
|
+
assert store.head(readme) is not None
|
|
@@ -76,6 +76,33 @@ def test_many_values_list_only_the_most_common(repository) -> None:
|
|
|
76
76
|
assert code["distinct_count"] > profiles.ALL_VALUES
|
|
77
77
|
|
|
78
78
|
|
|
79
|
+
def test_a_column_at_the_limit_lists_every_value(repository) -> None:
|
|
80
|
+
store, working = repository
|
|
81
|
+
write(working, store, f"CREATE TABLE lake.main.towns AS SELECT 'v' || i AS code FROM range({profiles.ALL_VALUES}) t(i)")
|
|
82
|
+
run("commit", working, "-m", "m", "--author", "t")
|
|
83
|
+
code = profile_of(store)["columns"][0]
|
|
84
|
+
assert len(code["values"]) == code["distinct_count"] == profiles.ALL_VALUES
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@pytest.mark.parametrize(
|
|
88
|
+
("values", "distinct", "kept"),
|
|
89
|
+
[
|
|
90
|
+
(3, 3, True),
|
|
91
|
+
(profiles.TOP_VALUES, profiles.ALL_VALUES + 1, True),
|
|
92
|
+
# Written under a lower limit: a short list of a column now listed whole.
|
|
93
|
+
(profiles.TOP_VALUES, profiles.ALL_VALUES - 1, False),
|
|
94
|
+
# Written under a higher limit: a whole list longer than now allowed.
|
|
95
|
+
(profiles.ALL_VALUES + 1, profiles.ALL_VALUES + 1, False),
|
|
96
|
+
],
|
|
97
|
+
)
|
|
98
|
+
def test_a_profile_written_under_another_limit_is_made_again(values: int, distinct: int, kept: bool) -> None:
|
|
99
|
+
table = {"fields": [{"name": "code", "type": "VARCHAR"}]}
|
|
100
|
+
column = {"name": "code", "type": "VARCHAR", "distinct_count": distinct}
|
|
101
|
+
column["values"] = [{"value": f"v{i}", "count": 1} for i in range(values)]
|
|
102
|
+
previous = json.dumps({"rows": distinct, "columns": [column]}).encode()
|
|
103
|
+
assert profiles.reusable(previous, table) is kept
|
|
104
|
+
|
|
105
|
+
|
|
79
106
|
def test_an_unchanged_table_keeps_its_parents_profile(repository, monkeypatch) -> None:
|
|
80
107
|
store, working = repository
|
|
81
108
|
write(working, store, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
|
|
@@ -387,6 +387,23 @@ def test_withdraw_removes_the_data_from_live_versions(tmp_path: Path, remote) ->
|
|
|
387
387
|
assert repo.head(remote) == clean
|
|
388
388
|
|
|
389
389
|
|
|
390
|
+
def test_withdrawing_a_version_past_retention_deletes_unread_files_at_once(tmp_path: Path, remote) -> None:
|
|
391
|
+
first = pushed(tmp_path, remote)
|
|
392
|
+
leaked = set(remote.list("data/"))
|
|
393
|
+
build(tmp_path / "work.sqlite", remote, "DROP TABLE lake.main.towns; CREATE TABLE lake.main.kept AS SELECT 1 AS id")
|
|
394
|
+
second = commit(tmp_path, remote, parent=first)
|
|
395
|
+
repo.push(remote, tmp_path / "local" / second.id, second)
|
|
396
|
+
kept = set(remote.list("data/")) - leaked
|
|
397
|
+
repo.expire(remote, first.id)
|
|
398
|
+
assert leaked <= set(remote.list("data/")) # waiting for the grace period
|
|
399
|
+
|
|
400
|
+
plan = repo.withdraw(remote, [first.id])
|
|
401
|
+
assert set(plan.deleted) == leaked
|
|
402
|
+
assert set(remote.list("data/")) == kept
|
|
403
|
+
assert json.loads(remote.get(layout.version_expired(first.id)))["reason"] == "withdrawn"
|
|
404
|
+
assert repo.head(remote) == second
|
|
405
|
+
|
|
406
|
+
|
|
390
407
|
def test_a_file_a_kept_version_still_reads_is_not_deleted(tmp_path: Path, remote) -> None:
|
|
391
408
|
first = pushed(tmp_path, remote)
|
|
392
409
|
second = pushed(tmp_path, remote, parent=first, sql="INSERT INTO lake.main.towns VALUES (2)")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|