fdl-cli 0.31.0__tar.gz → 0.33.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/CHANGELOG.md +101 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/PKG-INFO +8 -9
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/README.md +7 -7
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/cases/collection.json +6 -4
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/cases/data-dir.json +11 -3
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/concepts/repositories.md +3 -2
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/getting-started.md +19 -18
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/guides/mirrors.md +3 -2
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/guides/public-and-private.md +4 -4
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/guides/serving-over-http.md +4 -3
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/index.md +1 -1
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/internals/inside-a-repository.md +4 -4
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/concepts/repositories.md +3 -2
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/getting-started.md +19 -18
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/guides/mirrors.md +3 -2
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/guides/public-and-private.md +4 -4
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/guides/serving-over-http.md +4 -3
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/index.md +1 -1
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/internals/inside-a-repository.md +4 -4
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/reference/cli.md +7 -7
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/reference/python.md +2 -2
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/spec/collection-cases.md +3 -3
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/spec/data-files.md +17 -14
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/spec/http.md +5 -5
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/spec/index.md +4 -2
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/spec/layout.md +8 -7
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/spec/reading.md +15 -1
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/spec/versions.md +60 -108
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/spec/writing.md +16 -17
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/stability.md +4 -3
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/reference/cli.md +7 -7
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/reference/python.md +2 -2
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/spec/collection-cases.md +3 -3
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/spec/data-files.md +17 -14
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/spec/http.md +5 -5
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/spec/index.md +4 -2
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/spec/layout.md +8 -7
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/spec/reading.md +16 -2
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/spec/versions.md +59 -107
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/spec/writing.md +16 -17
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/stability.md +4 -3
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/pyproject.toml +1 -2
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/src/fdl/catalog.py +132 -110
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/src/fdl/cli.py +48 -12
- fdl_cli-0.33.0/src/fdl/dataset.py +216 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/src/fdl/diff.py +70 -64
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/src/fdl/layout.py +7 -5
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/src/fdl/merge.py +78 -8
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/src/fdl/objects.py +39 -23
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/src/fdl/profiles.py +6 -2
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/src/fdl/repo.py +189 -59
- fdl_cli-0.33.0/src/fdl/tags.py +160 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/src/fdl/version.py +0 -3
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/tests/test_cli.py +78 -9
- fdl_cli-0.33.0/tests/test_dataset.py +140 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/tests/test_http.py +5 -4
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/tests/test_merge.py +67 -2
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/tests/test_objects.py +73 -27
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/tests/test_profiles.py +66 -3
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/tests/test_repo.py +134 -26
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/tests/test_stores.py +1 -1
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/uv.lock +1 -3
- fdl_cli-0.31.0/src/fdl/dataset.py +0 -318
- fdl_cli-0.31.0/src/fdl/lineage.py +0 -127
- fdl_cli-0.31.0/tests/test_dataset.py +0 -228
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/.github/workflows/docs.yml +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/.github/workflows/publish.yml +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/.github/workflows/release.yml +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/.github/workflows/test.yml +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/.gitignore +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/.node-version +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/.python-version +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/CLAUDE.md +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/CONTRIBUTING.md +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/LICENSE +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/SECURITY.md +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/blume.config.ts +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/cases/retention.json +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/concepts/meta.ts +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/concepts/versions-and-refs.md +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/concepts/working-metadata.md +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/guides/building-a-host.md +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/guides/meta.ts +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/guides/reading-without-duckdb.md +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/guides/stores.md +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/internals/from-write-to-read.md +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/internals/meta.ts +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/concepts/meta.ts +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/concepts/versions-and-refs.md +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/concepts/working-metadata.md +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/guides/building-a-host.md +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/guides/meta.ts +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/guides/reading-without-duckdb.md +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/guides/stores.md +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/internals/from-write-to-read.md +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/internals/meta.ts +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/reference/meta.ts +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/spec/meta.ts +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/spec/refs.md +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/ja/spec/retention-cases.md +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/reference/meta.ts +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/spec/meta.ts +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/spec/refs.md +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/docs/spec/retention-cases.md +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/package-lock.json +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/package.json +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/public/CNAME +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/src/fdl/__init__.py +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/src/fdl/config.py +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/src/fdl/py.typed +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/src/fdl/refs.py +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/src/fdl/store.py +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/src/fdl/stores.py +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/tests/http_repo.py +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/tests/test_collection_cases.py +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/tests/test_data_dir_cases.py +0 -0
- {fdl_cli-0.31.0 → fdl_cli-0.33.0}/tests/test_retention_cases.py +0 -0
|
@@ -1,6 +1,107 @@
|
|
|
1
1
|
# CHANGELOG
|
|
2
2
|
|
|
3
3
|
|
|
4
|
+
## v0.33.0 (2026-11-21)
|
|
5
|
+
|
|
6
|
+
### Bug Fixes
|
|
7
|
+
|
|
8
|
+
- Allow descriptions kept as table rows, and keep comments when merging
|
|
9
|
+
([#80](https://github.com/queria-io/fdl/pull/80),
|
|
10
|
+
[`0675306`](https://github.com/queria-io/fdl/commit/0675306b4f74ee5e5197f0788728ada352721c1e))
|
|
11
|
+
|
|
12
|
+
- Profile a view again only when a table or view it reads changed
|
|
13
|
+
([#81](https://github.com/queria-io/fdl/pull/81),
|
|
14
|
+
[`3a760ed`](https://github.com/queria-io/fdl/commit/3a760ed884b7206538fcbc994c12fa0ca8aead9c))
|
|
15
|
+
|
|
16
|
+
### Documentation
|
|
17
|
+
|
|
18
|
+
- **spec**: Keep primary keys, assets and catalogs as DuckLake tags and drop dataset.json and
|
|
19
|
+
tree.json ([#77](https://github.com/queria-io/fdl/pull/77),
|
|
20
|
+
[`945dab3`](https://github.com/queria-io/fdl/commit/945dab3a94ac579386c1572007972aa7be8fbef4))
|
|
21
|
+
|
|
22
|
+
* docs(spec)!: keep primary keys, assets and catalogs as DuckLake tags and drop dataset.json and
|
|
23
|
+
tree.json
|
|
24
|
+
|
|
25
|
+
A version no longer carries dataset.json, tree.json or files under objects/. What FDL reads about a
|
|
26
|
+
table or a view is kept in the metadata as tags: fdl.primary_key and fdl.assets on tables,
|
|
27
|
+
fdl.catalogs on views. Titles, descriptions, licenses, sources and guides are left to the host.
|
|
28
|
+
|
|
29
|
+
Earlier versions keep their files: readers may ignore dataset.json and tree.json, read primary keys,
|
|
30
|
+
assets and catalogs from dataset.json when a version has no tags, and files under objects/ stay
|
|
31
|
+
shared objects until the versions that list them are taken out of retention.
|
|
32
|
+
|
|
33
|
+
* docs(spec): compare fdl. tags only, follow renamed columns, and keep checks for earlier versions
|
|
34
|
+
|
|
35
|
+
Tags are written after the parent's snapshot so a change to tags alone shows in diff. Renamed
|
|
36
|
+
columns are followed by ID. An earlier version is one with dataset or tree, its dataset.json stops
|
|
37
|
+
deletion when unreadable, and a push still needs the files its tree.json lists. diff.tags covers
|
|
38
|
+
views. The stability page lists the fdl. tags.
|
|
39
|
+
|
|
40
|
+
### Features
|
|
41
|
+
|
|
42
|
+
- Skip the profile of a table or view tagged fdl.profile = false
|
|
43
|
+
([#82](https://github.com/queria-io/fdl/pull/82),
|
|
44
|
+
[`31a743c`](https://github.com/queria-io/fdl/commit/31a743c803b2b5f88818972477c26ed62e7491c8))
|
|
45
|
+
|
|
46
|
+
- Write primary keys, assets and catalogs as fdl. tags in the metadata
|
|
47
|
+
([#78](https://github.com/queria-io/fdl/pull/78),
|
|
48
|
+
[`eeab3e9`](https://github.com/queria-io/fdl/commit/eeab3e9e35546000428f98c769baeed1ac8ef9d5))
|
|
49
|
+
|
|
50
|
+
* docs(spec)!: keep primary keys, assets and catalogs as DuckLake tags and drop dataset.json and
|
|
51
|
+
tree.json
|
|
52
|
+
|
|
53
|
+
A version no longer carries dataset.json, tree.json or files under objects/. What FDL reads about a
|
|
54
|
+
table or a view is kept in the metadata as tags: fdl.primary_key and fdl.assets on tables,
|
|
55
|
+
fdl.catalogs on views. Titles, descriptions, licenses, sources and guides are left to the host.
|
|
56
|
+
|
|
57
|
+
Earlier versions keep their files: readers may ignore dataset.json and tree.json, read primary keys,
|
|
58
|
+
assets and catalogs from dataset.json when a version has no tags, and files under objects/ stay
|
|
59
|
+
shared objects until the versions that list them are taken out of retention.
|
|
60
|
+
|
|
61
|
+
* docs(spec): compare fdl. tags only, follow renamed columns, and keep checks for earlier versions
|
|
62
|
+
|
|
63
|
+
Tags are written after the parent's snapshot so a change to tags alone shows in diff. Renamed
|
|
64
|
+
columns are followed by ID. An earlier version is one with dataset or tree, its dataset.json stops
|
|
65
|
+
deletion when unreadable, and a push still needs the files its tree.json lists. diff.tags covers
|
|
66
|
+
views. The stability page lists the fdl. tags.
|
|
67
|
+
|
|
68
|
+
* feat: write primary keys, assets and catalogs as fdl. tags in the metadata
|
|
69
|
+
|
|
70
|
+
fdl commit writes fdl.primary_key and fdl.assets on tables and fdl.catalogs on views from
|
|
71
|
+
dataset.json, still writing dataset.json as before. files.json carries each table's primary_key
|
|
72
|
+
and assets, and diff lists the tables and views whose tags changed.
|
|
73
|
+
|
|
74
|
+
* fix: give a change of tags alone its own snapshot and check primary keys
|
|
75
|
+
|
|
76
|
+
Tags are written before the snapshot range is taken. When the last snapshot is the parent's and the
|
|
77
|
+
tags differ, a new snapshot holds them; unchanged tags are not rewritten. dataset.json's
|
|
78
|
+
primary_key must be a list of the table's columns, each once. A tag naming a missing table raises
|
|
79
|
+
CatalogError. The fdl diff reference lists the tags key.
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
## v0.32.0 (2026-11-16)
|
|
83
|
+
|
|
84
|
+
### Features
|
|
85
|
+
|
|
86
|
+
- Keep only the last snapshot in a version's metadata and read earlier versions from their own
|
|
87
|
+
metadata ([#76](https://github.com/queria-io/fdl/pull/76),
|
|
88
|
+
[`9da0e4d`](https://github.com/queria-io/fdl/commit/9da0e4d6b97109a088754f856194b27be5fd3cd9))
|
|
89
|
+
|
|
90
|
+
- Keep tree files under objects/ and make tables/ the default data_dir
|
|
91
|
+
([#74](https://github.com/queria-io/fdl/pull/74),
|
|
92
|
+
[`df01560`](https://github.com/queria-io/fdl/commit/df0156099e3270c89869b7aa3fcf4bbe24b4993a))
|
|
93
|
+
|
|
94
|
+
- Read only dataset.json and leave its meaning to the writer and host
|
|
95
|
+
([#73](https://github.com/queria-io/fdl/pull/73),
|
|
96
|
+
[`48c15ce`](https://github.com/queria-io/fdl/commit/48c15ce56e22978a05d740f280e45253a527c666))
|
|
97
|
+
|
|
98
|
+
### Refactoring
|
|
99
|
+
|
|
100
|
+
- Stop renaming extensions in dataset.json and drop unused catalog helpers
|
|
101
|
+
([#75](https://github.com/queria-io/fdl/pull/75),
|
|
102
|
+
[`c5a8525`](https://github.com/queria-io/fdl/commit/c5a852580edbdf36a3c1f4f531d7d7e92444fa73))
|
|
103
|
+
|
|
104
|
+
|
|
4
105
|
## v0.31.0 (2026-11-15)
|
|
5
106
|
|
|
6
107
|
### Documentation
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: fdl-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.33.0
|
|
4
4
|
Summary: Versions of a DuckLake catalog on object storage, kept the way Git keeps commits
|
|
5
5
|
Project-URL: Homepage, https://fdl.queria.io/
|
|
6
6
|
Project-URL: Documentation, https://fdl.queria.io/
|
|
@@ -11,7 +11,6 @@ License-Expression: MIT
|
|
|
11
11
|
License-File: LICENSE
|
|
12
12
|
Requires-Python: >=3.10
|
|
13
13
|
Requires-Dist: duckdb<3.0,>=1.5.2
|
|
14
|
-
Requires-Dist: pyyaml>=6.0
|
|
15
14
|
Requires-Dist: tomli>=1.1; python_version < '3.11'
|
|
16
15
|
Requires-Dist: typer>=0.15
|
|
17
16
|
Provides-Extra: s3
|
|
@@ -36,7 +35,7 @@ FDL is a storage layout and a tool for it. Each version is a frozen metadata fil
|
|
|
36
35
|
versions/019a3c4e-…/
|
|
37
36
|
version.json parents, author, message, sha256
|
|
38
37
|
metadata.db the frozen metadata
|
|
39
|
-
|
|
38
|
+
tables/main/towns/ducklake-….parquet
|
|
40
39
|
```
|
|
41
40
|
|
|
42
41
|
The documentation and the specification are at [fdl.queria.io](https://fdl.queria.io).
|
|
@@ -66,17 +65,17 @@ CREATE TABLE lake.main.towns AS SELECT * FROM 'towns.csv';
|
|
|
66
65
|
|
|
67
66
|
Rows have to go to Parquet, not into the metadata (`DATA_INLINING_ROW_LIMIT 0`): a version is read over HTTP, and inlined rows would not be in it.
|
|
68
67
|
|
|
69
|
-
Describe what the version holds in `dataset.
|
|
68
|
+
Describe what the version holds in `dataset.json`, in a folder of its own. Only `name` is required; `fdl commit` fills in each table's columns and row count from the metadata. Every other file in that folder, such as a `README.md`, goes into the version beside the tables:
|
|
70
69
|
|
|
71
|
-
```
|
|
72
|
-
|
|
73
|
-
title: Towns
|
|
70
|
+
```bash
|
|
71
|
+
mkdir about
|
|
72
|
+
echo '{"name": "towns", "title": "Towns"}' > about/dataset.json
|
|
74
73
|
```
|
|
75
74
|
|
|
76
75
|
Then make a version and move `main` to it:
|
|
77
76
|
|
|
78
77
|
```bash
|
|
79
|
-
fdl commit work/lake.sqlite -m "First towns" --author alice
|
|
78
|
+
fdl commit work/lake.sqlite --dataset about/dataset.json -m "First towns" --author alice
|
|
80
79
|
fdl push work/lake.sqlite
|
|
81
80
|
fdl log s3://my-bucket/towns/
|
|
82
81
|
```
|
|
@@ -105,7 +104,7 @@ Credentials and the endpoint come from boto3's usual places (`AWS_*` environment
|
|
|
105
104
|
| `fdl push <working>` | Upload the committed version and move `main` to it |
|
|
106
105
|
| `fdl log <remote>` | List versions from `main` back to the first |
|
|
107
106
|
| `fdl status <working>` | What the working metadata changed since the clone |
|
|
108
|
-
| `fdl diff <remote> [version]` | What a version changed since its parent: tables, rows, columns, data files,
|
|
107
|
+
| `fdl diff <remote> [version]` | What a version changed since its parent: tables, rows, columns, data files, dataset.json |
|
|
109
108
|
| `fdl show <remote> [version]` | A version's record, its tables and its changes |
|
|
110
109
|
| `fdl url <remote> [--attach]` | Print the metadata URL of the version `main` holds, or the whole `ATTACH` statement |
|
|
111
110
|
| `fdl tag <remote> [name] [version]` | List tags, tag a version, or delete a tag (`-d`) |
|
|
@@ -16,7 +16,7 @@ FDL is a storage layout and a tool for it. Each version is a frozen metadata fil
|
|
|
16
16
|
versions/019a3c4e-…/
|
|
17
17
|
version.json parents, author, message, sha256
|
|
18
18
|
metadata.db the frozen metadata
|
|
19
|
-
|
|
19
|
+
tables/main/towns/ducklake-….parquet
|
|
20
20
|
```
|
|
21
21
|
|
|
22
22
|
The documentation and the specification are at [fdl.queria.io](https://fdl.queria.io).
|
|
@@ -46,17 +46,17 @@ CREATE TABLE lake.main.towns AS SELECT * FROM 'towns.csv';
|
|
|
46
46
|
|
|
47
47
|
Rows have to go to Parquet, not into the metadata (`DATA_INLINING_ROW_LIMIT 0`): a version is read over HTTP, and inlined rows would not be in it.
|
|
48
48
|
|
|
49
|
-
Describe what the version holds in `dataset.
|
|
49
|
+
Describe what the version holds in `dataset.json`, in a folder of its own. Only `name` is required; `fdl commit` fills in each table's columns and row count from the metadata. Every other file in that folder, such as a `README.md`, goes into the version beside the tables:
|
|
50
50
|
|
|
51
|
-
```
|
|
52
|
-
|
|
53
|
-
title: Towns
|
|
51
|
+
```bash
|
|
52
|
+
mkdir about
|
|
53
|
+
echo '{"name": "towns", "title": "Towns"}' > about/dataset.json
|
|
54
54
|
```
|
|
55
55
|
|
|
56
56
|
Then make a version and move `main` to it:
|
|
57
57
|
|
|
58
58
|
```bash
|
|
59
|
-
fdl commit work/lake.sqlite -m "First towns" --author alice
|
|
59
|
+
fdl commit work/lake.sqlite --dataset about/dataset.json -m "First towns" --author alice
|
|
60
60
|
fdl push work/lake.sqlite
|
|
61
61
|
fdl log s3://my-bucket/towns/
|
|
62
62
|
```
|
|
@@ -85,7 +85,7 @@ Credentials and the endpoint come from boto3's usual places (`AWS_*` environment
|
|
|
85
85
|
| `fdl push <working>` | Upload the committed version and move `main` to it |
|
|
86
86
|
| `fdl log <remote>` | List versions from `main` back to the first |
|
|
87
87
|
| `fdl status <working>` | What the working metadata changed since the clone |
|
|
88
|
-
| `fdl diff <remote> [version]` | What a version changed since its parent: tables, rows, columns, data files,
|
|
88
|
+
| `fdl diff <remote> [version]` | What a version changed since its parent: tables, rows, columns, data files, dataset.json |
|
|
89
89
|
| `fdl show <remote> [version]` | A version's record, its tables and its changes |
|
|
90
90
|
| `fdl url <remote> [--attach]` | Print the metadata URL of the version `main` holds, or the whole `ATTACH` statement |
|
|
91
91
|
| `fdl tag <remote> [name] [version]` | List tags, tag a version, or delete a tag (`-d`) |
|
|
@@ -315,7 +315,7 @@
|
|
|
315
315
|
"deleted": []
|
|
316
316
|
},
|
|
317
317
|
{
|
|
318
|
-
"name": "keeps the
|
|
318
|
+
"name": "keeps the files a live version's tree lists under objects/, not under assets/",
|
|
319
319
|
"files": {
|
|
320
320
|
"fdl.json": {
|
|
321
321
|
"fdl": "0.1",
|
|
@@ -374,11 +374,13 @@
|
|
|
374
374
|
}
|
|
375
375
|
},
|
|
376
376
|
"objects": {
|
|
377
|
-
"
|
|
378
|
-
"
|
|
377
|
+
"objects/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": "2026-11-01T00:00:00Z",
|
|
378
|
+
"objects/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb": "2026-11-01T00:00:00Z",
|
|
379
|
+
"assets/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": "2026-11-01T00:00:00Z"
|
|
379
380
|
},
|
|
380
381
|
"deleted": [
|
|
381
|
-
"assets/
|
|
382
|
+
"assets/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
|
383
|
+
"objects/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
|
382
384
|
]
|
|
383
385
|
},
|
|
384
386
|
{
|
|
@@ -5,15 +5,15 @@
|
|
|
5
5
|
"fdl_json": {
|
|
6
6
|
"fdl": "0.1"
|
|
7
7
|
},
|
|
8
|
-
"data_dir": "
|
|
8
|
+
"data_dir": "tables/"
|
|
9
9
|
},
|
|
10
10
|
{
|
|
11
11
|
"name": "the default, written out",
|
|
12
12
|
"fdl_json": {
|
|
13
13
|
"fdl": "0.1",
|
|
14
|
-
"data_dir": "
|
|
14
|
+
"data_dir": "tables/"
|
|
15
15
|
},
|
|
16
|
-
"data_dir": "
|
|
16
|
+
"data_dir": "tables/"
|
|
17
17
|
},
|
|
18
18
|
{
|
|
19
19
|
"name": "dots inside a segment",
|
|
@@ -191,6 +191,14 @@
|
|
|
191
191
|
},
|
|
192
192
|
"data_dir": null
|
|
193
193
|
},
|
|
194
|
+
{
|
|
195
|
+
"name": "objects",
|
|
196
|
+
"fdl_json": {
|
|
197
|
+
"fdl": "0.1",
|
|
198
|
+
"data_dir": "objects/"
|
|
199
|
+
},
|
|
200
|
+
"data_dir": null
|
|
201
|
+
},
|
|
194
202
|
{
|
|
195
203
|
"name": "not a string",
|
|
196
204
|
"fdl_json": {
|
|
@@ -28,8 +28,9 @@ Readers only read. A static HTTP server in front of the repository is enough for
|
|
|
28
28
|
refs/heads/<name> branches
|
|
29
29
|
refs/tags/<name> tags
|
|
30
30
|
versions/<ID>/ one folder per version
|
|
31
|
-
|
|
32
|
-
assets/<sha256> files a
|
|
31
|
+
tables/ Parquet files, shared by all versions
|
|
32
|
+
assets/<sha256> files the rows of a table point at, such as PDFs and images
|
|
33
|
+
objects/<sha256> files a version holds beside its tables, such as its README
|
|
33
34
|
```
|
|
34
35
|
|
|
35
36
|
The repository holds no metadata outside a version folder, and never a working metadata file. Either would hand readers a state that was never published.
|
|
@@ -36,20 +36,21 @@ towns/
|
|
|
36
36
|
You never write to the repository directly. You write to a working metadata file, a DuckLake metadata file in a SQLite file on your own disk, and turn it into a version when you are done.
|
|
37
37
|
|
|
38
38
|
```bash
|
|
39
|
-
|
|
39
|
+
mkdir work && cd work
|
|
40
|
+
fdl clone ../towns/ lake.sqlite
|
|
40
41
|
```
|
|
41
42
|
|
|
42
43
|
Before the first version, `clone` makes an empty metadata. Later it copies the metadata of the version `main` holds. It prints the statement to attach the working metadata with:
|
|
43
44
|
|
|
44
45
|
```sql
|
|
45
|
-
ATTACH 'ducklake:
|
|
46
|
+
ATTACH 'ducklake:lake.sqlite' AS lake (
|
|
46
47
|
DATA_PATH '/abs/path/towns/data/', OVERRIDE_DATA_PATH true,
|
|
47
48
|
META_TYPE 'sqlite', DATA_INLINING_ROW_LIMIT 0,
|
|
48
49
|
META_JOURNAL_MODE 'WAL', BUSY_TIMEOUT 5000
|
|
49
50
|
);
|
|
50
51
|
```
|
|
51
52
|
|
|
52
|
-
Beside the working metadata, `
|
|
53
|
+
Beside the working metadata, `lake.sqlite.fdl.json` remembers the repository, the branch and the version the clone was made from.
|
|
53
54
|
|
|
54
55
|
## Write
|
|
55
56
|
|
|
@@ -57,14 +58,14 @@ Use any DuckDB client. The Parquet files go straight into the repository's `data
|
|
|
57
58
|
|
|
58
59
|
```sql
|
|
59
60
|
CREATE TABLE lake.main.towns AS
|
|
60
|
-
SELECT * FROM 'towns.csv';
|
|
61
|
+
SELECT * FROM '../towns.csv';
|
|
61
62
|
```
|
|
62
63
|
|
|
63
64
|
`DATA_INLINING_ROW_LIMIT 0` makes DuckLake write every row to Parquet. Left at its default, DuckLake keeps small tables inside the metadata, and a version has to carry its rows in data files that a reader can fetch.
|
|
64
65
|
|
|
65
66
|
## Describe
|
|
66
67
|
|
|
67
|
-
Every version carries a `dataset.json` that says what it holds. Write one
|
|
68
|
+
Every version carries a `dataset.json` that says what it holds. Write one in the folder where you run `fdl commit`:
|
|
68
69
|
|
|
69
70
|
```json dataset.json
|
|
70
71
|
{
|
|
@@ -74,13 +75,13 @@ Every version carries a `dataset.json` that says what it holds. Write one next t
|
|
|
74
75
|
}
|
|
75
76
|
```
|
|
76
77
|
|
|
77
|
-
Only `name`
|
|
78
|
+
Only `name` is required. `fdl commit` checks the file against the metadata and fills in each table's row count and each column's type. Every other file in the folder, such as a `README.md`, goes into the version beside it, apart from hidden files and the working metadata with the files `fdl` keeps beside it.
|
|
78
79
|
|
|
79
80
|
## Commit and push
|
|
80
81
|
|
|
81
82
|
```bash
|
|
82
|
-
fdl commit
|
|
83
|
-
fdl push
|
|
83
|
+
fdl commit lake.sqlite -m "First towns" --author alice
|
|
84
|
+
fdl push lake.sqlite
|
|
84
85
|
```
|
|
85
86
|
|
|
86
87
|
`commit` freezes the working metadata into a version on your disk. `push` measures each table into a small summary (a profile) that readers can show without reading the data, uploads the version with them and moves `main` to it, and prints `refs/heads/main -> 019a3c4e-… (version 1)`. The repository now holds:
|
|
@@ -96,11 +97,11 @@ towns/
|
|
|
96
97
|
files.json
|
|
97
98
|
profiles/main/towns.json
|
|
98
99
|
version.json
|
|
99
|
-
|
|
100
|
+
tables/main/towns/ducklake-….parquet
|
|
100
101
|
```
|
|
101
102
|
|
|
102
103
|
```bash
|
|
103
|
-
fdl log
|
|
104
|
+
fdl log ../towns/
|
|
104
105
|
```
|
|
105
106
|
|
|
106
107
|
```
|
|
@@ -111,14 +112,14 @@ fdl log ./towns/
|
|
|
111
112
|
## Read
|
|
112
113
|
|
|
113
114
|
```bash
|
|
114
|
-
fdl url
|
|
115
|
+
fdl url ../towns/
|
|
115
116
|
```
|
|
116
117
|
|
|
117
118
|
prints the metadata of the version `main` holds. Attach it read-only, and tell DuckDB where the data files are:
|
|
118
119
|
|
|
119
120
|
```sql
|
|
120
|
-
ATTACH 'ducklake
|
|
121
|
-
DATA_PATH 'towns/data/', OVERRIDE_DATA_PATH true, READ_ONLY
|
|
121
|
+
ATTACH 'ducklake:../towns/versions/019a3c4e-…/metadata.db' AS towns (
|
|
122
|
+
DATA_PATH '../towns/data/', OVERRIDE_DATA_PATH true, READ_ONLY
|
|
122
123
|
);
|
|
123
124
|
SELECT count(*) FROM towns.main.towns;
|
|
124
125
|
```
|
|
@@ -128,19 +129,19 @@ A version's metadata never changes, so a reader may cache it forever. Only `HEAD
|
|
|
128
129
|
## Make a second version
|
|
129
130
|
|
|
130
131
|
```bash
|
|
131
|
-
fdl clone
|
|
132
|
+
fdl clone ../towns/ lake.sqlite
|
|
132
133
|
```
|
|
133
134
|
|
|
134
135
|
```sql
|
|
135
|
-
INSERT INTO lake.main.towns SELECT * FROM 'towns-2026.csv';
|
|
136
|
+
INSERT INTO lake.main.towns SELECT * FROM '../towns-2026.csv';
|
|
136
137
|
```
|
|
137
138
|
|
|
138
139
|
```bash
|
|
139
|
-
fdl commit
|
|
140
|
-
fdl push
|
|
140
|
+
fdl commit lake.sqlite -m "Add 2026" --author alice
|
|
141
|
+
fdl push lake.sqlite
|
|
141
142
|
```
|
|
142
143
|
|
|
143
|
-
The current folder already holds `dataset.json`, so the clone leaves it as it is. In an empty folder, the clone also writes the version's `dataset.json
|
|
144
|
+
The current folder already holds `dataset.json`, so the clone leaves it as it is. In an empty folder, the clone also writes there the version's `dataset.json` and the other files it holds, so whoever clones the repository can change the description as well as the data.
|
|
144
145
|
|
|
145
146
|
The new version lists the first as its parent. Both metadata files name the first Parquet file; only the second names the new one.
|
|
146
147
|
|
|
@@ -9,14 +9,15 @@ A mirror is a copy of a repository on another host. Since versions and data file
|
|
|
9
9
|
|
|
10
10
|
Copy in the same order a push writes, so a reader of the mirror never follows a ref to a version that is not all there:
|
|
11
11
|
|
|
12
|
-
1. data files under `
|
|
12
|
+
1. data files under `tables/`, assets under `assets/` and tree files under `objects/` that the mirror lacks
|
|
13
13
|
2. version folders under `versions/` that the mirror lacks, `version.json` last in each
|
|
14
14
|
3. `fdl.json`, if it changed
|
|
15
15
|
4. `refs/`, `HEAD` and `refs.json`
|
|
16
16
|
|
|
17
17
|
```bash
|
|
18
|
-
aws s3 sync s3://origin/towns/
|
|
18
|
+
aws s3 sync s3://origin/towns/tables/ s3://mirror/towns/tables/
|
|
19
19
|
aws s3 sync s3://origin/towns/assets/ s3://mirror/towns/assets/
|
|
20
|
+
aws s3 sync s3://origin/towns/objects/ s3://mirror/towns/objects/
|
|
20
21
|
aws s3 sync s3://origin/towns/versions/ s3://mirror/towns/versions/
|
|
21
22
|
aws s3 cp s3://origin/towns/fdl.json s3://mirror/towns/fdl.json
|
|
22
23
|
aws s3 sync s3://origin/towns/refs/ s3://mirror/towns/refs/ --delete
|
|
@@ -18,12 +18,12 @@ A version of the public repository is made from a version of the private one:
|
|
|
18
18
|
|
|
19
19
|
1. Clone the public repository
|
|
20
20
|
2. Copy the chosen tables from the private version into the working metadata. The Parquet files are copied into the public repository's data folder
|
|
21
|
-
3. Commit, and name the private version in `
|
|
21
|
+
3. Commit, and name the private version in the `annotations` of `version.json`, under a name of your own
|
|
22
22
|
|
|
23
23
|
```json
|
|
24
|
-
"
|
|
25
|
-
{ "
|
|
26
|
-
|
|
24
|
+
"annotations": {
|
|
25
|
+
"towns-publisher": { "made_from": { "version": "019a3c4e-7b21-7f3a-9c55-2f0d8e1a4b6c", "sha256": "9f2c…" } }
|
|
26
|
+
}
|
|
27
27
|
```
|
|
28
28
|
|
|
29
29
|
`sha256` is the private version's metadata checksum. Readers of the public repository see that the version was made from another one, without being able to open it.
|
|
@@ -15,7 +15,7 @@ Readers only read files, so a repository can be served by any HTTP server, a buc
|
|
|
15
15
|
|
|
16
16
|
```sql
|
|
17
17
|
ATTACH 'ducklake:https://data.example.org/towns/versions/019a3c4e-…/metadata.db' AS towns (
|
|
18
|
-
DATA_PATH 'https://data.example.org/towns/
|
|
18
|
+
DATA_PATH 'https://data.example.org/towns/tables/', OVERRIDE_DATA_PATH true, READ_ONLY
|
|
19
19
|
);
|
|
20
20
|
```
|
|
21
21
|
|
|
@@ -26,11 +26,12 @@ ATTACH 'ducklake:https://data.example.org/towns/versions/019a3c4e-…/metadata.d
|
|
|
26
26
|
| `HEAD`, `refs/**` | `no-store` | `text/plain; charset=utf-8` |
|
|
27
27
|
| `refs.json` | `no-store` | `application/json` |
|
|
28
28
|
| `versions/**` | `public, max-age=31536000, immutable` | as the file |
|
|
29
|
-
| `
|
|
29
|
+
| `tables/**` | `public, max-age=31536000, immutable` | `application/octet-stream` |
|
|
30
30
|
| `assets/**` | `public, max-age=31536000, immutable` | as uploaded |
|
|
31
|
+
| `objects/**` | `public, max-age=31536000, immutable` | as uploaded |
|
|
31
32
|
| `fdl.json` | a short `max-age` | `application/json` |
|
|
32
33
|
|
|
33
|
-
A version folder and a data file never change once written, so they can be cached for as long as the CDN allows. Only `HEAD` and the refs change, and a cached ref sends readers to an old version without telling them. `expired.json` can appear in a version folder later, so a host that expires versions should purge `versions/<ID>/` from its CDN when it does. On withdrawal, it also purges the data files and `
|
|
34
|
+
A version folder and a data file never change once written, so they can be cached for as long as the CDN allows. Only `HEAD` and the refs change, and a cached ref sends readers to an old version without telling them. `expired.json` can appear in a version folder later, so a host that expires versions should purge `versions/<ID>/` from its CDN when it does. On withdrawal, it also purges the data files, `assets/<sha256>` and `objects/<sha256>` it deleted.
|
|
34
35
|
|
|
35
36
|
The server must answer range requests (`Accept-Ranges: bytes`, `206 Partial Content`). DuckDB falls back to whole-file downloads without them, which is slow for large Parquet files.
|
|
36
37
|
|
|
@@ -15,7 +15,7 @@ s3://my-bucket/towns/
|
|
|
15
15
|
versions/019a3c4e-…/
|
|
16
16
|
version.json parents, author, message, sha256
|
|
17
17
|
metadata.db the frozen metadata
|
|
18
|
-
|
|
18
|
+
tables/main/towns/ducklake-….parquet
|
|
19
19
|
```
|
|
20
20
|
|
|
21
21
|
A reader needs nothing but DuckDB: it reads `refs/heads/main`, then attaches the metadata of that version read-only, over HTTP or S3. A writer needs a store that can read, write and write on condition. There is no server.
|
|
@@ -33,7 +33,7 @@ After `fdl clone` and a `CREATE TABLE`, DuckLake has written Parquet into the re
|
|
|
33
33
|
```
|
|
34
34
|
fdl.json
|
|
35
35
|
HEAD
|
|
36
|
-
|
|
36
|
+
tables/main/towns/ducklake-0199f1….parquet
|
|
37
37
|
```
|
|
38
38
|
|
|
39
39
|
No ref and no version names the file, so no reader can reach it. If the writer gives up, the file is left behind and maintenance deletes it after the grace period.
|
|
@@ -59,7 +59,7 @@ refs/heads/main 019a2f10-…
|
|
|
59
59
|
versions/019a2f10-…/metadata.db
|
|
60
60
|
versions/019a2f10-…/dataset.json
|
|
61
61
|
versions/019a2f10-…/version.json
|
|
62
|
-
|
|
62
|
+
tables/main/towns/ducklake-0199f1….parquet
|
|
63
63
|
```
|
|
64
64
|
|
|
65
65
|
The order is what keeps readers safe. A reader starts from a ref, and the ref is written last. If the push stops after step 6, 7 or 8, the repository holds a version folder no ref points at. Readers never see it, and it changes nothing.
|
|
@@ -76,8 +76,8 @@ versions/019a2f10-…/…
|
|
|
76
76
|
versions/019a3c4e-…/metadata.db
|
|
77
77
|
versions/019a3c4e-…/dataset.json
|
|
78
78
|
versions/019a3c4e-…/version.json
|
|
79
|
-
|
|
80
|
-
|
|
79
|
+
tables/main/towns/ducklake-0199f1….parquet named by both versions
|
|
80
|
+
tables/main/towns/ducklake-019a3b….parquet named by the second only
|
|
81
81
|
```
|
|
82
82
|
|
|
83
83
|
```json versions/019a3c4e-…/version.json
|
|
@@ -28,8 +28,9 @@ description: リポジトリとは何か、置き場に何が求められるか
|
|
|
28
28
|
refs/heads/<name> ブランチ
|
|
29
29
|
refs/tags/<name> タグ
|
|
30
30
|
versions/<ID>/ バージョンごとのフォルダ
|
|
31
|
-
|
|
32
|
-
assets/<sha256>
|
|
31
|
+
tables/ Parquet ファイル。全バージョンで共有
|
|
32
|
+
assets/<sha256> テーブルの行が指すファイル(PDF・画像など)
|
|
33
|
+
objects/<sha256> バージョンがテーブルのほかに持つファイル(README など)
|
|
33
34
|
```
|
|
34
35
|
|
|
35
36
|
リポジトリには、バージョンのフォルダの外にメタデータを置きません。作業メタデータも置きません。どちらも、公開していない状態を読み手に渡してしまいます。
|
|
@@ -36,20 +36,21 @@ towns/
|
|
|
36
36
|
リポジトリには直接書き込みません。自分のディスクにある SQLite ファイルの DuckLake メタデータ(作業メタデータ)に書き、書き終えたらそれをバージョンにします。
|
|
37
37
|
|
|
38
38
|
```bash
|
|
39
|
-
|
|
39
|
+
mkdir work && cd work
|
|
40
|
+
fdl clone ../towns/ lake.sqlite
|
|
40
41
|
```
|
|
41
42
|
|
|
42
43
|
最初のバージョンができる前は、`clone` は空のメタデータを作ります。その後は、`main` が指すバージョンのメタデータを写します。作業メタデータを ATTACH する文を表示します。
|
|
43
44
|
|
|
44
45
|
```sql
|
|
45
|
-
ATTACH 'ducklake:
|
|
46
|
+
ATTACH 'ducklake:lake.sqlite' AS lake (
|
|
46
47
|
DATA_PATH '/abs/path/towns/data/', OVERRIDE_DATA_PATH true,
|
|
47
48
|
META_TYPE 'sqlite', DATA_INLINING_ROW_LIMIT 0,
|
|
48
49
|
META_JOURNAL_MODE 'WAL', BUSY_TIMEOUT 5000
|
|
49
50
|
);
|
|
50
51
|
```
|
|
51
52
|
|
|
52
|
-
作業メタデータの隣の `
|
|
53
|
+
作業メタデータの隣の `lake.sqlite.fdl.json` に、リポジトリ、ブランチ、クローン元のバージョンが記録されます。
|
|
53
54
|
|
|
54
55
|
## 書く
|
|
55
56
|
|
|
@@ -57,14 +58,14 @@ DuckDB のクライアントなら何でも使えます。Parquet ファイル
|
|
|
57
58
|
|
|
58
59
|
```sql
|
|
59
60
|
CREATE TABLE lake.main.towns AS
|
|
60
|
-
SELECT * FROM 'towns.csv';
|
|
61
|
+
SELECT * FROM '../towns.csv';
|
|
61
62
|
```
|
|
62
63
|
|
|
63
64
|
`DATA_INLINING_ROW_LIMIT 0` を付けると、DuckLake はすべての行を Parquet に書きます。既定のままだと小さな表をメタデータの中に持ちますが、バージョンは行を、読み手が取得できるデータファイルとして持つ必要があります。
|
|
64
65
|
|
|
65
66
|
## 説明を書く
|
|
66
67
|
|
|
67
|
-
バージョンにはそれぞれ、中身を説明する `dataset.json` を付けます。`fdl commit`
|
|
68
|
+
バージョンにはそれぞれ、中身を説明する `dataset.json` を付けます。`fdl commit` を実行するフォルダに置いてください。
|
|
68
69
|
|
|
69
70
|
```json dataset.json
|
|
70
71
|
{
|
|
@@ -74,13 +75,13 @@ SELECT * FROM 'towns.csv';
|
|
|
74
75
|
}
|
|
75
76
|
```
|
|
76
77
|
|
|
77
|
-
必須なのは `name`
|
|
78
|
+
必須なのは `name` だけです。`fdl commit` がこのファイルをメタデータと照らし合わせ、表ごとの行数と列ごとの型を書き足します。フォルダにあるほかのファイル(`README.md` など)も、隠しファイルと、作業メタデータと `fdl` がその隣に置くファイルを除いて、すべてバージョンに入ります。
|
|
78
79
|
|
|
79
80
|
## コミットして push する
|
|
80
81
|
|
|
81
82
|
```bash
|
|
82
|
-
fdl commit
|
|
83
|
-
fdl push
|
|
83
|
+
fdl commit lake.sqlite -m "First towns" --author alice
|
|
84
|
+
fdl push lake.sqlite
|
|
84
85
|
```
|
|
85
86
|
|
|
86
87
|
`commit` は作業メタデータを固めて手元でバージョンにします。`push` はテーブルごとに、データを読まずに表示できる小さな要約(profile)を測り、バージョンと一緒にアップロードして `main` をそこへ動かし、`refs/heads/main -> 019a3c4e-… (version 1)` と表示します。リポジトリは次のようになります。
|
|
@@ -96,11 +97,11 @@ towns/
|
|
|
96
97
|
files.json
|
|
97
98
|
profiles/main/towns.json
|
|
98
99
|
version.json
|
|
99
|
-
|
|
100
|
+
tables/main/towns/ducklake-….parquet
|
|
100
101
|
```
|
|
101
102
|
|
|
102
103
|
```bash
|
|
103
|
-
fdl log
|
|
104
|
+
fdl log ../towns/
|
|
104
105
|
```
|
|
105
106
|
|
|
106
107
|
```
|
|
@@ -111,14 +112,14 @@ fdl log ./towns/
|
|
|
111
112
|
## 読む
|
|
112
113
|
|
|
113
114
|
```bash
|
|
114
|
-
fdl url
|
|
115
|
+
fdl url ../towns/
|
|
115
116
|
```
|
|
116
117
|
|
|
117
118
|
`main` が指すバージョンのメタデータを表示します。読み取り専用で ATTACH し、データファイルの場所を DuckDB に渡してください。
|
|
118
119
|
|
|
119
120
|
```sql
|
|
120
|
-
ATTACH 'ducklake
|
|
121
|
-
DATA_PATH 'towns/data/', OVERRIDE_DATA_PATH true, READ_ONLY
|
|
121
|
+
ATTACH 'ducklake:../towns/versions/019a3c4e-…/metadata.db' AS towns (
|
|
122
|
+
DATA_PATH '../towns/data/', OVERRIDE_DATA_PATH true, READ_ONLY
|
|
122
123
|
);
|
|
123
124
|
SELECT count(*) FROM towns.main.towns;
|
|
124
125
|
```
|
|
@@ -128,19 +129,19 @@ SELECT count(*) FROM towns.main.towns;
|
|
|
128
129
|
## 2 つめのバージョンを作る
|
|
129
130
|
|
|
130
131
|
```bash
|
|
131
|
-
fdl clone
|
|
132
|
+
fdl clone ../towns/ lake.sqlite
|
|
132
133
|
```
|
|
133
134
|
|
|
134
135
|
```sql
|
|
135
|
-
INSERT INTO lake.main.towns SELECT * FROM 'towns-2026.csv';
|
|
136
|
+
INSERT INTO lake.main.towns SELECT * FROM '../towns-2026.csv';
|
|
136
137
|
```
|
|
137
138
|
|
|
138
139
|
```bash
|
|
139
|
-
fdl commit
|
|
140
|
-
fdl push
|
|
140
|
+
fdl commit lake.sqlite -m "Add 2026" --author alice
|
|
141
|
+
fdl push lake.sqlite
|
|
141
142
|
```
|
|
142
143
|
|
|
143
|
-
今のフォルダには `dataset.json` があるので、クローンはそれをそのままにします。空のフォルダでは、バージョンの `dataset.json
|
|
144
|
+
今のフォルダには `dataset.json` があるので、クローンはそれをそのままにします。空のフォルダでは、バージョンの `dataset.json` とバージョンが持つほかのファイルもそこに書き出すので、リポジトリをクローンした人はデータだけでなく説明も変えられます。
|
|
144
145
|
|
|
145
146
|
新しいバージョンは、最初のバージョンを親として記録します。最初の Parquet ファイルは両方のメタデータが参照し、新しいファイルは 2 つめのメタデータだけが参照します。
|
|
146
147
|
|
|
@@ -9,14 +9,15 @@ description: リポジトリを別のホストに写し、最新に保ち、写
|
|
|
9
9
|
|
|
10
10
|
push と同じ順に写します。そうすれば、ミラーの読み手が、まだ揃っていないバージョンを ref からたどることはありません。
|
|
11
11
|
|
|
12
|
-
1. ミラーに無い `
|
|
12
|
+
1. ミラーに無い `tables/` の下のデータファイル、`assets/` の下のアセット、`objects/` の下の tree のファイル
|
|
13
13
|
2. ミラーに無い `versions/` の下のバージョンのフォルダ。各フォルダで `version.json` を最後に
|
|
14
14
|
3. 変わっていれば `fdl.json`
|
|
15
15
|
4. `refs/`、`HEAD`、`refs.json`
|
|
16
16
|
|
|
17
17
|
```bash
|
|
18
|
-
aws s3 sync s3://origin/towns/
|
|
18
|
+
aws s3 sync s3://origin/towns/tables/ s3://mirror/towns/tables/
|
|
19
19
|
aws s3 sync s3://origin/towns/assets/ s3://mirror/towns/assets/
|
|
20
|
+
aws s3 sync s3://origin/towns/objects/ s3://mirror/towns/objects/
|
|
20
21
|
aws s3 sync s3://origin/towns/versions/ s3://mirror/towns/versions/
|
|
21
22
|
aws s3 cp s3://origin/towns/fdl.json s3://mirror/towns/fdl.json
|
|
22
23
|
aws s3 sync s3://origin/towns/refs/ s3://mirror/towns/refs/ --delete
|