fdl-cli 0.21.1__tar.gz → 0.23.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fdl_cli-0.23.0/.github/workflows/test.yml +41 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/CHANGELOG.md +55 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/CLAUDE.md +1 -1
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/CONTRIBUTING.md +12 -4
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/PKG-INFO +28 -8
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/README.md +24 -4
- fdl_cli-0.23.0/SECURITY.md +7 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/blume.config.ts +1 -1
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/concepts/versions-and-refs.md +14 -3
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/concepts/working-catalogs.md +6 -5
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/getting-started.md +7 -3
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/index.md +24 -2
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/internals/from-write-to-read.md +1 -1
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/internals/inside-a-repository.md +2 -2
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/concepts/versions-and-refs.md +14 -3
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/concepts/working-catalogs.md +6 -5
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/getting-started.md +7 -3
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/index.md +23 -1
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/internals/from-write-to-read.md +1 -1
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/internals/inside-a-repository.md +2 -2
- fdl_cli-0.23.0/docs/ja/reference/cli.md +239 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/reference/python.md +37 -3
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/spec/data-files.md +2 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/spec/http.md +2 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/spec/layout.md +2 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/spec/reading.md +11 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/spec/refs.md +2 -2
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/spec/versions.md +45 -8
- fdl_cli-0.23.0/docs/ja/spec/writing.md +115 -0
- fdl_cli-0.23.0/docs/ja/stability.md +40 -0
- fdl_cli-0.23.0/docs/reference/cli.md +239 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/reference/python.md +37 -3
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/spec/data-files.md +2 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/spec/http.md +2 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/spec/layout.md +2 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/spec/reading.md +11 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/spec/refs.md +2 -2
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/spec/versions.md +45 -8
- fdl_cli-0.23.0/docs/spec/writing.md +115 -0
- fdl_cli-0.23.0/docs/stability.md +40 -0
- fdl_cli-0.23.0/public/CNAME +1 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/pyproject.toml +4 -4
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/src/fdl/catalog.py +361 -7
- fdl_cli-0.23.0/src/fdl/cli.py +891 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/src/fdl/dataset.py +23 -6
- fdl_cli-0.23.0/src/fdl/diff.py +298 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/src/fdl/layout.py +42 -0
- fdl_cli-0.23.0/src/fdl/lineage.py +127 -0
- fdl_cli-0.23.0/src/fdl/merge.py +476 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/src/fdl/refs.py +5 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/src/fdl/repo.py +263 -35
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/src/fdl/store.py +5 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/src/fdl/stores.py +82 -14
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/src/fdl/version.py +15 -11
- fdl_cli-0.23.0/tests/test_cli.py +460 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/tests/test_dataset.py +81 -3
- fdl_cli-0.23.0/tests/test_http.py +413 -0
- fdl_cli-0.23.0/tests/test_merge.py +336 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/tests/test_objects.py +15 -9
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/tests/test_profiles.py +12 -8
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/tests/test_repo.py +144 -3
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/uv.lock +2 -2
- fdl_cli-0.21.1/docs/ja/reference/cli.md +0 -123
- fdl_cli-0.21.1/docs/ja/spec/writing.md +0 -58
- fdl_cli-0.21.1/docs/reference/cli.md +0 -123
- fdl_cli-0.21.1/docs/spec/writing.md +0 -58
- fdl_cli-0.21.1/public/CNAME +0 -1
- fdl_cli-0.21.1/src/fdl/cli.py +0 -323
- fdl_cli-0.21.1/tests/test_cli.py +0 -203
- fdl_cli-0.21.1/tests/test_http.py +0 -219
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/.github/workflows/docs.yml +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/.github/workflows/publish.yml +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/.github/workflows/release.yml +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/.gitignore +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/.node-version +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/.python-version +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/LICENSE +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/concepts/meta.ts +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/concepts/repositories.md +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/guides/building-a-host.md +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/guides/meta.ts +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/guides/mirrors.md +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/guides/public-and-private.md +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/guides/reading-without-duckdb.md +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/guides/serving-over-http.md +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/guides/stores.md +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/internals/meta.ts +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/concepts/meta.ts +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/concepts/repositories.md +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/guides/building-a-host.md +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/guides/meta.ts +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/guides/mirrors.md +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/guides/public-and-private.md +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/guides/reading-without-duckdb.md +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/guides/serving-over-http.md +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/guides/stores.md +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/internals/meta.ts +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/reference/meta.ts +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/spec/index.md +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/ja/spec/meta.ts +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/reference/meta.ts +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/spec/index.md +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/docs/spec/meta.ts +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/package-lock.json +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/package.json +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/src/fdl/__init__.py +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/src/fdl/objects.py +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/src/fdl/profiles.py +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/src/fdl/py.typed +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/tests/http_repo.py +0 -0
- {fdl_cli-0.21.1 → fdl_cli-0.23.0}/tests/test_stores.py +0 -0
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
name: Test
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
pull_request:
|
|
5
|
+
push:
|
|
6
|
+
branches: [main]
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
pytest:
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
strategy:
|
|
15
|
+
fail-fast: false
|
|
16
|
+
matrix:
|
|
17
|
+
# The oldest Python the package supports, and the newest.
|
|
18
|
+
python: ["3.10", "3.13"]
|
|
19
|
+
# Catalogs written by either have to read on the other.
|
|
20
|
+
duckdb: ["1.5", "2.0"]
|
|
21
|
+
exclude:
|
|
22
|
+
# DuckDB 2.0 needs Python 3.11 or later.
|
|
23
|
+
- python: "3.10"
|
|
24
|
+
duckdb: "2.0"
|
|
25
|
+
steps:
|
|
26
|
+
- uses: actions/checkout@v5
|
|
27
|
+
- uses: astral-sh/setup-uv@v7
|
|
28
|
+
- run: uv sync --python ${{ matrix.python }}
|
|
29
|
+
- run: uv pip install --python .venv "duckdb~=${{ matrix.duckdb }}.0"
|
|
30
|
+
- run: uv run --no-sync pytest -q
|
|
31
|
+
|
|
32
|
+
docs:
|
|
33
|
+
runs-on: ubuntu-latest
|
|
34
|
+
steps:
|
|
35
|
+
- uses: actions/checkout@v5
|
|
36
|
+
- uses: actions/setup-node@v5
|
|
37
|
+
with:
|
|
38
|
+
node-version-file: .node-version
|
|
39
|
+
cache: npm
|
|
40
|
+
- run: npm ci
|
|
41
|
+
- run: npm run check
|
|
@@ -1,6 +1,61 @@
|
|
|
1
1
|
# CHANGELOG
|
|
2
2
|
|
|
3
3
|
|
|
4
|
+
## v0.23.0 (2026-11-11)
|
|
5
|
+
|
|
6
|
+
### Bug Fixes
|
|
7
|
+
|
|
8
|
+
- Re-ask the credential helper on 401, check IDs and paths, and specify merging
|
|
9
|
+
([#50](https://github.com/queria-io/fdl/pull/50),
|
|
10
|
+
[`74b2d9b`](https://github.com/queria-io/fdl/commit/74b2d9b88dcf77f7ff6eb1c184919ccf62f51d88))
|
|
11
|
+
|
|
12
|
+
- Restore, pending commits, status and gc after review
|
|
13
|
+
([#49](https://github.com/queria-io/fdl/pull/49),
|
|
14
|
+
[`73799be`](https://github.com/queria-io/fdl/commit/73799bedd7baedbd67054910b161e46cf5ea3883))
|
|
15
|
+
|
|
16
|
+
### Features
|
|
17
|
+
|
|
18
|
+
- Fdl merge and fdl branch ([#51](https://github.com/queria-io/fdl/pull/51),
|
|
19
|
+
[`5e88708`](https://github.com/queria-io/fdl/commit/5e88708ade5ac84131a2a69dc285886c25af2fc6))
|
|
20
|
+
|
|
21
|
+
* feat: fdl merge and fdl branch
|
|
22
|
+
|
|
23
|
+
fdl merge merges another branch into the working catalog's branch, as the spec's Merging section
|
|
24
|
+
describes: a table only one side changed comes over, rows of a table both changed are merged by
|
|
25
|
+
the primary key both declare, and anything both changed differently is a conflict that makes no
|
|
26
|
+
version. A merge whose base is ours checks out theirs, and push then moves the branch there. fdl
|
|
27
|
+
branch lists, starts and deletes branches.
|
|
28
|
+
|
|
29
|
+
A push is now accepted when the version descends from what the branch holds, through every parent,
|
|
30
|
+
and is_ancestor no longer stops at seq, which a merge does not raise past the version it merged.
|
|
31
|
+
|
|
32
|
+
* fix: merge review findings: revert by first parent, temp names, table aliases, warnings
|
|
33
|
+
|
|
34
|
+
- Fdl revert, fdl push separate from fdl commit, and fdl gc
|
|
35
|
+
([#48](https://github.com/queria-io/fdl/pull/48),
|
|
36
|
+
[`6e3ade9`](https://github.com/queria-io/fdl/commit/6e3ade956131cc9a5cd70fe1da6ebffb0aaa3bb7))
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
## v0.22.0 (2026-11-10)
|
|
40
|
+
|
|
41
|
+
### Features
|
|
42
|
+
|
|
43
|
+
- Checkout --storage writes new data files to the repository's storage; keep unknown dataset.json
|
|
44
|
+
keys ([#47](https://github.com/queria-io/fdl/pull/47),
|
|
45
|
+
[`3df2120`](https://github.com/queria-io/fdl/commit/3df212081c9cf79ebfab2582981171462a88fa96))
|
|
46
|
+
|
|
47
|
+
- Diff, show and status; DuckDB 2.0; keep the token on its origin; check catalogs on checkout; docs
|
|
48
|
+
for a beta ([#44](https://github.com/queria-io/fdl/pull/44),
|
|
49
|
+
[`13b83d5`](https://github.com/queria-io/fdl/commit/13b83d59f0ad936895e312b46fecf755c24b6138))
|
|
50
|
+
|
|
51
|
+
- Record what changed since the parent in version.json
|
|
52
|
+
([#46](https://github.com/queria-io/fdl/pull/46),
|
|
53
|
+
[`eee1ec3`](https://github.com/queria-io/fdl/commit/eee1ec3099bceea5938c68c4d656182512ea4718))
|
|
54
|
+
|
|
55
|
+
- **dataset**: Read depends_on from a table's SQL ([#45](https://github.com/queria-io/fdl/pull/45),
|
|
56
|
+
[`75003ba`](https://github.com/queria-io/fdl/commit/75003ba5d97efed5868b0297a9e28953b69b13f7))
|
|
57
|
+
|
|
58
|
+
|
|
4
59
|
## v0.21.1 (2026-11-08)
|
|
5
60
|
|
|
6
61
|
### Bug Fixes
|
|
@@ -18,18 +18,26 @@ uv run pytest
|
|
|
18
18
|
## Project structure
|
|
19
19
|
|
|
20
20
|
```
|
|
21
|
-
docs/
|
|
21
|
+
docs/ — the documentation site (Blume); docs/spec/ is the specification, docs/ja/ the Japanese mirror
|
|
22
22
|
src/fdl/
|
|
23
23
|
├── layout.py — where each file of a repository sits
|
|
24
24
|
├── version.py — version.json and version IDs
|
|
25
25
|
├── refs.py — reading refs and moving them on condition
|
|
26
26
|
├── catalog.py — DuckLake catalogs in SQLite and DuckDB, and the conversion
|
|
27
|
-
├──
|
|
27
|
+
├── dataset.py — dataset.json: reading the description and checking it against the catalog
|
|
28
|
+
├── objects.py — assets and tree.json
|
|
29
|
+
├── profiles.py — the profile of each table
|
|
30
|
+
├── repo.py — init, checkout, commit, push, log, expire, withdraw
|
|
28
31
|
├── store.py — what a store has to provide
|
|
29
|
-
├── stores.py — a folder and
|
|
32
|
+
├── stores.py — a folder, S3 and a host over HTTP as stores
|
|
30
33
|
└── cli.py — the fdl command
|
|
34
|
+
tests/ — pytest; tests/http_repo.py is a small host over HTTP for the tests
|
|
31
35
|
```
|
|
32
36
|
|
|
37
|
+
Change the specification first, then the code. Every page in `docs/` has a Japanese counterpart under `docs/ja/`; change both in the same pull request. Check the docs with `npm ci && npm run check`.
|
|
38
|
+
|
|
39
|
+
Report a vulnerability privately, as [SECURITY.md](SECURITY.md) says.
|
|
40
|
+
|
|
33
41
|
## Commit conventions
|
|
34
42
|
|
|
35
43
|
Follow [Conventional Commits](https://www.conventionalcommits.org/ja/v1.0.0/). Commit messages are parsed by [python-semantic-release](https://python-semantic-release.readthedocs.io/) to determine version bumps and generate the CHANGELOG automatically.
|
|
@@ -49,7 +57,7 @@ Do not manually edit the version in pyproject.toml or modify CHANGELOG.md — th
|
|
|
49
57
|
|
|
50
58
|
## Documentation
|
|
51
59
|
|
|
52
|
-
Documentation is built with [Zensical](https://zensical.com/) and deployed to [fdl.
|
|
60
|
+
Documentation is built with [Zensical](https://zensical.com/) and deployed to [fdl.queria.io](https://fdl.queria.io/).
|
|
53
61
|
|
|
54
62
|
To preview locally:
|
|
55
63
|
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: fdl-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.23.0
|
|
4
4
|
Summary: Versions of a DuckLake catalog on object storage, kept the way Git keeps commits
|
|
5
|
-
Project-URL: Homepage, https://fdl.
|
|
6
|
-
Project-URL: Documentation, https://fdl.
|
|
5
|
+
Project-URL: Homepage, https://fdl.queria.io/
|
|
6
|
+
Project-URL: Documentation, https://fdl.queria.io/
|
|
7
7
|
Project-URL: Repository, https://github.com/queria-io/fdl
|
|
8
8
|
Project-URL: Changelog, https://github.com/queria-io/fdl/blob/main/CHANGELOG.md
|
|
9
9
|
Author: flo8s
|
|
10
10
|
License-Expression: MIT
|
|
11
11
|
License-File: LICENSE
|
|
12
12
|
Requires-Python: >=3.10
|
|
13
|
-
Requires-Dist: duckdb<
|
|
13
|
+
Requires-Dist: duckdb<3.0,>=1.5.2
|
|
14
14
|
Requires-Dist: pyyaml>=6.0
|
|
15
15
|
Requires-Dist: tomli>=1.1; python_version < '3.11'
|
|
16
16
|
Requires-Dist: typer>=0.15
|
|
@@ -39,7 +39,7 @@ FDL is a storage layout and a tool for it. Each version is a frozen catalog with
|
|
|
39
39
|
data/main/towns/ducklake-….parquet
|
|
40
40
|
```
|
|
41
41
|
|
|
42
|
-
The documentation and the specification are at [fdl.
|
|
42
|
+
The documentation and the specification are at [fdl.queria.io](https://fdl.queria.io).
|
|
43
43
|
|
|
44
44
|
## Quick start
|
|
45
45
|
|
|
@@ -52,6 +52,8 @@ fdl checkout s3://my-bucket/towns/ work/lake.sqlite
|
|
|
52
52
|
|
|
53
53
|
The package is `fdl-cli` on PyPI. It installs the `fdl` command and the `fdl` Python package (`import fdl`).
|
|
54
54
|
|
|
55
|
+
`fdl init` makes the repository in the store itself, as `git init --bare` does on a server. Your own copy to write to is the working catalog that `checkout` makes, a SQLite file.
|
|
56
|
+
|
|
55
57
|
Write to the working catalog with DuckDB, attached the way `checkout` prints:
|
|
56
58
|
|
|
57
59
|
```sql
|
|
@@ -64,10 +66,18 @@ CREATE TABLE lake.main.towns AS SELECT * FROM 'towns.csv';
|
|
|
64
66
|
|
|
65
67
|
Rows have to go to Parquet, not into the catalog (`DATA_INLINING_ROW_LIMIT 0`): a version is read over HTTP, and inlined rows would not be in it.
|
|
66
68
|
|
|
69
|
+
Describe what the version holds in `dataset.yml`, next to where you run `fdl commit`. Only `name` and `title` are required; `fdl commit` fills in each table's columns and row count from the catalog:
|
|
70
|
+
|
|
71
|
+
```yaml
|
|
72
|
+
name: towns
|
|
73
|
+
title: Towns
|
|
74
|
+
```
|
|
75
|
+
|
|
67
76
|
Then make a version and move `main` to it:
|
|
68
77
|
|
|
69
78
|
```bash
|
|
70
79
|
fdl commit work/lake.sqlite -m "First towns" --author alice
|
|
80
|
+
fdl push work/lake.sqlite
|
|
71
81
|
fdl log s3://my-bucket/towns/
|
|
72
82
|
```
|
|
73
83
|
|
|
@@ -91,12 +101,22 @@ Credentials and the endpoint come from boto3's usual places (`AWS_*` environment
|
|
|
91
101
|
| --- | --- |
|
|
92
102
|
| `fdl init <remote>` | Start a repository (`fdl.json` and `HEAD`) |
|
|
93
103
|
| `fdl checkout <remote> <working>` | Make a working catalog from the version `main` holds |
|
|
94
|
-
| `fdl commit <working> -m … --author …` | Freeze the working catalog into a version
|
|
104
|
+
| `fdl commit <working> -m … --author …` | Freeze the working catalog into a version |
|
|
105
|
+
| `fdl push <working>` | Upload the committed version and move `main` to it |
|
|
95
106
|
| `fdl log <remote>` | List versions from `main` back to the first |
|
|
96
|
-
| `fdl
|
|
107
|
+
| `fdl status <working>` | What the working catalog changed since the checkout |
|
|
108
|
+
| `fdl diff <remote> [version]` | What a version changed since its parent: tables, rows, columns, data files, descriptions |
|
|
109
|
+
| `fdl show <remote> [version]` | A version's record, its tables and its changes |
|
|
110
|
+
| `fdl url <remote> [--attach]` | Print the catalog URL of the version `main` holds, or the whole `ATTACH` statement |
|
|
111
|
+
| `fdl tag <remote> [name] [version]` | List tags, tag a version, or delete a tag (`-d`) |
|
|
112
|
+
| `fdl branch <remote> [name] [version]` | List branches, start a branch at a version, or delete a branch (`-d`) |
|
|
113
|
+
| `fdl merge <ref> <working> --author …` | Merge another branch into the working catalog's branch, rows matched by primary key |
|
|
97
114
|
| `fdl expire <remote> --keep N` | Mark older versions as past retention and delete their catalogs |
|
|
115
|
+
| `fdl gc <remote> --yes` | Delete data files no live version reaches, after the grace period |
|
|
116
|
+
| `fdl withdraw <remote> <version>…` | Remove versions that hold data that must not be published |
|
|
117
|
+
| `fdl assets <folder>` | List files for an asset column (path, sha256, bytes, media type) |
|
|
98
118
|
|
|
99
|
-
A push that is not built on what `main` holds now is refused: check out again and redo the work.
|
|
119
|
+
A push that is not built on what `main` holds now is refused: check out again and redo the work, or work on a branch and merge it.
|
|
100
120
|
|
|
101
121
|
## Python
|
|
102
122
|
|
|
@@ -19,7 +19,7 @@ FDL is a storage layout and a tool for it. Each version is a frozen catalog with
|
|
|
19
19
|
data/main/towns/ducklake-….parquet
|
|
20
20
|
```
|
|
21
21
|
|
|
22
|
-
The documentation and the specification are at [fdl.
|
|
22
|
+
The documentation and the specification are at [fdl.queria.io](https://fdl.queria.io).
|
|
23
23
|
|
|
24
24
|
## Quick start
|
|
25
25
|
|
|
@@ -32,6 +32,8 @@ fdl checkout s3://my-bucket/towns/ work/lake.sqlite
|
|
|
32
32
|
|
|
33
33
|
The package is `fdl-cli` on PyPI. It installs the `fdl` command and the `fdl` Python package (`import fdl`).
|
|
34
34
|
|
|
35
|
+
`fdl init` makes the repository in the store itself, as `git init --bare` does on a server. Your own copy to write to is the working catalog that `checkout` makes, a SQLite file.
|
|
36
|
+
|
|
35
37
|
Write to the working catalog with DuckDB, attached the way `checkout` prints:
|
|
36
38
|
|
|
37
39
|
```sql
|
|
@@ -44,10 +46,18 @@ CREATE TABLE lake.main.towns AS SELECT * FROM 'towns.csv';
|
|
|
44
46
|
|
|
45
47
|
Rows have to go to Parquet, not into the catalog (`DATA_INLINING_ROW_LIMIT 0`): a version is read over HTTP, and inlined rows would not be in it.
|
|
46
48
|
|
|
49
|
+
Describe what the version holds in `dataset.yml`, next to where you run `fdl commit`. Only `name` and `title` are required; `fdl commit` fills in each table's columns and row count from the catalog:
|
|
50
|
+
|
|
51
|
+
```yaml
|
|
52
|
+
name: towns
|
|
53
|
+
title: Towns
|
|
54
|
+
```
|
|
55
|
+
|
|
47
56
|
Then make a version and move `main` to it:
|
|
48
57
|
|
|
49
58
|
```bash
|
|
50
59
|
fdl commit work/lake.sqlite -m "First towns" --author alice
|
|
60
|
+
fdl push work/lake.sqlite
|
|
51
61
|
fdl log s3://my-bucket/towns/
|
|
52
62
|
```
|
|
53
63
|
|
|
@@ -71,12 +81,22 @@ Credentials and the endpoint come from boto3's usual places (`AWS_*` environment
|
|
|
71
81
|
| --- | --- |
|
|
72
82
|
| `fdl init <remote>` | Start a repository (`fdl.json` and `HEAD`) |
|
|
73
83
|
| `fdl checkout <remote> <working>` | Make a working catalog from the version `main` holds |
|
|
74
|
-
| `fdl commit <working> -m … --author …` | Freeze the working catalog into a version
|
|
84
|
+
| `fdl commit <working> -m … --author …` | Freeze the working catalog into a version |
|
|
85
|
+
| `fdl push <working>` | Upload the committed version and move `main` to it |
|
|
75
86
|
| `fdl log <remote>` | List versions from `main` back to the first |
|
|
76
|
-
| `fdl
|
|
87
|
+
| `fdl status <working>` | What the working catalog changed since the checkout |
|
|
88
|
+
| `fdl diff <remote> [version]` | What a version changed since its parent: tables, rows, columns, data files, descriptions |
|
|
89
|
+
| `fdl show <remote> [version]` | A version's record, its tables and its changes |
|
|
90
|
+
| `fdl url <remote> [--attach]` | Print the catalog URL of the version `main` holds, or the whole `ATTACH` statement |
|
|
91
|
+
| `fdl tag <remote> [name] [version]` | List tags, tag a version, or delete a tag (`-d`) |
|
|
92
|
+
| `fdl branch <remote> [name] [version]` | List branches, start a branch at a version, or delete a branch (`-d`) |
|
|
93
|
+
| `fdl merge <ref> <working> --author …` | Merge another branch into the working catalog's branch, rows matched by primary key |
|
|
77
94
|
| `fdl expire <remote> --keep N` | Mark older versions as past retention and delete their catalogs |
|
|
95
|
+
| `fdl gc <remote> --yes` | Delete data files no live version reaches, after the grace period |
|
|
96
|
+
| `fdl withdraw <remote> <version>…` | Remove versions that hold data that must not be published |
|
|
97
|
+
| `fdl assets <folder>` | List files for an asset column (path, sha256, bytes, media type) |
|
|
78
98
|
|
|
79
|
-
A push that is not built on what `main` holds now is refused: check out again and redo the work.
|
|
99
|
+
A push that is not built on what `main` holds now is refused: check out again and redo the work, or work on a branch and merge it.
|
|
80
100
|
|
|
81
101
|
## Python
|
|
82
102
|
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# Security
|
|
2
|
+
|
|
3
|
+
Report a vulnerability in fdl or in the FDL specification privately, through GitHub's [private vulnerability reporting](https://github.com/queria-io/fdl/security/advisories/new). Do not open a public issue for it.
|
|
4
|
+
|
|
5
|
+
Say what is affected (the command, the part of the specification, the version of fdl-cli), how to reproduce it, and what an attacker gains. We reply within a week.
|
|
6
|
+
|
|
7
|
+
Fixes go out in a patch release of fdl-cli, and the advisory is published once the release is on PyPI.
|
|
@@ -56,7 +56,7 @@ Refs are the only files in a repository that change. Everything else is written
|
|
|
56
56
|
|
|
57
57
|
## Moving a branch
|
|
58
58
|
|
|
59
|
-
A branch moves forward only. A new version can become `main` when
|
|
59
|
+
A branch moves forward only. A new version can become `main` when it descends from the version `main` holds now: that version is reached by following `parents`. This is a fast-forward in Git's terms.
|
|
60
60
|
|
|
61
61
|
```
|
|
62
62
|
main → A
|
|
@@ -64,9 +64,20 @@ main → A
|
|
|
64
64
|
Bob checked out A too, commits C (parent A), pushes: refused
|
|
65
65
|
```
|
|
66
66
|
|
|
67
|
-
Bob's push is refused because C is not built on B. If it were accepted, B would vanish from the history of `main` with no one deciding that. Bob checks out B and makes his change again.
|
|
67
|
+
Bob's push is refused because C is not built on B. If it were accepted, B would vanish from the history of `main` with no one deciding that. Bob checks out B and makes his change again. Redoing the change on the new version is usually a matter of running the same SQL again.
|
|
68
68
|
|
|
69
|
-
|
|
69
|
+
## Merging
|
|
70
|
+
|
|
71
|
+
Work that takes longer goes on a branch of its own, and is merged into `main` when it is ready:
|
|
72
|
+
|
|
73
|
+
```
|
|
74
|
+
main → A → B
|
|
75
|
+
\
|
|
76
|
+
feature C
|
|
77
|
+
fdl merge refs/heads/feature on main makes M (parents B, C): main → M
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
The merge compares each table in three versions: B, C and their common ancestor A. A table only one branch changed comes over as that branch holds it. When both changed the same table, the rows are matched by the `primary_key` both declare in `dataset.json`, and a row only one branch changed comes over. A row both changed differently, or a table without a primary key that both changed, is a conflict, and then no version is made. A row number added for the merge is not a primary key: it does not say which rows are the same row. See [Merging](/spec/writing#merging).
|
|
70
81
|
|
|
71
82
|
Moving a branch back, to undo a version, is allowed only when asked for explicitly. The version that was undone stays in the repository and in the history.
|
|
72
83
|
|
|
@@ -11,7 +11,8 @@ A working catalog is a DuckLake catalog in a SQLite file, on the writer's own di
|
|
|
11
11
|
| --- | --- | --- |
|
|
12
12
|
| check out | `fdl checkout` | Copy the catalog of the branch's version into SQLite, and remember that version |
|
|
13
13
|
| write | any DuckDB client | DuckLake writes Parquet into the repository and bookkeeping into SQLite |
|
|
14
|
-
| commit | `fdl commit` | Freeze the last snapshot into a new version,
|
|
14
|
+
| commit | `fdl commit` | Freeze the last snapshot into a new version, kept beside the working catalog |
|
|
15
|
+
| push | `fdl push` | Upload the version and move the branch to it |
|
|
15
16
|
|
|
16
17
|
Checking out copies DuckLake's bookkeeping tables from the frozen DuckDB file into SQLite as they are. Freezing does the reverse. Neither rewrites a Parquet file, so both are fast even for a large dataset: the catalog is small and the data stays where it is.
|
|
17
18
|
|
|
@@ -19,7 +20,7 @@ Checking out copies DuckLake's bookkeeping tables from the frozen DuckDB file in
|
|
|
19
20
|
|
|
20
21
|
A working catalog has to be written by DuckDB and kept between sessions. SQLite does both without a database server, works the same on a laptop and in a container, and copies in and out of DuckDB's format without loss.
|
|
21
22
|
|
|
22
|
-
There is one kind of working catalog. A host that writes on behalf of its users (a scheduled import, a web form) checks out, writes and
|
|
23
|
+
There is one kind of working catalog. A host that writes on behalf of its users (a scheduled import, a web form) checks out, writes, commits and pushes the same way a person does on a laptop. When two of them push at once, the conditional write on the ref settles it: one wins, and the other checks out again.
|
|
23
24
|
|
|
24
25
|
## Attaching
|
|
25
26
|
|
|
@@ -33,15 +34,15 @@ ATTACH 'ducklake:work/lake.sqlite' AS lake (
|
|
|
33
34
|
);
|
|
34
35
|
```
|
|
35
36
|
|
|
36
|
-
- `DATA_PATH` is the repository's data folder, so new Parquet files land where readers will look for them. For a repository over HTTP it is a folder beside the working catalog instead, and `fdl
|
|
37
|
+
- `DATA_PATH` is the repository's data folder, so new Parquet files land where readers will look for them. For a repository over HTTP it is a folder beside the working catalog instead, and `fdl push` uploads the new files (see [Pushing over HTTP](/spec/http#writing-to-it))
|
|
37
38
|
- `DATA_INLINING_ROW_LIMIT 0` keeps every row out of the catalog. A version must not carry inlined rows, and `fdl commit` refuses a working catalog that holds some
|
|
38
39
|
- `META_TYPE 'sqlite'` has to be stated. Left to guess from the file, DuckDB passes an option to SQLite that SQLite rejects
|
|
39
40
|
|
|
40
|
-
## Data files written before a
|
|
41
|
+
## Data files written before a push
|
|
41
42
|
|
|
42
43
|
DuckLake writes Parquet as you go, so data files reach the repository before any version names them. Readers never see them: a reader only opens catalogs of versions, and no version names those files yet.
|
|
43
44
|
|
|
44
|
-
A file that stays unnamed for longer than the grace period (7 days by default) may be deleted by maintenance.
|
|
45
|
+
A file that stays unnamed for longer than the grace period (7 days by default) may be deleted by maintenance. Push within that time. If a push is refused because a file it names is missing, rebuild the table in the working catalog, commit and push again. A version committed and never pushed names its files only on your disk, so they are deleted in the same way.
|
|
45
46
|
|
|
46
47
|
## Snapshots in the working catalog
|
|
47
48
|
|
|
@@ -29,6 +29,8 @@ towns/
|
|
|
29
29
|
|
|
30
30
|
`HEAD` names the default branch, `main`. The branch itself does not exist yet: there is no version for it to point at.
|
|
31
31
|
|
|
32
|
+
`fdl init` makes the repository where readers will find it, as `git init --bare` does on a server. Your own place to write is the working catalog, made next.
|
|
33
|
+
|
|
32
34
|
## Check out a working catalog
|
|
33
35
|
|
|
34
36
|
You never write to the repository directly. You write to a working catalog, a DuckLake catalog in a SQLite file on your own disk, and turn it into a version when you are done.
|
|
@@ -74,13 +76,14 @@ Every version carries a `dataset.json` that says what it holds. Write one next t
|
|
|
74
76
|
|
|
75
77
|
Only `name` and `title` are required. `fdl commit` checks the file against the catalog and fills in each table's row count and each column's type.
|
|
76
78
|
|
|
77
|
-
## Commit
|
|
79
|
+
## Commit and push
|
|
78
80
|
|
|
79
81
|
```bash
|
|
80
82
|
fdl commit work/lake.sqlite -m "First towns" --author alice
|
|
83
|
+
fdl push work/lake.sqlite
|
|
81
84
|
```
|
|
82
85
|
|
|
83
|
-
`commit` freezes the working catalog into a version
|
|
86
|
+
`commit` freezes the working catalog into a version on your disk. `push` uploads it and moves `main` to it, and prints `refs/heads/main -> 019a3c4e-… (version 1)`. The repository now holds:
|
|
84
87
|
|
|
85
88
|
```
|
|
86
89
|
towns/
|
|
@@ -132,8 +135,9 @@ INSERT INTO lake.main.towns SELECT * FROM 'towns-2026.csv';
|
|
|
132
135
|
|
|
133
136
|
```bash
|
|
134
137
|
fdl commit work/lake.sqlite -m "Add 2026" --author alice
|
|
138
|
+
fdl push work/lake.sqlite
|
|
135
139
|
```
|
|
136
140
|
|
|
137
141
|
The new version lists the first as its parent. Both catalogs name the first Parquet file; only the second names the new one.
|
|
138
142
|
|
|
139
|
-
If someone else
|
|
143
|
+
If someone else pushed in between, the push is refused, because the new version would not be built on what `main` holds now. Check out again and redo the change. [Versions and refs](/concepts/versions-and-refs) explains why.
|
|
@@ -30,9 +30,30 @@ FDL is meant for data that is published, read by many and written by few:
|
|
|
30
30
|
- a dataset maintained by a team, where a change has to be reviewed before readers see it
|
|
31
31
|
- copies of the same dataset on several hosts (mirrors)
|
|
32
32
|
|
|
33
|
-
It is not a transactional database.
|
|
33
|
+
It is not a transactional database. Two people who push at once are settled by the store: the first push wins and the second is refused. The second builds again on the new version, or works on a branch of its own and merges it, rows matched by the table's primary key.
|
|
34
34
|
|
|
35
|
-
##
|
|
35
|
+
## FDL and Git
|
|
36
|
+
|
|
37
|
+
FDL keeps a catalog's versions the way Git keeps a project's commits. If you know Git, most of FDL maps onto it:
|
|
38
|
+
|
|
39
|
+
| Git | FDL |
|
|
40
|
+
| --- | --- |
|
|
41
|
+
| A bare repository on a server | A repository: a prefix of a bucket, or a folder |
|
|
42
|
+
| Commit | Version (`versions/<ID>/version.json`) |
|
|
43
|
+
| Tree | The frozen catalog (`ducklake.duckdb`) |
|
|
44
|
+
| Blob | Data file (Parquet), asset |
|
|
45
|
+
| Branch, tag, `HEAD` | `refs/heads/`, `refs/tags/`, `HEAD` |
|
|
46
|
+
| Working tree | Working catalog (SQLite on your disk) |
|
|
47
|
+
| `git init --bare` | `fdl init` |
|
|
48
|
+
| `git commit`, `git push` | `fdl commit`, `fdl push` |
|
|
49
|
+
| `git log`, `git tag` | `fdl log`, `fdl tag` |
|
|
50
|
+
| `git branch`, `git merge` | `fdl branch`, `fdl merge` |
|
|
51
|
+
|
|
52
|
+
A version has one parent, or two when it merges another branch. [From write to read](/internals/from-write-to-read) follows the files through a commit, step by step.
|
|
53
|
+
|
|
54
|
+
FDL is in beta. [Stability](/stability) says what stays the same across releases.
|
|
55
|
+
|
|
56
|
+
## How the documentation is organized
|
|
36
57
|
|
|
37
58
|
- [Getting started](/getting-started) makes a repository, writes a first version and reads it back
|
|
38
59
|
- [Concepts](/concepts/repositories) explains repositories, versions, refs and working catalogs
|
|
@@ -40,6 +61,7 @@ It is not a transactional database. Writers do not merge rows. Two people who pu
|
|
|
40
61
|
- [Guides](/guides/stores) cover stores, serving over HTTP, mirrors and building a host
|
|
41
62
|
- The [specification](/spec) is the normative description of the format. Tools are written against it
|
|
42
63
|
- The [reference](/reference/cli) lists the commands and the Python API
|
|
64
|
+
- [Stability](/stability) says what a repository written today can rely on
|
|
43
65
|
|
|
44
66
|
The `fdl` command and the `fdl` Python package are the reference implementation. They are published on PyPI as `fdl-cli` under the MIT license.
|
|
45
67
|
|
|
@@ -30,7 +30,7 @@ Three things differ on purpose:
|
|
|
30
30
|
|
|
31
31
|
- A version's ID is a UUIDv7, not a hash of its content. Two DuckDB releases can write the same catalog as different bytes, so a content hash would change with the tool. The sha256 in `version.json` checks the content instead
|
|
32
32
|
- There are no packfiles. Each file is its own object in the store, so a reader fetches only the parts of a Parquet file it needs
|
|
33
|
-
-
|
|
33
|
+
- A merge matches rows by the primary key that `dataset.json` declares, not by lines or files. Two branches that changed the same table without one are a conflict (see [Merging](/spec/writing#merging))
|
|
34
34
|
|
|
35
35
|
## Writing a version
|
|
36
36
|
|
|
@@ -40,7 +40,7 @@ No ref and no version names the file, so no reader can reach it. If the writer g
|
|
|
40
40
|
|
|
41
41
|
## The first commit
|
|
42
42
|
|
|
43
|
-
`fdl commit` does
|
|
43
|
+
`fdl commit` does steps 1 to 4 on the writer's disk, and `fdl push` does the rest, in this order:
|
|
44
44
|
|
|
45
45
|
1. Copy the working catalog from SQLite into a DuckDB file, keeping the snapshots up to the last one
|
|
46
46
|
2. Set the catalog's `data_path` to the repository's data folder, and refuse if any file is recorded with an absolute path elsewhere
|
|
@@ -62,7 +62,7 @@ versions/019a2f10-…/version.json
|
|
|
62
62
|
data/main/towns/ducklake-0199f1….parquet
|
|
63
63
|
```
|
|
64
64
|
|
|
65
|
-
The order is what keeps readers safe. A reader starts from a ref, and the ref is written last. If the
|
|
65
|
+
The order is what keeps readers safe. A reader starts from a ref, and the ref is written last. If the push stops after step 6, 7 or 8, the repository holds a version folder no ref points at. Readers never see it, and it changes nothing.
|
|
66
66
|
|
|
67
67
|
`version.json` is written after the catalog and `dataset.json` for the same reason: a tool that lists `versions/` can treat a folder without `version.json` as unfinished.
|
|
68
68
|
|
|
@@ -56,7 +56,7 @@ HEAD ref: refs/heads/main
|
|
|
56
56
|
|
|
57
57
|
## ブランチを動かす
|
|
58
58
|
|
|
59
|
-
ブランチは前へだけ動きます。新しいバージョンが `main`
|
|
59
|
+
ブランチは前へだけ動きます。新しいバージョンが `main` になれるのは、いま `main` が指すバージョンの子孫であるときです。子孫とは、`parents` をたどるとそのバージョンに着くバージョンです。Git でいう fast-forward です。
|
|
60
60
|
|
|
61
61
|
```
|
|
62
62
|
main → A
|
|
@@ -64,9 +64,20 @@ main → A
|
|
|
64
64
|
Bob も A をチェックアウトしていて、C(親 A)をコミットして push: 断られる
|
|
65
65
|
```
|
|
66
66
|
|
|
67
|
-
C は B の上に積まれていないので、Bob の push は断られます。受け付けると、誰も決めていないのに B が `main` の履歴から消えてしまいます。Bob は B
|
|
67
|
+
C は B の上に積まれていないので、Bob の push は断られます。受け付けると、誰も決めていないのに B が `main` の履歴から消えてしまいます。Bob は B をチェックアウトし、変更をやり直します。新しいバージョンの上でのやり直しは、たいてい同じ SQL を実行し直すだけで済みます。
|
|
68
68
|
|
|
69
|
-
|
|
69
|
+
## マージ
|
|
70
|
+
|
|
71
|
+
時間のかかる作業は自分のブランチで進め、できあがったら `main` にマージします。
|
|
72
|
+
|
|
73
|
+
```
|
|
74
|
+
main → A → B
|
|
75
|
+
\
|
|
76
|
+
feature C
|
|
77
|
+
main で fdl merge refs/heads/feature を実行すると M(親 B、C)ができる: main → M
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
マージは、テーブルごとに B、C、両者の共通の祖先 A の 3 つのバージョンを比較します。片方のブランチだけが変えたテーブルは、そのブランチの内容を取り込みます。両方が同じテーブルを変えたときは、両方が `dataset.json` で宣言した `primary_key` で行を照合し、片方だけが変えた行を取り込みます。両方が違う形に変えた行と、主キーの無いテーブルを両方が変えたときは衝突になり、バージョンは作りません。マージのために付けた行番号は主キーになりません。どの行が同じ行かを示さないためです。[マージ](/ja/spec/writing#マージ)を参照してください。
|
|
70
81
|
|
|
71
82
|
バージョンを取り消すためにブランチを後ろへ戻すのは、明示的に指示したときだけできます。取り消したバージョンも、リポジトリと履歴には残ります。
|
|
72
83
|
|
|
@@ -11,7 +11,8 @@ description: 書き込みをする場所です。SQLite の DuckLake カタロ
|
|
|
11
11
|
| --- | --- | --- |
|
|
12
12
|
| チェックアウト | `fdl checkout` | ブランチのバージョンのカタログを SQLite に写し、そのバージョンを覚える |
|
|
13
13
|
| 書く | DuckDB のクライアント | DuckLake が Parquet をリポジトリへ、管理用の行を SQLite へ書く |
|
|
14
|
-
| コミット | `fdl commit` |
|
|
14
|
+
| コミット | `fdl commit` | 最後のスナップショットを新しいバージョンに固め、作業カタログの隣に置く |
|
|
15
|
+
| push | `fdl push` | バージョンをアップロードし、ブランチをそこへ動かす |
|
|
15
16
|
|
|
16
17
|
チェックアウトでは、DuckLake の管理用の表を、固めた DuckDB ファイルから SQLite へそのまま写します。固めるときはその逆です。どちらも Parquet ファイルは書き換えないので、大きなデータセットでもすぐ終わります。カタログは小さく、データはその場に残ります。
|
|
17
18
|
|
|
@@ -19,7 +20,7 @@ description: 書き込みをする場所です。SQLite の DuckLake カタロ
|
|
|
19
20
|
|
|
20
21
|
作業カタログは、DuckDB から書けて、作業をまたいで残せる必要があります。SQLite はデータベースサーバーなしでその両方ができ、手元のパソコンでもコンテナでも同じように動き、DuckDB の形式との間で欠けなく写せます。
|
|
21
22
|
|
|
22
|
-
作業カタログは 1 種類です。利用者の代わりに書くホスト(定期の取り込みや Web
|
|
23
|
+
作業カタログは 1 種類です。利用者の代わりに書くホスト(定期の取り込みや Web のフォームなど)も、人がパソコンでするのと同じ手順でチェックアウトし、書き、コミットして push します。2 つが同時に push したときは、ref の条件付き書き込みで決まります。一方が通り、もう一方はチェックアウトし直します。
|
|
23
24
|
|
|
24
25
|
## ATTACH の仕方
|
|
25
26
|
|
|
@@ -33,15 +34,15 @@ ATTACH 'ducklake:work/lake.sqlite' AS lake (
|
|
|
33
34
|
);
|
|
34
35
|
```
|
|
35
36
|
|
|
36
|
-
- `DATA_PATH` はリポジトリのデータフォルダです。新しい Parquet ファイルは、読み手が探す場所に書かれます。HTTP 越しのリポジトリでは、かわりに作業カタログの隣のフォルダになり、新しいファイルは `fdl
|
|
37
|
+
- `DATA_PATH` はリポジトリのデータフォルダです。新しい Parquet ファイルは、読み手が探す場所に書かれます。HTTP 越しのリポジトリでは、かわりに作業カタログの隣のフォルダになり、新しいファイルは `fdl push` が上げます([HTTP での push](/ja/spec/http#書き方))
|
|
37
38
|
- `DATA_INLINING_ROW_LIMIT 0` で、行をカタログの中に入れないようにします。バージョンはカタログの中に行を持てないので、行を持つ作業カタログは `fdl commit` が断ります
|
|
38
39
|
- `META_TYPE 'sqlite'` は省略できません。省くと DuckDB がファイルから種類を推測し、SQLite が受け付けないオプションを渡してしまいます
|
|
39
40
|
|
|
40
|
-
##
|
|
41
|
+
## push 前に書かれたデータファイル
|
|
41
42
|
|
|
42
43
|
DuckLake は書くそばから Parquet を書くので、データファイルは、それを参照するバージョンより先にリポジトリに届きます。読み手には見えません。読み手はバージョンのカタログしか開かず、まだどのバージョンもそのファイルを参照していないからです。
|
|
43
44
|
|
|
44
|
-
どこからも参照されないまま猶予期間(既定で 7
|
|
45
|
+
どこからも参照されないまま猶予期間(既定で 7 日)を過ぎたファイルは、保守で消されることがあります。その前に push してください。参照しているファイルが無いと言われて push が断られたら、作業カタログで表を作り直し、コミットと push をやり直してください。コミットしたまま push しなかったバージョンは手元でしかファイルを参照していないので、そのファイルも同じように消されます。
|
|
45
46
|
|
|
46
47
|
## 作業カタログのスナップショット
|
|
47
48
|
|
|
@@ -29,6 +29,8 @@ towns/
|
|
|
29
29
|
|
|
30
30
|
`HEAD` は既定のブランチ `main` を指します。ブランチそのものはまだありません。指す先のバージョンがないためです。
|
|
31
31
|
|
|
32
|
+
`fdl init` は、サーバーで `git init --bare` を打つのと同じく、読み手が読む場所にリポジトリを作ります。手元で書く場所は、次に作る作業カタログです。
|
|
33
|
+
|
|
32
34
|
## 作業カタログをチェックアウトする
|
|
33
35
|
|
|
34
36
|
リポジトリには直接書き込みません。自分のディスクにある SQLite ファイルの DuckLake カタログ(作業カタログ)に書き、書き終えたらそれをバージョンにします。
|
|
@@ -74,13 +76,14 @@ SELECT * FROM 'towns.csv';
|
|
|
74
76
|
|
|
75
77
|
必須なのは `name` と `title` だけです。`fdl commit` がこのファイルをカタログと照らし合わせ、表ごとの行数と列ごとの型を書き足します。
|
|
76
78
|
|
|
77
|
-
##
|
|
79
|
+
## コミットして push する
|
|
78
80
|
|
|
79
81
|
```bash
|
|
80
82
|
fdl commit work/lake.sqlite -m "First towns" --author alice
|
|
83
|
+
fdl push work/lake.sqlite
|
|
81
84
|
```
|
|
82
85
|
|
|
83
|
-
`commit`
|
|
86
|
+
`commit` は作業カタログを固めて手元でバージョンにします。`push` はそれをアップロードして `main` をそこへ動かし、`refs/heads/main -> 019a3c4e-… (version 1)` と表示します。リポジトリは次のようになります。
|
|
84
87
|
|
|
85
88
|
```
|
|
86
89
|
towns/
|
|
@@ -132,8 +135,9 @@ INSERT INTO lake.main.towns SELECT * FROM 'towns-2026.csv';
|
|
|
132
135
|
|
|
133
136
|
```bash
|
|
134
137
|
fdl commit work/lake.sqlite -m "Add 2026" --author alice
|
|
138
|
+
fdl push work/lake.sqlite
|
|
135
139
|
```
|
|
136
140
|
|
|
137
141
|
新しいバージョンは、最初のバージョンを親として記録します。最初の Parquet ファイルは両方のカタログが参照し、新しいファイルは 2 つめのカタログだけが参照します。
|
|
138
142
|
|
|
139
|
-
|
|
143
|
+
途中でほかの人が push していたら、この push は断られます。新しいバージョンが、いまの `main` の上に積まれていないためです。チェックアウトし直して、変更をやり直してください。理由は[バージョンと ref](/ja/concepts/versions-and-refs) で説明しています。
|