fdl-cli 0.24.1__tar.gz → 0.24.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/CHANGELOG.md +47 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/PKG-INFO +1 -1
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/spec/refs.md +2 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/spec/refs.md +2 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/pyproject.toml +1 -1
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/src/fdl/repo.py +30 -2
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/src/fdl/stores.py +9 -1
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/tests/test_http.py +17 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/tests/test_profiles.py +41 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/tests/test_repo.py +41 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/uv.lock +1 -1
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/.github/workflows/docs.yml +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/.github/workflows/publish.yml +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/.github/workflows/release.yml +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/.github/workflows/test.yml +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/.gitignore +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/.node-version +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/.python-version +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/CLAUDE.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/CONTRIBUTING.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/LICENSE +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/README.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/SECURITY.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/blume.config.ts +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/cases/collection.json +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/concepts/meta.ts +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/concepts/repositories.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/concepts/versions-and-refs.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/concepts/working-catalogs.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/getting-started.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/guides/building-a-host.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/guides/meta.ts +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/guides/mirrors.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/guides/public-and-private.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/guides/reading-without-duckdb.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/guides/serving-over-http.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/guides/stores.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/index.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/internals/from-write-to-read.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/internals/inside-a-repository.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/internals/meta.ts +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/concepts/meta.ts +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/concepts/repositories.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/concepts/versions-and-refs.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/concepts/working-catalogs.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/getting-started.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/guides/building-a-host.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/guides/meta.ts +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/guides/mirrors.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/guides/public-and-private.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/guides/reading-without-duckdb.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/guides/serving-over-http.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/guides/stores.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/index.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/internals/ducklake-formats.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/internals/from-write-to-read.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/internals/inside-a-repository.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/internals/meta.ts +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/reference/cli.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/reference/meta.ts +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/reference/python.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/spec/collection-cases.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/spec/data-files.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/spec/http.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/spec/index.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/spec/layout.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/spec/meta.ts +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/spec/reading.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/spec/versions.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/spec/writing.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/ja/stability.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/reference/cli.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/reference/meta.ts +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/reference/python.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/spec/collection-cases.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/spec/data-files.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/spec/http.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/spec/index.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/spec/layout.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/spec/meta.ts +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/spec/reading.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/spec/versions.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/spec/writing.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/docs/stability.md +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/package-lock.json +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/package.json +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/public/CNAME +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/src/fdl/__init__.py +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/src/fdl/catalog.py +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/src/fdl/cli.py +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/src/fdl/dataset.py +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/src/fdl/diff.py +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/src/fdl/layout.py +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/src/fdl/lineage.py +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/src/fdl/merge.py +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/src/fdl/objects.py +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/src/fdl/profiles.py +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/src/fdl/py.typed +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/src/fdl/refs.py +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/src/fdl/store.py +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/src/fdl/version.py +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/tests/http_repo.py +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/tests/test_cli.py +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/tests/test_collection_cases.py +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/tests/test_dataset.py +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/tests/test_merge.py +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/tests/test_objects.py +0 -0
- {fdl_cli-0.24.1 → fdl_cli-0.24.3}/tests/test_stores.py +0 -0
|
@@ -1,6 +1,53 @@
|
|
|
1
1
|
# CHANGELOG
|
|
2
2
|
|
|
3
3
|
|
|
4
|
+
## v0.24.3 (2026-11-13)
|
|
5
|
+
|
|
6
|
+
### Bug Fixes
|
|
7
|
+
|
|
8
|
+
- Write the ref again when a push finds it already at the version
|
|
9
|
+
([#56](https://github.com/queria-io/fdl/pull/56),
|
|
10
|
+
[`a11abaf`](https://github.com/queria-io/fdl/commit/a11abaf3643d914167ee4ef41421daa68b769955))
|
|
11
|
+
|
|
12
|
+
* fix: write the ref again when a push finds it already at the version
|
|
13
|
+
|
|
14
|
+
A host that does its own work when a ref moves may stop after moving it. Pushing again returned
|
|
15
|
+
early because the ref already held the version, so the host never finished. The writer now writes
|
|
16
|
+
the ref again, to the same version and on condition of the ETag it read.
|
|
17
|
+
|
|
18
|
+
* fix: write the ref again after a retried write and in forward
|
|
19
|
+
|
|
20
|
+
A ref write whose first attempt moved the ref, with its answer lost, may leave the host stopped
|
|
21
|
+
after the move. forward returned early on the same version as push did. Both now write the ref
|
|
22
|
+
again, and the test checks the write is conditional on the ETag read.
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
## v0.24.2 (2026-11-13)
|
|
26
|
+
|
|
27
|
+
### Bug Fixes
|
|
28
|
+
|
|
29
|
+
- Give requests over HTTP a timeout so a silent host fails instead of hanging
|
|
30
|
+
([#54](https://github.com/queria-io/fdl/pull/54),
|
|
31
|
+
[`a7070a5`](https://github.com/queria-io/fdl/commit/a7070a5c06de0a6cd1e8862d9afa766fb822759f))
|
|
32
|
+
|
|
33
|
+
- Reuse a parent's profile when only files.json's columns key differs
|
|
34
|
+
([#55](https://github.com/queria-io/fdl/pull/55),
|
|
35
|
+
[`30223cb`](https://github.com/queria-io/fdl/commit/30223cb158d86db0a5aad216dd06d0f9483c9637))
|
|
36
|
+
|
|
37
|
+
* fix: reuse a parent's profile when only files.json's columns key differs
|
|
38
|
+
|
|
39
|
+
A parent written before files.json listed each table's columns never matched, so every commit on top
|
|
40
|
+
of it profiled every table from all of its data. Compare only the data files; profiles.reusable
|
|
41
|
+
already checks the columns.
|
|
42
|
+
|
|
43
|
+
* fix: still compare the columns when the parent's files.json lists them
|
|
44
|
+
|
|
45
|
+
Comparing only the data files kept a stale profile for a column dropped and added again under the
|
|
46
|
+
same name and type: the data files and the names and types match, but the column is new and has no
|
|
47
|
+
values. Leave the columns out of the comparison only for a parent written before files.json listed
|
|
48
|
+
them.
|
|
49
|
+
|
|
50
|
+
|
|
4
51
|
## v0.24.1 (2026-11-13)
|
|
5
52
|
|
|
6
53
|
### Bug Fixes
|
|
@@ -57,3 +57,5 @@ HEAD ref: refs/heads/main\n
|
|
|
57
57
|
## 送り直した書き込み
|
|
58
58
|
|
|
59
59
|
最初の返事を受け取れずに ref の書き込みを送り直した書き手は、自分の最初の書き込みのせいで条件が合わず断られることがあります。そのとき ref が、動かそうとしていたバージョンを指していれば、動かすのは成功しています。
|
|
60
|
+
|
|
61
|
+
動かそうとしているバージョンを ref が既に指していたら(書き込む前でも、送り直した書き込みが断られた後でも)、書き手は ref を同じバージョンで書き直します。条件には読んだときの ETag を付けます。ref を動かすときに自分の処理(新しいバージョンの記録など)をするホストは、ref を動かした後で止まっていることがあり、この書き込みで処理を終えられます。ホストは、ref を動かさない書き込みも受け付けます。読んだ後に ref が動いていたら、書き手はそのままにします。
|
|
@@ -57,3 +57,5 @@ A reader over plain HTTP cannot list the store, so it cannot find branches and t
|
|
|
57
57
|
## Retried writes
|
|
58
58
|
|
|
59
59
|
A writer that sends a ref write again after losing the answer to the first one may find the condition refused by its own first attempt. If the ref now holds the version it was moving to, the move succeeded.
|
|
60
|
+
|
|
61
|
+
A writer that finds the ref already holding the version it is moving the ref to, before its write or after a retried one, writes the ref again, to the same version and on condition of the ETag it read. A host that does its own work when a ref moves (recording the new version, say) may have stopped after moving the ref, and this write lets it finish. A host accepts a write that leaves a ref where it is. If the ref has moved since it was read, the writer leaves it.
|
|
@@ -462,7 +462,10 @@ def push(
|
|
|
462
462
|
current_ref = refs.read(store, layout.check_ref(ref))
|
|
463
463
|
if current_ref is not None and current_ref.target == version.id:
|
|
464
464
|
# This push again, after a run whose answer was lost. That run may
|
|
465
|
-
# have stopped before rewriting refs.json
|
|
465
|
+
# have stopped before rewriting refs.json, and a host that does its
|
|
466
|
+
# own work when a ref moves may have stopped after moving it.
|
|
467
|
+
if move_ref:
|
|
468
|
+
_write_ref_again(store, current_ref)
|
|
466
469
|
update_refs_index(store)
|
|
467
470
|
return
|
|
468
471
|
_refuse_unless_fast_forward(store, current_ref, version)
|
|
@@ -489,9 +492,26 @@ def push(
|
|
|
489
492
|
now = refs.read(store, ref)
|
|
490
493
|
if now is None or now.target != version.id:
|
|
491
494
|
raise NotFastForward(f"{ref} moved during the push. Pull and build again.") from exc
|
|
495
|
+
# The first attempt moved it, but its answer was lost: the host may
|
|
496
|
+
# have stopped after the move.
|
|
497
|
+
_write_ref_again(store, now)
|
|
492
498
|
update_refs_index(store)
|
|
493
499
|
|
|
494
500
|
|
|
501
|
+
def _write_ref_again(store: Store, ref: refs.Ref) -> None:
|
|
502
|
+
"""Write ``ref`` to the version it holds, so a host finishes the move.
|
|
503
|
+
|
|
504
|
+
A ref that moved meanwhile was moved by a later push, which did the
|
|
505
|
+
host's work for its own version: nothing is left to finish.
|
|
506
|
+
"""
|
|
507
|
+
if not ref.etag:
|
|
508
|
+
return
|
|
509
|
+
try:
|
|
510
|
+
refs.write(store, ref.name, ref.target, expected=ref)
|
|
511
|
+
except refs.RefConflict:
|
|
512
|
+
pass
|
|
513
|
+
|
|
514
|
+
|
|
495
515
|
def tag(store: Store, name: str, version_id: str) -> None:
|
|
496
516
|
"""Create ``refs/tags/<name>`` at a version. Tags are created, never moved."""
|
|
497
517
|
ref = layout.check_ref(f"refs/tags/{name}")
|
|
@@ -568,7 +588,12 @@ def write_profiles(
|
|
|
568
588
|
key = f"{table['schema']}.{table['name']}"
|
|
569
589
|
relative = profiles.path(table["schema"], table["name"])
|
|
570
590
|
previous = None
|
|
571
|
-
|
|
591
|
+
before, now = files_before.get(key), files_now.get(key, {})
|
|
592
|
+
if before is not None and "columns" not in before:
|
|
593
|
+
# A parent written before files.json listed the columns: compare the
|
|
594
|
+
# rest, and leave the names and types to profiles.reusable.
|
|
595
|
+
now = {name: value for name, value in now.items() if name != "columns"}
|
|
596
|
+
if parent is not None and before is not None and before == now:
|
|
572
597
|
try:
|
|
573
598
|
previous = store.get(layout.version_dir(parent.id) + relative)
|
|
574
599
|
except Exception as exc: # noqa: BLE001
|
|
@@ -1055,6 +1080,8 @@ def forward(store: Store, ref: str, version: Version) -> None:
|
|
|
1055
1080
|
"""
|
|
1056
1081
|
current = refs.read(store, layout.check_ref(ref))
|
|
1057
1082
|
if current is not None and current.target == version.id:
|
|
1083
|
+
# As in `push`: a host may have stopped after a lost move.
|
|
1084
|
+
_write_ref_again(store, current)
|
|
1058
1085
|
update_refs_index(store)
|
|
1059
1086
|
return
|
|
1060
1087
|
_refuse_unless_fast_forward(store, current, version)
|
|
@@ -1065,6 +1092,7 @@ def forward(store: Store, ref: str, version: Version) -> None:
|
|
|
1065
1092
|
now = refs.read(store, ref)
|
|
1066
1093
|
if now is None or now.target != version.id:
|
|
1067
1094
|
raise NotFastForward(f"{ref} moved meanwhile. Pull and merge again.") from exc
|
|
1095
|
+
_write_ref_again(store, now)
|
|
1068
1096
|
update_refs_index(store)
|
|
1069
1097
|
|
|
1070
1098
|
|
|
@@ -69,8 +69,14 @@ class _SameOriginRedirects(urllib.request.HTTPRedirectHandler):
|
|
|
69
69
|
_opener = urllib.request.build_opener(_SameOriginRedirects)
|
|
70
70
|
|
|
71
71
|
|
|
72
|
+
#: Seconds a request may wait to connect, or between two reads of the answer,
|
|
73
|
+
#: before it fails. Without it a connection a host stops answering holds the
|
|
74
|
+
#: command forever instead of failing so it can be run again.
|
|
75
|
+
TIMEOUT = 120
|
|
76
|
+
|
|
77
|
+
|
|
72
78
|
def _open(request: urllib.request.Request):
|
|
73
|
-
return _opener.open(request)
|
|
79
|
+
return _opener.open(request, timeout=TIMEOUT)
|
|
74
80
|
|
|
75
81
|
|
|
76
82
|
class StoreError(RuntimeError):
|
|
@@ -318,6 +324,8 @@ class HttpStore:
|
|
|
318
324
|
) from exc
|
|
319
325
|
except urllib.error.URLError as exc:
|
|
320
326
|
raise StoreError(f"{method} {key or '/'}: {exc.reason}") from exc
|
|
327
|
+
except TimeoutError as exc:
|
|
328
|
+
raise StoreError(f"{method} {key or '/'}: no answer in {TIMEOUT} s") from exc
|
|
321
329
|
|
|
322
330
|
@staticmethod
|
|
323
331
|
def _conditions(
|
|
@@ -411,3 +411,20 @@ def test_a_merge_over_http(served, tmp_path: Path) -> None:
|
|
|
411
411
|
)
|
|
412
412
|
assert con.execute("SELECT count(*) FROM v.main.towns").fetchone() == (4,)
|
|
413
413
|
assert con.execute("SELECT count(*) FROM v.main.roads").fetchone() == (1,)
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def test_a_host_that_stops_answering_fails_instead_of_waiting(monkeypatch) -> None:
|
|
417
|
+
import socket
|
|
418
|
+
import threading
|
|
419
|
+
|
|
420
|
+
listener = socket.socket()
|
|
421
|
+
listener.bind(("127.0.0.1", 0))
|
|
422
|
+
listener.listen()
|
|
423
|
+
accepted = []
|
|
424
|
+
threading.Thread(target=lambda: accepted.append(listener.accept()), daemon=True).start()
|
|
425
|
+
monkeypatch.setattr(stores, "TIMEOUT", 0.5)
|
|
426
|
+
try:
|
|
427
|
+
with pytest.raises(StoreError, match="no answer"):
|
|
428
|
+
HttpStore(f"http://127.0.0.1:{listener.getsockname()[1]}/").get("fdl.json")
|
|
429
|
+
finally:
|
|
430
|
+
listener.close()
|
|
@@ -129,6 +129,47 @@ def test_an_unchanged_table_keeps_its_parents_profile(repository, monkeypatch) -
|
|
|
129
129
|
assert profile_of(store, "other")["rows"] == 2
|
|
130
130
|
|
|
131
131
|
|
|
132
|
+
def test_a_parent_without_columns_in_its_files_keeps_its_profile(repository, monkeypatch) -> None:
|
|
133
|
+
store, working = repository
|
|
134
|
+
write(working, store, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
|
|
135
|
+
write(working, store, "CREATE TABLE lake.main.other AS SELECT 1 AS id")
|
|
136
|
+
commit(working, "-m", "m", "--author", "t")
|
|
137
|
+
parent = repo.head(store)
|
|
138
|
+
key = layout.version_dir(parent.id) + parent.files.path
|
|
139
|
+
files = json.loads(store.get(key))
|
|
140
|
+
for table in files["tables"].values():
|
|
141
|
+
table.pop("columns", None)
|
|
142
|
+
store.put(key, json.dumps(files).encode())
|
|
143
|
+
write(working, store, "INSERT INTO lake.main.other VALUES (2)")
|
|
144
|
+
|
|
145
|
+
built: list[str] = []
|
|
146
|
+
original = profiles.build
|
|
147
|
+
|
|
148
|
+
def spy(working, wanted, **kwargs):
|
|
149
|
+
wanted = list(wanted)
|
|
150
|
+
built.extend(t["name"] for t in wanted)
|
|
151
|
+
return original(working, wanted, **kwargs)
|
|
152
|
+
|
|
153
|
+
monkeypatch.setattr(profiles, "build", spy)
|
|
154
|
+
commit(working, "-m", "again", "--author", "t")
|
|
155
|
+
assert built == ["other"]
|
|
156
|
+
assert profile_of(store, "towns")["rows"] == 1
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def test_a_column_dropped_and_added_again_is_profiled_again(repository) -> None:
|
|
160
|
+
store, working = repository
|
|
161
|
+
write(working, store, "CREATE TABLE lake.main.towns AS SELECT i AS id, i AS c FROM range(5) t(i)")
|
|
162
|
+
write(working, store, "CREATE TABLE lake.main.other AS SELECT 1 AS id")
|
|
163
|
+
commit(working, "-m", "m", "--author", "t")
|
|
164
|
+
# The same name and type, and the same data files, but a new column with no values.
|
|
165
|
+
write(working, store, "ALTER TABLE lake.main.towns DROP COLUMN c")
|
|
166
|
+
write(working, store, "ALTER TABLE lake.main.towns ADD COLUMN c BIGINT")
|
|
167
|
+
write(working, store, "INSERT INTO lake.main.other VALUES (2)")
|
|
168
|
+
commit(working, "-m", "again", "--author", "t")
|
|
169
|
+
c = {column["name"]: column for column in profile_of(store)["columns"]}["c"]
|
|
170
|
+
assert c.get("max") is None
|
|
171
|
+
|
|
172
|
+
|
|
132
173
|
def test_expiry_deletes_the_profiles(repository) -> None:
|
|
133
174
|
store, working = repository
|
|
134
175
|
write(working, store, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
|
|
@@ -17,6 +17,8 @@ from fdl import catalog, diff, layout, refs, repo
|
|
|
17
17
|
from fdl.stores import LocalStore
|
|
18
18
|
from fdl.version import is_id, new_id, parse
|
|
19
19
|
|
|
20
|
+
REAL_READ = refs.read
|
|
21
|
+
|
|
20
22
|
AUTHOR = {"name": "tester"}
|
|
21
23
|
|
|
22
24
|
|
|
@@ -138,6 +140,45 @@ def test_pushing_the_same_version_again_is_accepted(tmp_path: Path, remote) -> N
|
|
|
138
140
|
assert repo.head(remote) == first
|
|
139
141
|
|
|
140
142
|
|
|
143
|
+
def test_a_push_sent_again_writes_the_ref_again(tmp_path: Path, remote, monkeypatch) -> None:
|
|
144
|
+
"""A host that does its work when the ref moves may have stopped after
|
|
145
|
+
moving it. The second push gives it the write again."""
|
|
146
|
+
build(tmp_path / "work.sqlite", remote, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
|
|
147
|
+
version = commit(tmp_path, remote)
|
|
148
|
+
repo.push(remote, tmp_path / "local" / version.id, version)
|
|
149
|
+
seen = refs.read(remote, "refs/heads/main")
|
|
150
|
+
written: list[tuple[str, str | None]] = []
|
|
151
|
+
put = remote.put
|
|
152
|
+
monkeypatch.setattr(
|
|
153
|
+
remote,
|
|
154
|
+
"put",
|
|
155
|
+
lambda key, *a, **kw: written.append((key, kw.get("if_match"))) or put(key, *a, **kw),
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
repo.push(remote, tmp_path / "local" / version.id, version)
|
|
159
|
+
|
|
160
|
+
# Written on condition of the ETag read, so a ref moved since is left.
|
|
161
|
+
assert ("refs/heads/main", seen.etag) in written
|
|
162
|
+
assert repo.head(remote) == version
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def test_a_push_sent_again_leaves_a_ref_moved_since(tmp_path: Path, remote, monkeypatch) -> None:
|
|
166
|
+
build(tmp_path / "work.sqlite", remote, "CREATE TABLE lake.main.towns AS SELECT 1 AS id")
|
|
167
|
+
version = commit(tmp_path, remote)
|
|
168
|
+
repo.push(remote, tmp_path / "local" / version.id, version)
|
|
169
|
+
later = new_id()
|
|
170
|
+
seen = refs.read(remote, "refs/heads/main")
|
|
171
|
+
# Another push moves main between this one's read and its write.
|
|
172
|
+
monkeypatch.setattr(
|
|
173
|
+
refs, "read", lambda store, name: seen if name == "refs/heads/main" else REAL_READ(store, name)
|
|
174
|
+
)
|
|
175
|
+
refs.write(remote, "refs/heads/main", later, expected=seen)
|
|
176
|
+
|
|
177
|
+
repo.push(remote, tmp_path / "local" / version.id, version)
|
|
178
|
+
|
|
179
|
+
assert REAL_READ(remote, "refs/heads/main").target == later
|
|
180
|
+
|
|
181
|
+
|
|
141
182
|
def test_push_without_moving_the_ref_leaves_readers_where_they_were(
|
|
142
183
|
tmp_path: Path, remote
|
|
143
184
|
) -> None:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|