fdl-cli 0.12.1__py3-none-any.whl

fdl/stores.py

@@ -0,0 +1,177 @@
+"""Two stores: a folder on this machine, and an S3-compatible bucket.
+
+Both raise `StoreError` with the HTTP status a failure would have had (404 for
+missing, 412 for a failed condition), which is all `fdl.store` looks at.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+from typing import Sequence
+from urllib.parse import urlparse
+
+
+class StoreError(RuntimeError):
+    def __init__(self, message: str, *, status: int | None = None) -> None:
+        super().__init__(message)
+        self.status = status
+
+
+class LocalStore:
+    """A repository in a folder. ETags are content hashes, which is enough for
+    the conditional writes refs rely on. One writer at a time: the condition
+    is checked and the file written in two steps."""
+
+    def __init__(self, root: Path | str) -> None:
+        self.root = Path(root)
+
+    @property
+    def url(self) -> str:
+        return str(self.root.resolve())
+
+    def _path(self, key: str) -> Path:
+        path = (self.root / key).resolve()
+        if not path.is_relative_to(self.root.resolve()):
+            raise StoreError(f"key escapes the store root: {key!r}")
+        return path
+
+    @staticmethod
+    def _etag(data: bytes) -> str:
+        return '"' + hashlib.md5(data, usedforsecurity=False).hexdigest() + '"'
+
+    def put(
+        self,
+        key: str,
+        data: bytes,
+        *,
+        content_type: str | None = None,
+        if_match: str | None = None,
+        if_none_match: str | None = None,
+    ) -> str:
+        path = self._path(key)
+        current = path.read_bytes() if path.is_file() else None
+        if if_none_match == "*" and current is not None:
+            raise StoreError(f"{key} already exists", status=412)
+        if if_match is not None and (current is None or self._etag(current) != if_match):
+            raise StoreError(f"{key} does not match {if_match}", status=412)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_bytes(data)
+        return self._etag(data)
+
+    def put_file(self, key: str, path: Path, **kwargs) -> str:
+        return self.put(key, Path(path).read_bytes(), **kwargs)
+
+    def get(self, key: str) -> bytes:
+        path = self._path(key)
+        if not path.is_file():
+            raise StoreError(f"no such object: {key}", status=404)
+        return path.read_bytes()
+
+    def head(self, key: str) -> dict[str, str] | None:
+        path = self._path(key)
+        return {"ETag": self._etag(path.read_bytes())} if path.is_file() else None
+
+    def list(self, prefix: str = "") -> list[str]:
+        if not self.root.is_dir():
+            return []
+        keys = (str(p.relative_to(self.root)) for p in sorted(self.root.rglob("*")) if p.is_file())
+        return [key for key in keys if key.startswith(prefix)]
+
+    def delete(self, keys: Sequence[str]) -> None:
+        for key in keys:
+            self._path(key).unlink(missing_ok=True)
+
+
+class S3Store:
+    """A repository under ``s3://<bucket>/<prefix>``.
+
+    Credentials and the endpoint come from boto3's usual places (the
+    ``AWS_*`` environment variables, ``AWS_ENDPOINT_URL`` for R2 and other
+    S3-compatible stores, or a profile).
+    """
+
+    def __init__(self, bucket: str, prefix: str = "", *, client=None) -> None:
+        if client is None:
+            try:
+                import boto3
+            except ImportError as exc:
+                raise StoreError("S3 needs boto3: pip install 'fdl-cli[s3]'") from exc
+
+            client = boto3.client("s3")
+        self.client = client
+        self.bucket = bucket
+        self.prefix = prefix.strip("/") + "/" if prefix.strip("/") else ""
+
+    @property
+    def url(self) -> str:
+        return f"s3://{self.bucket}/{self.prefix}"
+
+    def _call(self, method: str, key: str, **kwargs):
+        from botocore.exceptions import ClientError
+
+        try:
+            return getattr(self.client, method)(Bucket=self.bucket, **kwargs)
+        except ClientError as exc:
+            status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
+            raise StoreError(f"{method} {key}: {exc}", status=status) from exc
+
+    def put(
+        self,
+        key: str,
+        data: bytes,
+        *,
+        content_type: str | None = None,
+        if_match: str | None = None,
+        if_none_match: str | None = None,
+    ) -> str:
+        options = {
+            name: value
+            for name, value in (
+                ("ContentType", content_type),
+                ("IfMatch", if_match),
+                ("IfNoneMatch", if_none_match),
+            )
+            if value is not None
+        }
+        answer = self._call("put_object", key, Key=self.prefix + key, Body=data, **options)
+        return answer["ETag"]
+
+    def put_file(self, key: str, path: Path, **kwargs) -> str:
+        # Catalogs are a few MB, so one request is enough.
+        return self.put(key, Path(path).read_bytes(), **kwargs)
+
+    def get(self, key: str) -> bytes:
+        return self._call("get_object", key, Key=self.prefix + key)["Body"].read()
+
+    def head(self, key: str) -> dict[str, str] | None:
+        try:
+            answer = self._call("head_object", key, Key=self.prefix + key)
+        except StoreError as exc:
+            if exc.status == 404:
+                return None
+            raise
+        return {"ETag": answer["ETag"]}
+
+    def list(self, prefix: str = "") -> list[str]:
+        keys: list[str] = []
+        pages = self.client.get_paginator("list_objects_v2").paginate(
+            Bucket=self.bucket, Prefix=self.prefix + prefix
+        )
+        for page in pages:
+            keys.extend(item["Key"][len(self.prefix) :] for item in page.get("Contents", []))
+        return keys
+
+    def delete(self, keys: Sequence[str]) -> None:
+        for key in keys:
+            self._call("delete_object", key, Key=self.prefix + key)
+
+
+def open_store(url: str) -> LocalStore | S3Store:
+    """``s3://bucket/prefix/`` or a path on this machine."""
+    parsed = urlparse(url)
+    if parsed.scheme == "s3":
+        return S3Store(parsed.netloc, parsed.path)
+    if parsed.scheme in ("", "file"):
+        return LocalStore(parsed.path if parsed.scheme == "file" else url)
+    raise StoreError(f"not a store this knows: {url!r} (s3://bucket/prefix/ or a folder)")

fdl/version.py

@@ -0,0 +1,140 @@
+"""The record of a version (`version.json`) and how its ID is made."""
+
+from __future__ import annotations
+
+import datetime as dt
+import json
+import os
+import re
+import time
+from dataclasses import dataclass, field
+
+from . import layout
+
+_ID = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}")
+
+
+class VersionError(ValueError):
+    """A version record this format cannot read."""
+
+
+def new_id(now_ms: int | None = None) -> str:
+    """A UUIDv7: ordered by the time it was made, and safe to make without
+    asking anyone, so two writers never collide.
+
+    By hand because `uuid.uuid7` arrived in Python 3.14.
+    """
+    ms = int(time.time() * 1000) if now_ms is None else now_ms
+    value = (ms & ((1 << 48) - 1)) << 80
+    value |= 0x7 << 76
+    value |= (int.from_bytes(os.urandom(2), "big") & 0x0FFF) << 64
+    value |= 0b10 << 62
+    value |= int.from_bytes(os.urandom(8), "big") & ((1 << 62) - 1)
+    text = f"{value:032x}"
+    return f"{text[:8]}-{text[8:12]}-{text[12:16]}-{text[16:20]}-{text[20:]}"
+
+
+def is_id(value: str) -> bool:
+    return bool(_ID.fullmatch(value))
+
+
+@dataclass(frozen=True)
+class Catalog:
+    path: str
+    storage: str
+    bytes: int
+    sha256: str
+
+
+@dataclass(frozen=True)
+class DuckLake:
+    format: str
+    snapshot_id: int
+    #: The first and last snapshot this version holds past its parent's.
+    snapshots: tuple[int, int]
+
+
+@dataclass(frozen=True)
+class Version:
+    id: str
+    seq: int
+    parents: tuple[str, ...]
+    created_at: str
+    author: dict
+    message: str
+    catalog: Catalog
+    ducklake: DuckLake
+    changes: tuple[dict, ...] = ()
+    inputs: tuple[dict, ...] = ()
+    extensions: dict = field(default_factory=dict)
+    fdl: str = layout.FDL_VERSION
+
+    def to_json(self) -> bytes:
+        record = {
+            "fdl": self.fdl,
+            "id": self.id,
+            "seq": self.seq,
+            "parents": list(self.parents),
+            "created_at": self.created_at,
+            "author": self.author,
+            "message": self.message,
+            "catalog": {
+                "path": self.catalog.path,
+                "storage": self.catalog.storage,
+                "bytes": self.catalog.bytes,
+                "sha256": self.catalog.sha256,
+            },
+            "ducklake": {
+                "format": self.ducklake.format,
+                "snapshot_id": self.ducklake.snapshot_id,
+                "snapshots": list(self.ducklake.snapshots),
+            },
+            "changes": list(self.changes),
+            "inputs": list(self.inputs),
+            "extensions": self.extensions,
+        }
+        return (json.dumps(record, ensure_ascii=False, indent=2) + "\n").encode()
+
+
+def parse(data: bytes, *, expected_id: str | None = None) -> Version:
+    """Read a `version.json`, refusing what a reader of this format must not open."""
+    try:
+        record = json.loads(data)
+        major = str(record["fdl"]).split(".")[0]
+        if major != layout.FDL_VERSION.split(".")[0]:
+            raise VersionError(f"written for fdl {record['fdl']}, this reads {layout.FDL_VERSION}")
+        if expected_id is not None and record["id"] != expected_id:
+            raise VersionError(f"records id {record['id']} but sits under {expected_id}")
+        catalog = record["catalog"]
+        ducklake = record["ducklake"]
+        return Version(
+            fdl=record["fdl"],
+            id=record["id"],
+            seq=int(record["seq"]),
+            parents=tuple(record["parents"]),
+            created_at=record["created_at"],
+            author=dict(record["author"]),
+            message=record["message"],
+            catalog=Catalog(
+                path=catalog["path"],
+                storage=catalog["storage"],
+                bytes=int(catalog["bytes"]),
+                sha256=catalog["sha256"],
+            ),
+            ducklake=DuckLake(
+                format=ducklake["format"],
+                snapshot_id=int(ducklake["snapshot_id"]),
+                snapshots=tuple(ducklake["snapshots"]),
+            ),
+            changes=tuple(record.get("changes", ())),
+            inputs=tuple(record.get("inputs", ())),
+            extensions=dict(record.get("extensions", {})),
+        )
+    except VersionError:
+        raise
+    except (KeyError, TypeError, ValueError) as exc:
+        raise VersionError(f"not a version record: {exc}") from exc
+
+
+def utc_now() -> str:
+    return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")

fdl_cli-0.12.1.dist-info/METADATA

@@ -0,0 +1,116 @@
+Metadata-Version: 2.5
+Name: fdl-cli
+Version: 0.12.1
+Summary: Versions of a DuckLake catalog on object storage, kept the way Git keeps commits
+Project-URL: Homepage, https://fdl.flo8s.com/
+Project-URL: Documentation, https://fdl.flo8s.com/
+Project-URL: Repository, https://github.com/queria-io/fdl
+Project-URL: Changelog, https://github.com/queria-io/fdl/blob/main/CHANGELOG.md
+Author: flo8s
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: duckdb<2.0,>=1.5.2
+Requires-Dist: typer>=0.15
+Provides-Extra: s3
+Requires-Dist: boto3>=1.36; extra == 's3'
+Description-Content-Type: text/markdown
+
+# fdl
+
+[![PyPI](https://img.shields.io/pypi/v/fdl-cli)](https://pypi.org/project/fdl-cli/)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue)](https://pypi.org/project/fdl-cli/)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
+
+Versions of a [DuckLake](https://ducklake.select/) catalog on object storage, kept the way Git keeps commits.
+
+FDL is a storage layout and a tool for it. Each version is a frozen catalog with a record of its parent, author and message; refs such as `refs/heads/main` say which version readers get. Data files are shared between versions and never rewritten. All it needs is a store that can read, write and write on condition: a folder, S3, R2 or anything S3-compatible. There is no server.
+
+```
+<root>/
+  fdl.json
+  HEAD                              ref: refs/heads/main
+  refs/heads/main                   019a3c4e-…
+  versions/019a3c4e-…/
+    version.json                    parents, author, message, sha256
+    ducklake.duckdb                 the frozen catalog
+  data/main/towns/ducklake-….parquet
+```
+
+The format is in [docs/spec.md](docs/spec.md).
+
+## Quick start
+
+```bash
+pip install 'fdl-cli[s3]'
+
+fdl init s3://my-bucket/towns/
+fdl checkout s3://my-bucket/towns/ work/lake.sqlite
+```
+
+The package is `fdl-cli` on PyPI. It installs the `fdl` command and the `fdl` Python package (`import fdl`).
+
+Write to the working catalog with DuckDB, attached the way `checkout` prints:
+
+```sql
+ATTACH 'ducklake:work/lake.sqlite' AS lake (
+  DATA_PATH 's3://my-bucket/towns/data/', OVERRIDE_DATA_PATH true,
+  META_TYPE 'sqlite', DATA_INLINING_ROW_LIMIT 0, META_JOURNAL_MODE 'WAL', BUSY_TIMEOUT 5000
+);
+CREATE TABLE lake.main.towns AS SELECT * FROM 'towns.csv';
+```
+
+Rows have to go to Parquet, not into the catalog (`DATA_INLINING_ROW_LIMIT 0`): a version is read over HTTP, and inlined rows would not be in it.
+
+Then make a version and move `main` to it:
+
+```bash
+fdl commit work/lake.sqlite -m "First towns" --author alice
+fdl log s3://my-bucket/towns/
+```
+
+Anyone who can read the bucket can attach it:
+
+```bash
+fdl url s3://my-bucket/towns/
+```
+
+```sql
+ATTACH 'ducklake:s3://my-bucket/towns/versions/019a3c4e-…/ducklake.duckdb' AS towns (READ_ONLY);
+```
+
+Credentials and the endpoint come from boto3's usual places (`AWS_*` environment variables, `AWS_ENDPOINT_URL` for R2 and other S3-compatible stores). A folder works as a repository too: pass its path instead of an `s3://` URL.
+
+## Commands
+
+| Command | What it does |
+| --- | --- |
+| `fdl init <remote>` | Start a repository (`fdl.json` and `HEAD`) |
+| `fdl checkout <remote> <working>` | Make a working catalog from the version `main` holds |
+| `fdl commit <working> -m … --author …` | Freeze the working catalog into a version and push it |
+| `fdl log <remote>` | List versions from `main` back to the first |
+| `fdl url <remote>` | Print the catalog URL of the version `main` holds |
+| `fdl expire <remote> --keep N` | Mark older versions as past retention and delete their catalogs |
+
+A push that is not built on what `main` holds now is refused: check out again and redo the work.
+
+## Python
+
+```python
+from fdl import repo
+from fdl.stores import open_store
+
+store = open_store("s3://my-bucket/towns/")
+for version in repo.log(store):
+    print(version.seq, version.id, version.message)
+```
+
+`fdl.repo` has the operations without a terminal (`init`, `checkout`, `commit`, `push`, `log`, `expire`, `prune`), for tools and services that build on FDL.
+
+## Earlier releases
+
+Releases up to 0.11 were a different tool for managing a single frozen catalog, published as `frozen-ducklake`. Their code is at the [`legacy`](https://github.com/queria-io/fdl/tree/legacy) branch.
+
+## License
+
+MIT

fdl_cli-0.12.1.dist-info/RECORD

@@ -0,0 +1,15 @@
+fdl/__init__.py,sha256=NwwwbIa96aDPoy1fgl_hcY1zurqKD0O6wjoRBsZHN2s,872
+fdl/catalog.py,sha256=VORkt_HpyKVt2wQwkdt3XF-vuPdhEyduEMd0ryL8uCw,17782
+fdl/cli.py,sha256=Ac0Q4tNA318G_b7qQmOaY4OYLSTHRFk45cLKd_Mo4Os,5456
+fdl/layout.py,sha256=rFmZwp7EgNTMXI2AErYNlkmOSbPMw5VFnhO48V52NBA,1806
+fdl/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fdl/refs.py,sha256=MslKbcWO9dRrRSjNYvp0ZAhh7vfjVacEuN0BIEPaGho,4219
+fdl/repo.py,sha256=NC8DNtUkmmsVk_PzwYmNAsJ8yfCyvLM2WEj8O3RTiHA,15675
+fdl/store.py,sha256=ADqdZU4Nr9jk7yVyBXoPis2s72OAedXekX0wT5Imr5s,1929
+fdl/stores.py,sha256=UXV4OX8mv78A_pTJtnlbeVM365t7Aya5zHsPhZ2oQ-8,6258
+fdl/version.py,sha256=K-CKkSs6zeMYDBHEcQeQZJPrl59toZnpnHjjPiOqIzQ,4470
+fdl_cli-0.12.1.dist-info/METADATA,sha256=kuw0xnYmLPtfaP0eSW0y-L13OX1q26AIPfOZY3xtmhM,4356
+fdl_cli-0.12.1.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+fdl_cli-0.12.1.dist-info/entry_points.txt,sha256=GOQe2LF3vdQ0W8wCI9i8Ldyikn26dfUDiJa5JUaZ7j8,37
+fdl_cli-0.12.1.dist-info/licenses/LICENSE,sha256=t0Of_GryPUMuBvGrtopnujMcxjgVBWMbUIyL7cCZMvk,1062
+fdl_cli-0.12.1.dist-info/RECORD,,

fdl_cli-0.12.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

fdl_cli-0.12.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+fdl = fdl.cli:main

fdl_cli-0.12.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 flo8s
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.