roar-cli 0.3.3__tar.gz → 0.3.4__tar.gz

{roar_cli-0.3.3 → roar_cli-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: roar-cli
-Version: 0.3.3
+Version: 0.3.4
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research

{roar_cli-0.3.3 → roar_cli-0.3.4}/pyproject.toml

@@ -1,10 +1,11 @@
+
 [build-system]
 requires = ["maturin>=1.11.5,<2.0"]
 build-backend = "maturin"
 
 [project]
 name = "roar-cli"
-version = "0.3.3"
+version = "0.3.4"
 description = "Reproducibility and provenance tracker for ML training pipelines"
 authors = [
     { name="TReqs Team", email="info@treqs.ai" }

roar_cli-0.3.4/roar/application/composite/__init__.py

@@ -0,0 +1,37 @@
+"""Composite artifacts v2 — qualifying hash + structural detection.
+
+The host-agnostic ``sha256-tree`` qualifying hash (the dataset GLaaS key) and the
+declared/structural detectors that decide what a composite is and which files carry
+its identity. Composites form at the boundaries (``get``/``put``/``register``), not
+inside ``roar run``.
+
+This package is additive; it does not yet replace the legacy heuristic composite
+path in ``roar.application.publish``.
+"""
+
+from .canonical import normalize_relpath, preimage
+from .detect import Detection, detect, detect_nested
+from .qualifying import (
+    Composite,
+    Leaf,
+    blake3_tree,
+    build,
+    build_nested,
+    sha256_tree,
+    tree,
+)
+
+__all__ = [
+    "Composite",
+    "Detection",
+    "Leaf",
+    "blake3_tree",
+    "build",
+    "build_nested",
+    "detect",
+    "detect_nested",
+    "normalize_relpath",
+    "preimage",
+    "sha256_tree",
+    "tree",
+]

roar_cli-0.3.4/roar/application/composite/canonical.py

@@ -0,0 +1,71 @@
+"""Canonical preimage for the qualifying composite hash (``sha256-tree``).
+
+The qualifying hash is roar's host-agnostic, content-addressable dataset key: a
+``sha256`` over a canonical, injective serialization of a composite's leaves. It is
+path-sensitive (moving a file changes the key), content-only (no host metadata
+enters), and locally computable, so the key registered by ``roar put`` equals the
+key produced by ``roar get`` for the same bytes and layout.
+
+This is distinct from roar's internal ``blake3`` file hashing: blake3 stays the
+fast hash for primitive file lineage; ``sha256-tree`` is the *dataset* key.
+
+Frozen decisions (any change is a version bump of the domain tag):
+  - leaf = (normalized_posix_relpath, "<component-algo>:<hexdigest>")
+  - component-algo is always ``sha256`` for a qualifying leaf; nested composites
+    use ``sha256-tree``
+  - leaves are sorted by the UTF-8 bytes of their normalized relpath
+  - size and mode do not enter the preimage
+  - NUL separates path from digest; newline terminates each leaf line
+  - a domain-separation prefix prevents cross-protocol collisions
+"""
+
+from __future__ import annotations
+
+DOMAIN_V1 = b"roar:sha256-tree:v1\n"
+QUALIFYING_ALGO = "sha256"
+NESTED_ALGO = "sha256-tree"
+
+
+def normalize_relpath(path: str) -> str:
+    """Normalize a relative path to canonical POSIX form.
+
+    Backslashes become forward slashes, a leading ``./`` and leading slashes are
+    stripped. ``..`` components are rejected (a leaf must stay within its composite
+    root) and an empty result is an error.
+    """
+    p = path.replace("\\", "/")
+    while p.startswith("./"):
+        p = p[2:]
+    p = p.lstrip("/")
+    if not p:
+        raise ValueError("empty relative path is not a valid leaf key")
+    if any(segment == ".." for segment in p.split("/")):
+        raise ValueError(f"path escapes composite root: {path!r}")
+    return p
+
+
+def leaf_line(relpath: str, algo: str, hexdigest: str) -> bytes:
+    """Serialize one leaf to its canonical line (relpath already normalized)."""
+    if not hexdigest:
+        raise ValueError(f"leaf {relpath!r} has empty digest")
+    return relpath.encode("utf-8") + b"\x00" + f"{algo}:{hexdigest.lower()}".encode() + b"\n"
+
+
+def preimage(leaves: list[tuple[str, str, str]], *, domain: bytes = DOMAIN_V1) -> bytes:
+    """Build the canonical preimage from ``(relpath, algo, hexdigest)`` triples.
+
+    Relpaths are normalized and the set is sorted by normalized relpath bytes.
+    Duplicate normalized relpaths are rejected.
+    """
+    normalized: dict[str, tuple[str, str]] = {}
+    for relpath, algo, hexdigest in leaves:
+        key = normalize_relpath(relpath)
+        if key in normalized:
+            raise ValueError(f"duplicate leaf path in composite: {key!r}")
+        normalized[key] = (algo, hexdigest)
+
+    out = bytearray(domain)
+    for key in sorted(normalized, key=lambda k: k.encode("utf-8")):
+        algo, hexdigest = normalized[key]
+        out += leaf_line(key, algo, hexdigest)
+    return bytes(out)

roar_cli-0.3.4/roar/application/composite/detect.py

@@ -0,0 +1,120 @@
+"""Structural dataset detection — declared/structural, never scored.
+
+Each detector is a pure function of a file listing (relpaths) and returns a
+:class:`Detection` naming the structural ``kind``, which files are identity-bearing
+(``data``/``meta``) versus excluded ``boilerplate``, and any nested sub-datasets.
+
+This replaces the role of the confidence scorer: a format's on-disk structure
+either declares it or it does not. There are no thresholds or weights. When nothing
+matches, ``kind`` is ``"unstructured"`` and the caller decides (an explicit
+``roar put --dataset`` may still declare it; an unstructured pile falls to the
+download size-gate when imported from a host).
+"""
+
+from __future__ import annotations
+
+import posixpath
+from dataclasses import dataclass
+
+# Repo/git boilerplate present everywhere — never data, never identity-bearing.
+#
+# IDENTITY CONTRACT: this set is excluded from every composite digest (both the get
+# sha256-tree and the put/register blake3-tree paths derive their exclusions from
+# `detect().boilerplate`). It is therefore part of the `roar:{combiner}-tree:v1` digest
+# domain: changing a member silently re-digests existing datasets and breaks the
+# put-export <-> get-import identity guarantee. Any edit here MUST bump the tree domain
+# version (v1 -> v2) in `canonical.py` and `composite_builder.py`.
+BOILERPLATE = frozenset({".gitattributes", ".gitignore", "README.md", "LICENSE", ".DS_Store"})
+
+
+@dataclass(frozen=True)
+class Detection:
+    kind: str
+    data: list[str]  # identity-bearing data files
+    meta: list[str]  # identity-bearing, format-declared metadata
+    boilerplate: list[str]  # excluded
+    nested: list[str]  # subdir prefixes that are themselves structural datasets
+
+
+def _split_boilerplate(paths: list[str]) -> tuple[list[str], list[str]]:
+    boiler = [p for p in paths if posixpath.basename(p) in BOILERPLATE]
+    rest = [p for p in paths if posixpath.basename(p) not in BOILERPLATE]
+    return rest, boiler
+
+
+def _has(paths: set[str], name: str) -> bool:
+    return any(p == name or p.endswith("/" + name) for p in paths)
+
+
+def detect(paths: list[str]) -> Detection:
+    """Detect the structural kind of a flat list of relpaths (one composite root)."""
+    rest, boiler = _split_boilerplate(paths)
+    pset = set(rest)
+
+    # --- LeRobot v2.x/v3.x: meta/info.json + data/chunk-*/*.parquet ---
+    if _has(pset, "meta/info.json") and any(p.startswith("data/chunk-") for p in rest):
+        meta = sorted(p for p in rest if p.startswith("meta/"))
+        data = sorted(p for p in rest if p.startswith("data/") or p.startswith("videos/"))
+        other = sorted(set(rest) - set(meta) - set(data))
+        return Detection("lerobot", data, meta, boiler + other, [])
+
+    # --- Zarr store: v2 (.zarray/.zgroup) or v3 (zarr.json) ---
+    if any(posixpath.basename(p) in (".zarray", ".zgroup", "zarr.json") for p in rest):
+        return Detection("zarr", sorted(rest), [], boiler, [])
+
+    # --- Lance dataset: <name>.lance/ with _versions/ + data/ ---
+    if any(p.endswith(".lance") or ".lance/" in ("/" + p) for p in rest) or _has(pset, "_versions"):
+        return Detection("lance", sorted(rest), [], boiler, [])
+
+    # --- TFDS / RLDS: dataset_info.json + features.json (+ *.tfrecord-*) ---
+    if _has(pset, "dataset_info.json") and _has(pset, "features.json"):
+        meta = sorted(
+            p for p in rest if posixpath.basename(p) in ("dataset_info.json", "features.json")
+        )
+        data = sorted(set(rest) - set(meta))
+        return Detection("rlds", data, meta, boiler, [])
+
+    # --- WebDataset: a set of .tar shards ---
+    tars = [p for p in rest if p.endswith(".tar")]
+    if len(tars) >= 2 and len(tars) == len(rest):
+        return Detection("webdataset", sorted(tars), [], boiler, [])
+
+    # --- Flat parquet shards (climbmix / esm2 style) ---
+    parquets = [p for p in rest if p.endswith(".parquet")]
+    if len(parquets) >= 2:
+        excluded = boiler + sorted(set(rest) - set(parquets))
+        return Detection("parquet-shards", sorted(parquets), [], excluded, [])
+
+    return Detection("unstructured", sorted(rest), [], boiler, [])
+
+
+# Formats that declare themselves with a manifest/layout file are nestable as
+# sub-datasets. Loose-shard formats (parquet-shards, webdataset) are deliberately
+# excluded: a directory of partition dirs full of parquet shards (Hive-style
+# ``date=.../part-*.parquet``) is ONE partitioned dataset, not nested sub-datasets,
+# and is indistinguishable from a true container without a per-child manifest.
+_NESTABLE_KINDS = frozenset({"lerobot", "zarr", "lance", "rlds"})
+
+
+def detect_nested(paths: list[str]) -> Detection:
+    """Detect a *container* of sub-datasets (the nesting case).
+
+    Groups paths by top-level directory; if at least two top dirs each independently
+    detect as a *manifest-bearing* structural dataset, returns kind ``"nested"`` with
+    those dirs as nested composites. Otherwise falls back to flat :func:`detect` —
+    so partitioned single datasets and loose-shard splits stay one flat composite.
+    """
+    rest, boiler = _split_boilerplate(paths)
+    by_top: dict[str, list[str]] = {}
+    for p in rest:
+        if "/" in p:
+            top, sub = p.split("/", 1)
+            by_top.setdefault(top, []).append(sub)
+
+    structural_children = [
+        top for top, subs in by_top.items() if detect(subs).kind in _NESTABLE_KINDS
+    ]
+
+    if len(structural_children) >= 2:
+        return Detection("nested", [], [], boiler, sorted(structural_children))
+    return detect(paths)

roar_cli-0.3.4/roar/application/composite/qualifying.py

@@ -0,0 +1,128 @@
+"""The qualifying composite hash: ``<algo>-tree``.
+
+The composite key follows the *source's natural content hash*, not a forced single
+algorithm: a dataset built locally is keyed ``blake3-tree`` (roar already hashed it
+with blake3 during the run — no re-hashing), while a dataset fetched from HF is
+keyed ``sha256-tree`` (HF publishes sha256 LFS oids for free). The component
+digests and the membership bloom use the same algorithm, so identity is internally
+consistent. The ``-tree`` *construction* is one standard, path-sensitive canonical
+form; only the content hash flowing through it varies by provenance.
+
+A :class:`Leaf` is one file's content identity. ``kind`` records dataset ``data``,
+format-declared ``meta`` (e.g. LeRobot ``meta/``), or excluded ``boilerplate``;
+only ``data`` and ``meta`` leaves enter identity. ``verified`` distinguishes
+*asserted* identity (a digest taken from a host's published metadata — trusts the
+host) from *verified* identity (a digest roar computed from the bytes). Verification
+never changes the digest; it only annotates trust.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, field
+from typing import Any
+
+from .canonical import preimage
+
+_IDENTITY_KINDS = ("data", "meta")
+DEFAULT_ALGO = "sha256"
+
+
+def _hasher(algo: str) -> Any:
+    """Return a fresh hasher for the content algorithm (blake3 or a hashlib name)."""
+    if algo == "blake3":
+        import blake3
+
+        return blake3.blake3()
+    return hashlib.new(algo)
+
+
+def _domain(algo: str) -> bytes:
+    """Per-algorithm domain-separation prefix (``roar:<algo>-tree:v1``)."""
+    return f"roar:{algo}-tree:v1\n".encode()
+
+
+@dataclass(frozen=True)
+class Leaf:
+    relpath: str
+    digest: str
+    kind: str = "data"  # data | meta | boilerplate
+    size: int | None = None
+    verified: bool = False  # False => asserted (from host metadata)
+
+
+@dataclass(frozen=True)
+class Composite:
+    digest: str  # the <algo>-tree hex
+    algo: str  # e.g. "sha256-tree" | "blake3-tree"
+    leaves: tuple[Leaf, ...]
+    nested: tuple[Composite, ...] = field(default_factory=tuple)
+    kind: str | None = None  # structural format name; a LABEL, not part of the key
+
+    @property
+    def verified(self) -> bool:
+        """True iff every identity-bearing leaf was content-verified."""
+        return all(leaf.verified for leaf in self.leaves if leaf.kind in _IDENTITY_KINDS)
+
+    @property
+    def identity_leaves(self) -> tuple[Leaf, ...]:
+        return tuple(leaf for leaf in self.leaves if leaf.kind in _IDENTITY_KINDS)
+
+
+def tree(leaves: list[Leaf], *, algo: str = DEFAULT_ALGO) -> str:
+    """Qualifying composite hash over identity-bearing leaves, in ``<algo>-tree``.
+
+    ``algo`` is the per-file content algorithm (``sha256`` for HF/asserted,
+    ``blake3`` for local) and is used both for the leaf digest label and the outer
+    combiner, so the key, components, and bloom all agree.
+    """
+    triples = [(leaf.relpath, algo, leaf.digest) for leaf in leaves if leaf.kind in _IDENTITY_KINDS]
+    hasher = _hasher(algo)
+    hasher.update(preimage(triples, domain=_domain(algo)))
+    return hasher.hexdigest()
+
+
+def sha256_tree(leaves: list[Leaf]) -> str:
+    """``tree`` specialized to sha256 (HF / asserted identity)."""
+    return tree(leaves, algo="sha256")
+
+
+def blake3_tree(leaves: list[Leaf]) -> str:
+    """``tree`` specialized to blake3 (locally-produced identity)."""
+    return tree(leaves, algo="blake3")
+
+
+def build(leaves: list[Leaf], *, algo: str = DEFAULT_ALGO, kind: str | None = None) -> Composite:
+    """Build a flat composite from leaves keyed by the source's content algorithm."""
+    return Composite(
+        digest=tree(leaves, algo=algo),
+        algo=f"{algo}-tree",
+        leaves=tuple(leaves),
+        kind=kind,
+    )
+
+
+def build_nested(
+    children: list[tuple[str, Composite]],
+    *,
+    algo: str = DEFAULT_ALGO,
+    kind: str | None = None,
+) -> Composite:
+    """Composite-of-composites: a parent whose leaves are child composite hashes.
+
+    Used when a declared dataset contains independently-structured sub-datasets
+    (e.g. a directory of ``.zarr`` stores). The parent's identity is over
+    ``(child_relpath, child.digest)`` tagged ``<algo>-tree``, so nesting is preserved
+    and the result is still purely content-derived.
+    """
+    nested_label = f"{algo}-tree"
+    triples = [(relpath, nested_label, child.digest) for relpath, child in children]
+    hasher = _hasher(algo)
+    hasher.update(preimage(triples, domain=_domain(algo)))
+    return Composite(
+        digest=hasher.hexdigest(),
+        algo=nested_label,
+        leaves=(),
+        nested=tuple(child for _relpath, child in children),
+        kind=kind,
+    )

{roar_cli-0.3.3 → roar_cli-0.3.4}/roar/application/get/requests.py

@@ -21,3 +21,5 @@ class GetRequest:
     force: bool = False
     tag: bool = False
     step_name: str | None = None
+    limit: int | None = None  # download only the first N data files (subset get)
+    full_anchor: bool = False  # re-hash non-LFS identity files past the budget

{roar_cli-0.3.3 → roar_cli-0.3.4}/roar/application/get/results.py

@@ -11,6 +11,7 @@ class GetDownloadedFile:
     local_path: str
     hash: str | None = None
     size: int | None = None
+    remote_key: str | None = None  # full manifest path (joins to the LFS sha256 oid)
 
 
 @dataclass(frozen=True)
@@ -33,3 +34,6 @@ class GetResponse:
     git_tag: str | None = None
     warnings: list[str] = field(default_factory=list)
     error: str | None = None
+    # MB of non-LFS identity files excluded from the anchor (over budget, no
+    # --full-anchor); None when the anchor is complete. Drives the CLI warning + hint.
+    anchor_lfs_only_mb: float | None = None