ossllms 0.1.0__py3-none-any.whl

ossllms/__init__.py

@@ -0,0 +1,33 @@
+"""ossllms — pull open-weight AI models from the ossllms preservation network.
+
+A drop-in for huggingface_hub that resolves from the torrent layer, verifies
+integrity + publisher signature, and lands files in the HF-compatible cache.
+
+    from ossllms import snapshot_download
+    path = snapshot_download("Qwen/Qwen2.5-7B")
+"""
+from __future__ import annotations
+
+from .api import (
+    IntegrityError,
+    PullResult,
+    UnverifiedOriginError,
+    hf_hub_download,
+    pull,
+    snapshot_download,
+)
+from .verify import TrustStore, Verdict
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "pull",
+    "snapshot_download",
+    "hf_hub_download",
+    "PullResult",
+    "Verdict",
+    "TrustStore",
+    "IntegrityError",
+    "UnverifiedOriginError",
+    "__version__",
+]

ossllms/api.py

@@ -0,0 +1,194 @@
+"""High-level API. `pull()` is the core; `snapshot_download` / `hf_hub_download`
+mirror the huggingface_hub names for drop-in use.
+"""
+from __future__ import annotations
+
+import shutil
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+from typing import Callable, List, Optional
+
+from . import config
+from .cache import place_into_cache, reuse_from_blobs
+from .denylist import load_denylist
+from .engine import EngineError, get_engine
+from .manifest import model_id, version
+from .resolve import resolve_manifest_with_source
+from .verify import TrustStore, Verdict, verify_manifest
+
+
+class IntegrityError(RuntimeError):
+    pass
+
+
+class UnverifiedOriginError(RuntimeError):
+    pass
+
+
+@dataclass
+class PullResult:
+    path: Path
+    repo_id: str
+    revision: str
+    manifest: dict
+    verdict: Verdict
+
+
+def pull(
+    ref: str,
+    *,
+    catalog: Optional[str] = None,
+    cache_dir=None,
+    revision: Optional[str] = None,
+    include: Optional[List[str]] = None,
+    engine: str = "auto",
+    engine_impl=None,
+    trust: Optional[TrustStore] = None,
+    denylist: Optional[str] = None,
+    require_signature: bool = False,
+    progress: Optional[Callable[[str], None]] = None,
+) -> PullResult:
+    """Resolve, download, verify, and cache a model version.
+
+    Integrity (per-file SHA-256) is ALWAYS enforced. Signature/origin failure
+    raises only when require_signature=True; otherwise it returns an Unverified
+    verdict (community upload) and proceeds, matching the trust UX.
+    """
+    catalog = catalog or config.default_catalog()
+    cache_dir = Path(cache_dir) if cache_dir else config.hf_hub_cache()
+    trust = trust if trust is not None else TrustStore.from_dir(config.trust_dir())
+
+    resolution = resolve_manifest_with_source(ref, catalog)
+    manifest = resolution.manifest
+    repo_id = model_id(manifest)
+    ver = version(manifest)
+    rev = revision or ver
+    denylist = denylist or config.default_denylist()
+    if denylist is not None:
+        load_denylist(denylist, trust).check_manifest(manifest)
+
+    with tempfile.TemporaryDirectory(prefix="ossllms-") as staged:
+        selected = include_to_paths(manifest, include)
+        for path in selected:
+            _safe_artifact_relpath(path)
+        sel_set = set(selected)
+        selected_artifacts = [a for a in manifest["artifacts"] if a["path"] in sel_set]
+
+        # Reuse byte-identical files already in the blob store; fetch only the rest.
+        reused = set(reuse_from_blobs(staged, cache_dir, repo_id, selected_artifacts))
+        missing = [p for p in selected if p not in reused]
+        local = set()
+        if resolution.local_source_dir is not None and missing:
+            local = set(
+                _copy_from_local_source(
+                    resolution.local_source_dir,
+                    staged,
+                    missing,
+                )
+            )
+            missing = [p for p in missing if p not in local]
+
+        eng = None
+        if missing:
+            eng = engine_impl if engine_impl is not None else get_engine(engine, manifest)
+        if progress:
+            engine_name = eng.name if eng is not None else "none"
+            progress(
+                f"engine: {engine_name}; reuse {len(reused)} file(s), "
+                f"local {len(local)} file(s), fetch {len(missing)}"
+            )
+        if missing:
+            try:
+                eng.fetch(manifest, staged, include=missing, progress=progress)
+            except EngineError as exc:
+                _raise_integrity_error_if_staged_file_failed(manifest, staged, trust, sel_set, exc)
+                raise
+
+        verdict = verify_manifest(manifest, staged, trust, only=sel_set)
+
+        if not verdict.integrity_ok:
+            bad = [f"{f.path} ({f.reason})" for f in verdict.files if not f.ok]
+            raise IntegrityError("integrity check failed: " + "; ".join(bad))
+        if require_signature and not verdict.origin_ok:
+            raise UnverifiedOriginError(
+                "publisher signature could not be verified: " + "; ".join(verdict.messages)
+            )
+
+        target = place_into_cache(staged, cache_dir, repo_id, rev, manifest=manifest)
+
+    return PullResult(target, repo_id, rev, manifest, verdict)
+
+
+def include_to_paths(manifest: dict, include: Optional[List[str]]) -> List[str]:
+    import fnmatch
+
+    if not include:
+        return [a["path"] for a in manifest["artifacts"]]
+    return [
+        a["path"]
+        for a in manifest["artifacts"]
+        if any(fnmatch.fnmatch(a["path"], pat) for pat in include)
+    ]
+
+
+def _copy_from_local_source(source_dir: Path, staged_dir, paths: List[str]) -> List[str]:
+    source = Path(source_dir).resolve()
+    staged = Path(staged_dir)
+    copied: List[str] = []
+    for artifact_path in paths:
+        rel = _safe_artifact_relpath(artifact_path)
+        src = (source / rel).resolve()
+        try:
+            src.relative_to(source)
+        except ValueError as exc:
+            raise IntegrityError(f"artifact path escapes local source: {artifact_path!r}") from exc
+        if not src.is_file():
+            continue
+        dst = staged / rel
+        dst.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(src, dst)
+        copied.append(artifact_path)
+    return copied
+
+
+def _safe_artifact_relpath(path: str) -> Path:
+    if not isinstance(path, str) or not path:
+        raise IntegrityError(f"unsafe artifact path: {path!r}")
+    rel = PurePosixPath(path)
+    if rel.is_absolute() or not rel.parts or any(part in ("", ".", "..") for part in rel.parts):
+        raise IntegrityError(f"unsafe artifact path: {path!r}")
+    return Path(*rel.parts)
+
+
+def _raise_integrity_error_if_staged_file_failed(
+    manifest: dict,
+    staged_dir,
+    trust: TrustStore,
+    only: set,
+    cause: EngineError,
+) -> None:
+    verdict = verify_manifest(manifest, staged_dir, trust, only=only)
+    bad = [f"{f.path} ({f.reason})" for f in verdict.files if not f.ok and f.reason != "missing"]
+    if bad:
+        raise IntegrityError("integrity check failed: " + "; ".join(bad)) from cause
+
+
+# --- huggingface_hub-compatible surface -------------------------------------
+
+def snapshot_download(repo_id: str, *, revision: Optional[str] = None,
+                      allow_patterns: Optional[List[str]] = None,
+                      cache_dir=None, catalog: Optional[str] = None,
+                      **_ignored) -> str:
+    """Drop-in for huggingface_hub.snapshot_download. Returns the local path."""
+    ref = repo_id if not revision else f"{repo_id}@{revision}"
+    res = pull(ref, catalog=catalog, cache_dir=cache_dir, include=allow_patterns)
+    return str(res.path)
+
+
+def hf_hub_download(repo_id: str, filename: str, *, revision: Optional[str] = None,
+                    cache_dir=None, catalog: Optional[str] = None, **_ignored) -> str:
+    """Drop-in for huggingface_hub.hf_hub_download. Returns the local file path."""
+    ref = repo_id if not revision else f"{repo_id}@{revision}"
+    res = pull(ref, catalog=catalog, cache_dir=cache_dir, include=[filename])
+    return str(res.path / filename)

ossllms/cache.py

@@ -0,0 +1,102 @@
+"""HF-compatible cache with a blob store + hardlink dedup.
+
+Layout (identical to huggingface_hub, so transformers/vllm/llama.cpp load unchanged
+AND versions/quants/repos share bytes):
+
+    <cache>/models--<org>--<name>/
+      blobs/<sha256>                 one physical copy per unique file
+      snapshots/<revision>/<path>    hardlink into blobs/
+      refs/<channel>                 e.g. refs/main -> a revision
+
+Identical files across versions point at the same blob inode, so disk cost is the
+size of the UNIQUE bytes, not N x per version. `reuse_from_blobs` lets a pull skip
+downloading any file already present as a blob (the "only changed bytes move"
+update path in docs/VERSIONING.md).
+"""
+from __future__ import annotations
+
+import os
+import shutil
+from pathlib import Path
+from typing import List
+
+
+def repo_folder_name(repo_id: str) -> str:
+    return "models--" + repo_id.replace("/", "--")
+
+
+def repo_dir(cache_root, repo_id: str) -> Path:
+    return Path(cache_root) / repo_folder_name(repo_id)
+
+
+def blobs_dir(cache_root, repo_id: str) -> Path:
+    return repo_dir(cache_root, repo_id) / "blobs"
+
+
+def blob_path(cache_root, repo_id: str, sha256: str) -> Path:
+    return blobs_dir(cache_root, repo_id) / sha256
+
+
+def snapshot_dir(cache_root, repo_id: str, revision: str) -> Path:
+    return repo_dir(cache_root, repo_id) / "snapshots" / revision
+
+
+def has_blob(cache_root, repo_id: str, sha256: str) -> bool:
+    return blob_path(cache_root, repo_id, sha256).exists()
+
+
+def _link_or_copy(src: Path, dst: Path) -> None:
+    dst.parent.mkdir(parents=True, exist_ok=True)
+    if dst.exists():
+        return
+    try:
+        os.link(src, dst)           # hardlink (same filesystem)
+    except OSError:
+        shutil.copy2(src, dst)      # cross-filesystem fallback
+
+
+def reuse_from_blobs(staged_dir, cache_root, repo_id: str, artifacts) -> List[str]:
+    """Hardlink already-present blobs into the staging dir; return reused paths.
+
+    Call before downloading so files already on disk (from a prior version, a quant,
+    or another pull) are not fetched again.
+    """
+    reused: List[str] = []
+    staged = Path(staged_dir)
+    for a in artifacts:
+        bp = blob_path(cache_root, repo_id, a["sha256"])
+        if bp.exists():
+            _link_or_copy(bp, staged / a["path"])
+            reused.append(a["path"])
+    return reused
+
+
+def place_into_cache(staged_dir, cache_root, repo_id: str, revision: str, manifest=None) -> Path:
+    """Store staged files as blobs (keyed by sha256) and hardlink them into the
+    snapshot dir. Files in the manifest are deduped via the blob store; any extras
+    are copied as-is."""
+    target = snapshot_dir(cache_root, repo_id, revision)
+    target.mkdir(parents=True, exist_ok=True)
+    staged = Path(staged_dir)
+    sha_by_path = {a["path"]: a["sha256"] for a in (manifest or {}).get("artifacts", [])}
+
+    for src in staged.rglob("*"):
+        if not src.is_file():
+            continue
+        rel = src.relative_to(staged)
+        sha = sha_by_path.get(str(rel))
+        snap_dst = target / rel
+        snap_dst.parent.mkdir(parents=True, exist_ok=True)
+        if sha:
+            bp = blob_path(cache_root, repo_id, sha)
+            _link_or_copy(src, bp)          # store blob (idempotent)
+            if snap_dst.exists():
+                snap_dst.unlink()
+            _link_or_copy(bp, snap_dst)     # snapshot hardlinks to blob
+        elif not snap_dst.exists():
+            shutil.copy2(src, snap_dst)
+
+    refs = repo_dir(cache_root, repo_id) / "refs"
+    refs.mkdir(parents=True, exist_ok=True)
+    (refs / "main").write_text(revision, encoding="utf-8")
+    return target