bladekit 4.5.0__py3-none-any.whl

bladekit/_bundle.py

@@ -0,0 +1,459 @@
+"""Project bundling: deterministic tar.gz of the project + sha256.
+
+The bundle root is the directory of the entrypoint script. ``.forgeignore`` is
+honored with ``.gitignore`` syntax and matching; ``.git``, ``.venv``,
+``__pycache__`` and a few other always-noise paths are excluded unconditionally.
+
+Determinism matters for dedup (conventions.md / openapi dedup by sha256): the
+same project content must produce the same sha256 across machines, so we sort
+entries and zero out mtimes / uid / gid.
+"""
+
+from __future__ import annotations
+
+import gzip
+import hashlib
+import io
+import os
+import posixpath
+import re
+import tarfile
+from pathlib import Path
+from typing import Callable, List, NamedTuple, Optional, Tuple
+
+# Always excluded, regardless of .forgeignore.
+ALWAYS_IGNORE = (
+    ".git",
+    ".venv",
+    "venv",
+    "__pycache__",
+    ".forge",
+    ".mypy_cache",
+    ".pytest_cache",
+    ".DS_Store",
+)
+ALWAYS_IGNORE_SUFFIXES = (".pyc", ".pyo")
+
+
+# ── .forgeignore: git's .gitignore rules (dir.c + wildmatch WM_PATHNAME) ──────
+
+_POSIX_CLASSES = {
+    "alnum": "a-zA-Z0-9",
+    "alpha": "a-zA-Z",
+    "blank": " \\t",
+    "cntrl": "\\x00-\\x1f\\x7f",
+    "digit": "0-9",
+    "graph": "\\x21-\\x7e",
+    "lower": "a-z",
+    "print": "\\x20-\\x7e",
+    "punct": "!-/:-@\\[-`{-~",
+    "space": " \\t\\n\\r\\f\\v",
+    "upper": "A-Z",
+    "xdigit": "0-9A-Fa-f",
+}
+
+
+class _GitignoreRule(NamedTuple):
+    match: Callable[[str], bool]
+    negate: bool  # "!pattern" re-includes
+    dir_only: bool  # "pattern/" matches directories only
+    basename: bool  # no "/" in the pattern: match the name at any depth
+
+
+def _never(_path: str) -> bool:
+    return False
+
+
+def _wildmatch_class(pat: str, i: int) -> Optional[Tuple[str, int]]:
+    """Translate the bracket expression starting at ``pat[i] == "["``.
+
+    Returns ``(regex, index after "]")``, or ``None`` when git's wildmatch
+    would abort (unclosed bracket, unknown ``[:class:]``): such a pattern
+    never matches.
+    """
+    n = len(pat)
+    j = i + 1
+    negate = j < n and pat[j] in "!^"
+    if negate:
+        j += 1
+    parts: List[str] = []
+    first = True
+    while j < n:
+        c = pat[j]
+        if c == "]" and not first:
+            return ("[^" if negate else "[") + "".join(parts) + "]", j + 1
+        first = False
+        if c == "\\" and j + 1 < n:
+            parts.append(re.escape(pat[j + 1]))
+            j += 2
+        elif pat.startswith("[:", j):
+            end = pat.find(":]", j + 2)
+            if end == -1:
+                parts.append(re.escape(c))
+                j += 1
+                continue
+            cls = _POSIX_CLASSES.get(pat[j + 2 : end])
+            if cls is None:
+                return None
+            parts.append(cls)
+            j = end + 2
+        elif c == "-" and parts and j + 1 < n and pat[j + 1] != "]":
+            parts.append("-")  # range between the neighbouring characters
+            j += 1
+        else:
+            parts.append(re.escape(c))
+            j += 1
+    return None
+
+
+def _compile_gitignore_glob(pat: str) -> Callable[[str], bool]:
+    """wildmatch(WM_PATHNAME): ``*``/``?``/``[...]`` never match ``/``; ``**``
+    spans directories only as a whole segment (``**/x``, ``x/**/y``, ``x/**``)
+    and is a plain ``*`` anywhere else."""
+    out: List[str] = []
+    i, n = 0, len(pat)
+    while i < n:
+        c = pat[i]
+        if c == "*":
+            j = i
+            while j < n and pat[j] == "*":
+                j += 1
+            whole_segment = (i == 0 or pat[i - 1] == "/") and (j == n or pat[j] == "/")
+            if j - i >= 2 and whole_segment:
+                if j == n:
+                    out.append(".*")
+                else:
+                    out.append("(?:.*/)?")
+                    j += 1  # the "/" is part of "**/"
+            else:
+                out.append("[^/]*")
+            i = j
+        elif c == "?":
+            out.append("[^/]")
+            i += 1
+        elif c == "[":
+            cls = _wildmatch_class(pat, i)
+            if cls is None:
+                return _never
+            out.append("(?!/)" + cls[0])
+            i = cls[1]
+        elif c == "\\":
+            if i + 1 == n:
+                return _never  # dangling escape: wildmatch never matches
+            out.append(re.escape(pat[i + 1]))
+            i += 2
+        else:
+            out.append(re.escape(c))
+            i += 1
+    try:
+        compiled = re.compile("".join(out))
+    except re.error:  # e.g. a reversed range "[z-a]"
+        return _never
+    return lambda path: compiled.fullmatch(path) is not None
+
+
+def _trim_trailing_spaces(line: str) -> str:
+    """Drop trailing spaces unless escaped with a backslash (git's rule)."""
+    end = len(line)
+    while end and line[end - 1] == " ":
+        backslashes = 0
+        k = end - 2
+        while k >= 0 and line[k] == "\\":
+            backslashes += 1
+            k -= 1
+        if backslashes % 2:
+            break
+        end -= 1
+    return line[:end]
+
+
+def _parse_gitignore_line(line: str) -> Optional[_GitignoreRule]:
+    if line.startswith("#"):
+        return None
+    pattern = _trim_trailing_spaces(line)
+    negate = pattern.startswith("!")
+    if negate:
+        pattern = pattern[1:]
+    dir_only = pattern.endswith("/")
+    if dir_only:
+        pattern = pattern[:-1]
+    if not pattern:
+        return None
+    basename = "/" not in pattern
+    if pattern.startswith("/"):
+        pattern = pattern[1:]
+    return _GitignoreRule(_compile_gitignore_glob(pattern), negate, dir_only, basename)
+
+
+def _load_forgeignore(root: Path) -> List[_GitignoreRule]:
+    path = root / ".forgeignore"
+    if not path.exists():
+        return []
+    text = path.read_text(encoding="utf-8", errors="replace").removeprefix("\ufeff")
+    rules = [_parse_gitignore_line(line) for line in text.splitlines()]
+    return [r for r in rules if r is not None]
+
+
+def _forgeignored(rel_posix: str, is_dir: bool, rules: List[_GitignoreRule]) -> bool:
+    """Last matching rule wins, as in git. Callers never descend into an
+    ignored directory, so (as in git) nothing below it can be re-included."""
+    ignored = False
+    name = rel_posix.rsplit("/", 1)[-1]
+    for rule in rules:
+        if rule.dir_only and not is_dir:
+            continue
+        if rule.match(name if rule.basename else rel_posix):
+            ignored = not rule.negate
+    return ignored
+
+
+def _is_always_ignored(name: str) -> bool:
+    if name in ALWAYS_IGNORE:
+        return True
+    return any(name.endswith(suf) for suf in ALWAYS_IGNORE_SUFFIXES)
+
+
+def collect_files(root: Path) -> List[Tuple[Path, str]]:
+    """Return ``(absolute_path, arcname)`` for every file to include, sorted."""
+    root = root.resolve()
+    rules = _load_forgeignore(root)
+    out: List[Tuple[Path, str]] = []
+
+    for dirpath, dirnames, filenames in os.walk(root):
+        dir_abs = Path(dirpath)
+        rel_dir = dir_abs.relative_to(root)
+
+        # Prune ignored directories in place so os.walk doesn't descend.
+        kept_dirs = []
+        for d in dirnames:
+            if _is_always_ignored(d):
+                continue
+            rel_posix = (rel_dir / d).as_posix() if str(rel_dir) != "." else d
+            if _forgeignored(rel_posix, True, rules):
+                continue
+            kept_dirs.append(d)
+        dirnames[:] = sorted(kept_dirs)
+
+        for f in sorted(filenames):
+            if _is_always_ignored(f):
+                continue
+            rel_posix = (rel_dir / f).as_posix() if str(rel_dir) != "." else f
+            if _forgeignored(rel_posix, False, rules):
+                continue
+            abs_path = dir_abs / f
+            if abs_path.is_symlink() or not abs_path.is_file():
+                # skip symlinks/specials — keep bundles simple and safe.
+                continue
+            out.append((abs_path, rel_posix))
+
+    out.sort(key=lambda t: t[1])
+    return out
+
+
+def build_bundle(root: Path) -> Tuple[bytes, str, List[str]]:
+    """Build the tar.gz in memory; return ``(data, sha256_hex, arcnames)``.
+
+    Deterministic: sorted entries, mtime/uid/gid zeroed, fixed gzip mtime so the
+    same content always hashes identically (enables server-side dedup).
+    """
+    files = collect_files(root)
+    raw = io.BytesIO()
+    # mtime=0 in the gzip header keeps the archive byte-stable.
+    with gzip.GzipFile(fileobj=raw, mode="wb", mtime=0) as gz:
+        with tarfile.open(fileobj=gz, mode="w") as tar:  # type: ignore[arg-type]
+            for abs_path, arcname in files:
+                info = tar.gettarinfo(str(abs_path), arcname=arcname)
+                info.mtime = 0
+                info.uid = info.gid = 0
+                info.uname = info.gname = ""
+                info.mode = 0o644
+                with abs_path.open("rb") as fh:
+                    tar.addfile(info, fh)
+
+    data = raw.getvalue()
+    digest = hashlib.sha256(data).hexdigest()
+    return data, digest, [a for _, a in files]
+
+
+# ── from_dockerfile build context (docs/specs/image-from-dockerfile.md) ──────
+
+
+# Regex metacharacters that are literal in a Docker ignore pattern; the rest of
+# the pattern syntax (* ? ** [ ] \) is translated by _compile_dockerignore.
+_DOCKERIGNORE_LITERAL_META = frozenset(".+()|{}$")
+
+
+def _clean_dockerignore_pattern(pattern: str) -> str:
+    """``filepath.Clean`` + leading-``/`` strip, as docker's ignorefile.ReadAll does.
+
+    Every pattern is relative to the context root, so ``/a`` and ``a`` are the
+    same pattern.
+    """
+    if pattern.startswith("/"):
+        pattern = "/" + pattern.lstrip("/")  # Go collapses "//a"; normpath keeps it
+    pattern = posixpath.normpath(pattern)
+    if len(pattern) > 1 and pattern.startswith("/"):
+        pattern = pattern[1:]
+    return pattern
+
+
+def _compile_dockerignore(pattern: str) -> Callable[[str], bool]:
+    """Matcher for one cleaned pattern — a port of moby/patternmatcher.
+
+    ``*`` and ``?`` never cross ``/``; ``**`` spans any number of directories
+    (``**/`` included). The returned callable takes a root-relative POSIX path.
+    Raises ``ValueError`` on a malformed pattern (e.g. an unclosed ``[``).
+    """
+    regex = ""
+    kind = "exact"
+    i, n = 0, len(pattern)
+    first = True
+    while i < n:
+        ch = pattern[i]
+        i += 1
+        if ch == "*":
+            if i < n and pattern[i] == "*":
+                i += 1
+                if i < n and pattern[i] == "/":
+                    i += 1
+                if i >= n:
+                    # trailing "**": everything below (gitignore semantics)
+                    if kind == "exact":
+                        kind = "prefix"
+                    else:
+                        regex += ".*"
+                        kind = "regex"
+                else:
+                    regex += "(.*/)?"
+                    kind = "regex"
+                if first:
+                    kind = "suffix"
+            else:
+                regex += "[^/]*"
+                kind = "regex"
+        elif ch == "?":
+            regex += "[^/]"
+            kind = "regex"
+        elif ch in _DOCKERIGNORE_LITERAL_META:
+            regex += "\\" + ch
+        elif ch == "\\":
+            if i < n:
+                regex += "\\" + pattern[i]
+                i += 1
+                kind = "regex"
+            else:
+                regex += "\\"
+        elif ch in "[]":
+            regex += ch
+            kind = "regex"
+        else:
+            regex += ch
+        first = False
+
+    if kind == "exact":
+        return lambda path: path == pattern
+    if kind == "prefix":
+        prefix = pattern[:-2]
+        return lambda path: path.startswith(prefix)
+    if kind == "suffix":
+        suffix = pattern[2:]
+        bare = suffix[1:] if suffix.startswith("/") else None  # "**/foo" matches "foo"
+        return lambda path: path.endswith(suffix) or path == bare
+    try:
+        compiled = re.compile(regex)
+    except re.error as e:
+        raise ValueError(f".dockerignore: invalid pattern {pattern!r}: {e}") from None
+    return lambda path: compiled.fullmatch(path) is not None
+
+
+def _load_dockerignore(root: Path) -> List[Callable[[str], bool]]:
+    """Load ``.dockerignore`` matchers from the context root, parsed like docker.
+
+    ``#`` comments (only at the very start of a line), a leading UTF-8 BOM, and
+    surrounding whitespace follow docker's ``ignorefile.ReadAll``. ``!``
+    negation is not supported in v1 and is rejected loudly rather than silently
+    including/excluding files differently than docker would.
+    """
+    path = root / ".dockerignore"
+    if not path.exists():
+        return []
+    text = path.read_text(encoding="utf-8", errors="replace").removeprefix("\ufeff")
+    matchers: List[Callable[[str], bool]] = []
+    for line in text.splitlines():
+        if line.startswith("#"):
+            continue
+        stripped = line.strip()
+        if not stripped:
+            continue
+        if stripped.startswith("!"):
+            raise ValueError(
+                f".dockerignore: negation pattern {stripped!r} is not supported "
+                "by AI Fabric yet — restructure the ignore rules without '!'"
+            )
+        matchers.append(_compile_dockerignore(_clean_dockerignore_pattern(stripped)))
+    return matchers
+
+
+def collect_context_files(root: Path) -> List[Tuple[Path, str]]:
+    """``(absolute_path, arcname)`` for every build-context file, sorted.
+
+    Docker-compatible: only ``.dockerignore`` filters the tree (no
+    ``.forgeignore``, no built-in excludes — ignore ``.git``/``.venv``
+    explicitly if unwanted), with docker's matching rules: patterns are
+    relative to the context root and a matching directory drops its whole
+    subtree (moby ``MatchesOrParentMatches``). Symlinks and special files are
+    skipped, as in code bundles: the builder refuses to extract them.
+    """
+    root = root.resolve()
+    matchers = _load_dockerignore(root)
+    out: List[Tuple[Path, str]] = []
+
+    for dirpath, dirnames, filenames in os.walk(root):
+        dir_abs = Path(dirpath)
+        rel_dir = dir_abs.relative_to(root)
+
+        kept_dirs = []
+        for d in dirnames:
+            rel_posix = (rel_dir / d).as_posix() if str(rel_dir) != "." else d
+            if any(m(rel_posix) for m in matchers):
+                continue
+            kept_dirs.append(d)
+        dirnames[:] = sorted(kept_dirs)
+
+        for f in sorted(filenames):
+            rel_posix = (rel_dir / f).as_posix() if str(rel_dir) != "." else f
+            if any(m(rel_posix) for m in matchers):
+                continue
+            abs_path = dir_abs / f
+            if abs_path.is_symlink() or not abs_path.is_file():
+                continue
+            out.append((abs_path, rel_posix))
+
+    out.sort(key=lambda t: t[1])
+    return out
+
+
+def build_context_bundle(root: Path) -> Tuple[bytes, str, List[str]]:
+    """Deterministic tar.gz of a Dockerfile build context; ``(data, sha256, arcnames)``.
+
+    Same determinism rules as :func:`build_bundle` (sorted entries, zeroed
+    mtime/uid/gid, fixed gzip mtime ⇒ same content, same sha256, free
+    server-side dedup), with one difference: the executable bit is preserved
+    (``COPY`` keeps file modes — a copied script must stay runnable).
+    """
+    files = collect_context_files(root)
+    raw = io.BytesIO()
+    with gzip.GzipFile(fileobj=raw, mode="wb", mtime=0) as gz:
+        with tarfile.open(fileobj=gz, mode="w") as tar:  # type: ignore[arg-type]
+            for abs_path, arcname in files:
+                info = tar.gettarinfo(str(abs_path), arcname=arcname)
+                info.mtime = 0
+                info.uid = info.gid = 0
+                info.uname = info.gname = ""
+                info.mode = 0o755 if info.mode & 0o100 else 0o644
+                with abs_path.open("rb") as fh:
+                    tar.addfile(info, fh)
+
+    data = raw.getvalue()
+    digest = hashlib.sha256(data).hexdigest()
+    return data, digest, [a for _, a in files]