aixand-lib-github-source 0.1.0b0__py3-none-any.whl

github_source/contracts.py

@@ -0,0 +1,211 @@
+"""Bounded, portable, immutable GitHub source trees. No persistence or application policy."""
+
+import base64
+import hashlib
+import json
+import unicodedata
+from pathlib import PurePosixPath
+
+from .errors import GitHubError
+
+MAX_DOWNLOAD = 64 * 1024 * 1024
+MAX_TOTAL = 128 * 1024 * 1024
+MAX_FILE = 16 * 1024 * 1024
+MAX_ENTRIES = 10000
+
+
+def require(condition, message, code="validation"):
+    if not condition:
+        raise GitHubError(code, message)
+
+
+def canonical(value):
+    def check(item):
+        if isinstance(item, dict):
+            require(
+                all(isinstance(key, str) for key in item),
+                "JSON keys must be text",
+                "integrity",
+            )
+            for child in item.values():
+                check(child)
+        elif isinstance(item, list):
+            for child in item:
+                check(child)
+        else:
+            require(
+                item is None or type(item) in (str, int, float, bool),
+                "Invalid JSON value",
+                "integrity",
+            )
+
+    try:
+        check(value)
+        return json.dumps(
+            value,
+            ensure_ascii=False,
+            sort_keys=True,
+            separators=(",", ":"),
+            allow_nan=False,
+        ).encode("utf-8")
+    except (ValueError, TypeError, UnicodeError, RecursionError) as error:
+        raise GitHubError("integrity", "Expected finite UTF-8 JSON") from error
+
+
+def digest(value):
+    return "sha256:" + hashlib.sha256(canonical(value)).hexdigest()
+
+
+def loads(data):
+    def pairs(items):
+        result = {}
+        for key, value in items:
+            require(key not in result, "Duplicate JSON key", "integrity")
+            result[key] = value
+        return result
+
+    try:
+        result = json.loads(
+            data,
+            object_pairs_hook=pairs,
+            parse_constant=lambda _: require(False, "Non-finite JSON", "integrity"),
+        )
+        canonical(result)
+        return result
+    except (ValueError, UnicodeError) as error:
+        raise GitHubError("validation", "Invalid UTF-8 JSON") from error
+
+
+def relative(value, *, empty=False):
+    require(
+        isinstance(value, str)
+        and len(value) <= 1024
+        and "\\" not in value
+        and "\0" not in value
+        and ":" not in value,
+        "Invalid package path",
+    )
+    if value == "" and empty:
+        return ""
+    try:
+        value.encode("utf-8")
+    except UnicodeError as error:
+        raise GitHubError("validation", "Source path must be valid UTF-8") from error
+    p = PurePosixPath(value)
+    require(
+        value
+        and not p.is_absolute()
+        and all(part not in ("", ".", "..") for part in value.split("/"))
+        and str(p) == value,
+        "Package path must stay inside its root",
+    )
+    require(
+        len(p.parts) <= 32
+        and all(
+            len(part.encode("utf-8")) <= 255
+            and not part.endswith((".", " "))
+            and not any(ord(c) < 32 or ord(c) == 127 for c in part)
+            and not any(c in part for c in '<>"|?*')
+            and part.split(".")[0].upper()
+            not in {
+                "CON",
+                "PRN",
+                "AUX",
+                "NUL",
+                *(f"COM{i}" for i in range(1, 10)),
+                *(f"LPT{i}" for i in range(1, 10)),
+            }
+            for part in p.parts
+        ),
+        "Source path is not portable",
+    )
+    return value
+
+
+def validate_tree(tree):
+    require(
+        isinstance(tree, dict)
+        and set(tree) == {"schema", "files", "directories"}
+        and tree.get("schema") == "github-source/tree-1"
+        and isinstance(tree.get("files"), dict)
+        and isinstance(tree.get("directories"), list),
+        "Invalid source tree",
+        "integrity",
+    )
+    require(
+        len(tree["files"]) + len(tree["directories"]) <= MAX_ENTRIES,
+        "Too many source entries",
+    )
+    require(
+        all(isinstance(path, str) for path in [*tree["directories"], *tree["files"]]),
+        "Source paths must be text",
+        "integrity",
+    )
+    total, folded = 0, set()
+    require(
+        len(set(tree["directories"])) == len(tree["directories"]),
+        "Duplicate source directories",
+    )
+    all_directories = set(tree["directories"])
+    for path in [*tree["directories"], *tree["files"]]:
+        relative(path)
+        all_directories.update(
+            str(parent) for parent in PurePosixPath(path).parents if str(parent) != "."
+        )
+    require(
+        len(all_directories) + len(tree["files"]) <= MAX_ENTRIES,
+        "Too many source entries",
+        "size",
+    )
+    for path in [*all_directories, *tree["files"]]:
+        relative(path)
+        require(
+            unicodedata.normalize("NFC", path).casefold() not in folded,
+            "Source contains colliding paths",
+        )
+        folded.add(unicodedata.normalize("NFC", path).casefold())
+    for path, row in tree["files"].items():
+        require(
+            isinstance(row, dict)
+            and set(row) == {"content_b64", "executable"}
+            and type(row["executable"]) is bool
+            and isinstance(row["content_b64"], str),
+            "Invalid source file",
+            "integrity",
+        )
+        try:
+            data = base64.b64decode(row["content_b64"], validate=True)
+        except (ValueError, TypeError) as error:
+            raise GitHubError("integrity", "Invalid source file encoding") from error
+        total += len(data)
+        require(
+            len(data) <= MAX_FILE and total <= MAX_TOTAL, "Source exceeds size limits"
+        )
+        parents = PurePosixPath(path).parents
+        require(
+            not any(str(p) in tree["files"] for p in parents),
+            "Source file is also a directory",
+        )
+    return {
+        **tree,
+        "directories": sorted(all_directories),
+        "files": dict(sorted(tree["files"].items())),
+    }
+
+
+def subtree(tree, path):
+    relative(path, empty=True)
+    if not path:
+        return tree
+    prefix = path + "/"
+    return {
+        "schema": tree["schema"],
+        "files": {
+            key[len(prefix) :]: value
+            for key, value in tree["files"].items()
+            if key.startswith(prefix)
+        },
+        "directories": [
+            key[len(prefix) :] for key in tree["directories"] if key.startswith(prefix)
+        ],
+    }

github_source/errors.py

@@ -0,0 +1,17 @@
+"""Stable acquisition errors; content activation is always the caller's decision."""
+
+
+class GitHubError(Exception):
+    def __init__(self, code, message, *, status=None, retry_after=None):
+        super().__init__(message)
+        self.code = code
+        self.status = status
+        self.retry_after = retry_after
+
+    def to_dict(self):
+        result = {"code": self.code, "message": str(self)}
+        if self.status is not None:
+            result["status"] = self.status
+        if self.retry_after is not None:
+            result["retry_after"] = self.retry_after
+        return result

github_source/snapshots.py

@@ -0,0 +1,113 @@
+"""Safe, bounded in-memory repository archive capture."""
+
+import base64
+import gzip
+import io
+import tarfile
+import zlib
+
+from .contracts import (
+    MAX_DOWNLOAD,
+    MAX_ENTRIES,
+    MAX_FILE,
+    MAX_TOTAL,
+    relative,
+    require,
+    validate_tree,
+)
+from .errors import GitHubError
+
+
+class ExpandedLimit:
+    """Also bound tar metadata/PAX expansion before tarfile allocates it."""
+
+    def __init__(self, stream):
+        self.stream, self.count = stream, 0
+
+    def read(self, size=-1):
+        remaining = MAX_TOTAL + MAX_ENTRIES * 4096 - self.count
+        value = self.stream.read(
+            min(size, remaining + 1) if size >= 0 else remaining + 1
+        )
+        self.count += len(value)
+        require(
+            len(value) <= remaining, "Expanded archive metadata exceeds limits", "size"
+        )
+        return value
+
+
+def archive_tree(data, *, check=None):
+    files, directories, total, seen, prefix = {}, set(), 0, set(), None
+    require(len(data) <= MAX_DOWNLOAD, "Archive exceeds size limit")
+    expanded = gzip.GzipFile(fileobj=io.BytesIO(data))
+    try:
+        with tarfile.open(fileobj=ExpandedLimit(expanded), mode="r|") as archive:
+            count = 0
+            for entry in archive:
+                if check is not None:
+                    check()
+                count += 1
+                require(
+                    count <= MAX_ENTRIES + 1, "Archive has too many entries", "size"
+                )
+                name = entry.name.rstrip("/")
+                relative(name)
+                parts = name.split("/", 1)
+                prefix = prefix or parts[0]
+                require(parts[0] == prefix, "Archive contains several roots")
+                if len(parts) == 1:
+                    require(entry.isdir(), "Archive root must be a directory")
+                    continue
+                path = relative(parts[1])
+                require(path.casefold() not in seen, "Archive contains duplicate paths")
+                seen.add(path.casefold())
+                require(
+                    entry.isdir() or entry.isfile(),
+                    "Archive contains a link or special file",
+                )
+                if entry.isdir():
+                    directories.add(path)
+                else:
+                    require(
+                        0 <= entry.size <= MAX_FILE,
+                        "Archive file exceeds size limit",
+                        "size",
+                    )
+                    total += entry.size
+                    require(
+                        total <= MAX_TOTAL,
+                        "Expanded archive exceeds size limit",
+                        "size",
+                    )
+                    stream = archive.extractfile(entry)
+                    require(stream is not None, "Archive file unavailable", "integrity")
+                    content = stream.read(entry.size + 1)
+                    require(
+                        len(content) == entry.size, "Truncated archive", "integrity"
+                    )
+                    require(
+                        not content.startswith(
+                            b"version https://git-lfs.github.com/spec/v1\n"
+                        ),
+                        "Git LFS assets require a complete local checkout; import that local package",
+                        "unsupported",
+                    )
+                    files[path] = {
+                        "content_b64": base64.b64encode(content).decode("ascii"),
+                        "executable": bool(entry.mode & 0o111),
+                    }
+                    parent = path.rpartition("/")[0]
+                    while parent:
+                        directories.add(parent)
+                        parent = parent.rpartition("/")[0]
+    except (tarfile.TarError, EOFError, gzip.BadGzipFile, zlib.error) as error:
+        raise GitHubError("integrity", "Invalid repository archive") from error
+    finally:
+        expanded.close()
+    return validate_tree(
+        {
+            "schema": "github-source/tree-1",
+            "files": files,
+            "directories": sorted(directories),
+        }
+    )

github_source/types.py

@@ -0,0 +1,40 @@
+"""Versioned JSON contracts. Paths in a selection remain repository-relative."""
+
+from typing import Literal, NotRequired, TypedDict
+
+
+class ResolvedSource(TypedDict):
+    schema: str
+    kind: Literal["github"]
+    repository_id: str
+    repository: str
+    url: str
+    ref: str
+    path: str
+    commit: str
+    reference_kind: Literal["branch", "tag", "commit"]
+    tree_sha: NotRequired[str]
+
+
+class FileEntry(TypedDict):
+    content_b64: str
+    executable: bool
+
+
+class SourceTree(TypedDict):
+    schema: str
+    files: dict[str, FileEntry]
+    directories: list[str]
+
+
+class Snapshot(TypedDict):
+    schema: str
+    source: ResolvedSource
+    tree: SourceTree
+    digest: str
+
+
+class Comparison(TypedDict):
+    before: str
+    after: str
+    relationship: Literal["ahead", "identical", "behind", "diverged"]

github_source/urls.py

@@ -0,0 +1,67 @@
+"""Public GitHub URLs and restricted HTTPS redirects."""
+
+import re
+from urllib.parse import unquote, urlsplit
+from urllib.request import HTTPRedirectHandler
+
+from .contracts import require
+from .errors import GitHubError
+
+COMMIT = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})")
+
+REPO_PART = re.compile(r"[A-Za-z0-9_.-]{1,100}")
+
+
+class Redirects(HTTPRedirectHandler):
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        parts = urlsplit(newurl)
+        require(
+            parts.scheme == "https"
+            and parts.hostname
+            in {
+                "api.github.com",
+                "codeload.github.com",
+                "github.com",
+                "raw.githubusercontent.com",
+            }
+            and parts.port in (None, 443)
+            and not parts.username,
+            "GitHub redirected outside the supported hosts",
+            "network",
+        )
+        return super().redirect_request(req, fp, code, msg, headers, newurl)
+
+
+def parse_url(url):
+    require(isinstance(url, str) and len(url) <= 2048, "Invalid GitHub URL")
+    require(not any(ord(c) < 32 or ord(c) == 127 for c in url), "Invalid GitHub URL")
+    try:
+        parts = urlsplit(url)
+        parts.port
+    except ValueError as error:
+        raise GitHubError("validation", "Invalid GitHub URL") from error
+    require(
+        parts.scheme == "https"
+        and parts.hostname == "github.com"
+        and parts.port in (None, 443)
+        and not parts.username
+        and not parts.password
+        and not parts.query
+        and not parts.fragment,
+        "Use an HTTPS github.com repository or file URL",
+    )
+    path = unquote(parts.path).strip("/").split("/")
+    require(
+        len(path) >= 2
+        and all(REPO_PART.fullmatch(p) and p not in (".", "..") for p in path[:2]),
+        "Invalid GitHub repository",
+    )
+    owner, repo = path[:2]
+    repo = repo.removesuffix(".git")
+    require(repo and repo not in (".", ".."), "Invalid GitHub repository")
+    tail = path[2:]
+    require(
+        not tail or (tail[0] in ("tree", "blob") and len(tail) >= 2),
+        "Use a repository, tree or blob URL",
+    )
+    return owner, repo, tail[1:] if tail else []