codeimprint 0.5.0a11__py3-none-any.whl

codeimprint_cli/linecount.py

@@ -0,0 +1,340 @@
+"""The line count of a seal (docs/line-count.md): the size a project is priced by.
+
+Non-blank lines in the files Git tracks at the sealed commit, without binary, vendored and
+generated files (path and content rules below, then ``linguist-vendored`` and
+``linguist-generated`` from ``.gitattributes``, read by ``git check-attr``). The count is sent
+with the seal and kept in the Imprint's public header, so the digest you sign covers it.
+
+This is counting only: nothing here looks at what the code means. It stays fast on large
+repositories: one ``git check-attr`` and one ``git cat-file --batch`` process per count, path
+rules before any content is read, and a per-blob cache in the Git directory (blobs are
+content-addressed, so a count never goes stale).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import subprocess
+import threading
+from dataclasses import dataclass, field
+from pathlib import Path, PurePosixPath
+
+RULES = "lines/1"
+
+# -- the rules (the same as the engine's spot-check) -------------------------------------------
+VENDORED_DIRS = frozenset({
+    "node_modules", "bower_components", "jspm_packages", "vendor", "vendors", "third_party",
+    "third-party", "thirdparty", "Pods", "Carthage", ".yarn", ".pnpm-store", "dist", "build",
+    "__pycache__", ".venv", "venv", "site-packages",
+})
+GENERATED_NAMES = frozenset({
+    "package-lock.json", "npm-shrinkwrap.json", "yarn.lock", "pnpm-lock.yaml", "bun.lock",
+    "bun.lockb", "deno.lock", "poetry.lock", "Pipfile.lock", "uv.lock", "pdm.lock",
+    "Cargo.lock", "Gemfile.lock", "composer.lock", "go.sum", "mix.lock", "pubspec.lock",
+    "Podfile.lock", "Package.resolved", "packages.lock.json", "flake.lock", "gradle.lockfile",
+    "conan.lock", "paket.lock",
+})
+GENERATED_SUFFIXES = (
+    ".min.js", ".min.css", ".min.mjs", ".min.cjs", ".js.map", ".css.map", ".map",
+    "_pb2.py", "_pb2.pyi", "_pb2_grpc.py", ".pb.go", ".pb.cc", ".pb.h", "_pb.js", "_pb.ts",
+    "_grpc_pb.js", "_grpc_pb.ts", ".g.dart", ".freezed.dart", ".designer.cs",
+)
+_MINIFIED = re.compile(r"\.min\.[A-Za-z0-9]+$")
+_GENERATED_MARK = re.compile(rb"(?:Code generated .{0,200}DO NOT EDIT|@generated\b)")
+BINARY_SUFFIXES = frozenset({
+    ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".ico", ".icns", ".webp", ".avif", ".tif",
+    ".tiff", ".psd", ".pdf", ".zip", ".gz", ".tgz", ".bz2", ".xz", ".zst", ".7z", ".rar",
+    ".tar", ".jar", ".war", ".ear", ".whl", ".egg", ".exe", ".dll", ".so", ".dylib", ".a",
+    ".lib", ".o", ".obj", ".class", ".pyc", ".pyo", ".wasm", ".bin", ".dat", ".db",
+    ".sqlite", ".sqlite3", ".woff", ".woff2", ".ttf", ".otf", ".eot", ".mp3", ".mp4", ".m4a",
+    ".wav", ".ogg", ".flac", ".mov", ".avi", ".mkv", ".webm", ".iso", ".dmg", ".pkg",
+    ".deb", ".rpm", ".apk", ".ipa", ".keystore", ".jks", ".p12", ".pfx", ".der",
+})
+BINARY_SNIFF = 8000
+GENERATED_SNIFF = 1024
+MAX_FILE_BYTES = 10 * 1024 * 1024        # larger files are data, not code: not counted
+
+# Languages by file name, then by extension (case-insensitive); anything else is "Other".
+FILENAMES = {"Dockerfile": "Dockerfile", "Makefile": "Makefile", "makefile": "Makefile",
+             "GNUmakefile": "Makefile", "CMakeLists.txt": "CMake", "Jenkinsfile": "Groovy",
+             "Rakefile": "Ruby", "Gemfile": "Ruby", "Vagrantfile": "Ruby", "BUILD": "Starlark",
+             "WORKSPACE": "Starlark", "BUILD.bazel": "Starlark"}
+EXTENSIONS = {
+    ".py": "Python", ".pyi": "Python", ".pyx": "Cython", ".ipynb": "Jupyter Notebook",
+    ".ts": "TypeScript", ".tsx": "TypeScript", ".mts": "TypeScript", ".cts": "TypeScript",
+    ".js": "JavaScript", ".jsx": "JavaScript", ".mjs": "JavaScript", ".cjs": "JavaScript",
+    ".vue": "Vue", ".svelte": "Svelte", ".astro": "Astro",
+    ".go": "Go", ".rs": "Rust", ".java": "Java", ".kt": "Kotlin", ".kts": "Kotlin",
+    ".scala": "Scala", ".groovy": "Groovy", ".gradle": "Groovy", ".clj": "Clojure",
+    ".c": "C", ".h": "C", ".cc": "C++", ".cpp": "C++", ".cxx": "C++", ".hh": "C++",
+    ".hpp": "C++", ".hxx": "C++", ".m": "Objective-C", ".mm": "Objective-C++",
+    ".cs": "C#", ".fs": "F#", ".vb": "Visual Basic", ".swift": "Swift", ".dart": "Dart",
+    ".rb": "Ruby", ".php": "PHP", ".pl": "Perl", ".pm": "Perl", ".lua": "Lua", ".r": "R",
+    ".jl": "Julia", ".ex": "Elixir", ".exs": "Elixir", ".erl": "Erlang", ".hs": "Haskell",
+    ".ml": "OCaml", ".elm": "Elm", ".zig": "Zig", ".nim": "Nim", ".sol": "Solidity",
+    ".sh": "Shell", ".bash": "Shell", ".zsh": "Shell", ".fish": "Shell", ".ps1": "PowerShell",
+    ".psm1": "PowerShell", ".bat": "Batchfile", ".cmd": "Batchfile",
+    ".sql": "SQL", ".graphql": "GraphQL", ".gql": "GraphQL", ".proto": "Protocol Buffers",
+    ".html": "HTML", ".htm": "HTML", ".css": "CSS", ".scss": "SCSS", ".sass": "Sass",
+    ".less": "Less", ".md": "Markdown", ".mdx": "MDX", ".rst": "reStructuredText",
+    ".tex": "TeX", ".json": "JSON", ".jsonc": "JSON", ".yaml": "YAML", ".yml": "YAML",
+    ".toml": "TOML", ".xml": "XML", ".ini": "INI", ".cfg": "INI", ".tf": "HCL",
+    ".hcl": "HCL", ".nix": "Nix", ".cmake": "CMake", ".txt": "Text",
+}
+
+
+def path_excluded(path: str) -> bool:
+    """Vendored or generated by its path alone (``.gitattributes`` can override)."""
+    p = PurePosixPath(path)
+    if any(part in VENDORED_DIRS for part in p.parts[:-1]):
+        return True
+    name = p.name
+    return (name in GENERATED_NAMES or name.endswith(GENERATED_SUFFIXES)
+            or bool(_MINIFIED.search(name)))
+
+
+def binary_path(path: str) -> bool:
+    return PurePosixPath(path).suffix.lower() in BINARY_SUFFIXES
+
+
+def nonblank(data: bytes) -> int:
+    """Lines (split on LF) with any byte other than ASCII whitespace (space, tab, CR, VT, FF)."""
+    return sum(1 for line in data.split(b"\n") if line.strip())
+
+
+def measure(data: bytes) -> tuple[int, str]:
+    """(non-blank lines, kind): kind is "text", "binary" or "generated" (a marker in the
+    first 1 KiB: Go's "Code generated ... DO NOT EDIT", or @generated)."""
+    if len(data) > MAX_FILE_BYTES or b"\0" in data[:BINARY_SNIFF]:
+        return 0, "binary"
+    if _GENERATED_MARK.search(data[:GENERATED_SNIFF]):
+        return nonblank(data), "generated"
+    return nonblank(data), "text"
+
+
+def language(path: str) -> str:
+    p = PurePosixPath(path)
+    return FILENAMES.get(p.name) or EXTENSIONS.get(p.suffix.lower(), "Other")
+
+
+# -- the result ------------------------------------------------------------------------------
+@dataclass
+class Count:
+    total: int = 0
+    files: int = 0
+    languages: dict[str, int] = field(default_factory=dict)
+
+    def add(self, path: str, lines: int) -> None:
+        self.files += 1
+        self.total += lines
+        name = language(path)
+        self.languages[name] = self.languages.get(name, 0) + lines
+
+    def payload(self) -> dict:
+        """What is sent with the seal (and kept in the Imprint's public header)."""
+        return {"rules": RULES, "total": self.total, "files": self.files,
+                "languages": {k: v for k, v in sorted(self.languages.items()) if v}}
+
+
+def summary(lines: dict) -> str:
+    """'123,456 (Python 80%, TypeScript 20%)': the three largest languages, then the rest."""
+    total = int(lines.get("total") or 0)
+    if not total:
+        return "0"
+    ranked = sorted(((n, k) for k, n in (lines.get("languages") or {}).items() if n),
+                    key=lambda t: (-t[0], t[1]))
+    parts = [f"{k} {round(100 * n / total)}%" for n, k in ranked[:3]
+             if round(100 * n / total) >= 1]
+    rest = total - sum(n for n, _ in ranked[:3])
+    if len(ranked) > 3 and rest:
+        parts.append(f"other {max(1, round(100 * rest / total))}%")
+    return f"{total:,}" + (f" ({', '.join(parts)})" if parts else "")
+
+
+# -- .gitattributes ---------------------------------------------------------------------------
+_ATTRS = ("linguist-vendored", "linguist-generated")
+
+
+def _attr_value(value: str) -> bool | None:
+    """set/true: excluded; unset/false: counted even if the path rules would exclude it."""
+    if value in ("set", "true"):
+        return True
+    if value in ("unset", "false"):
+        return False
+    return None
+
+
+def _git_attrs(top: Path, commit: str | None, paths: list[str]) -> dict[str, dict]:
+    """{path: {attr: True/False}} from ``git check-attr`` (at ``commit`` with Git 2.40+,
+    else from the index)."""
+    if not paths:
+        return {}
+    data = b"\0".join(p.encode("utf-8", "surrogateescape") for p in paths) + b"\0"
+    attempts = ([["--source", commit]] if commit else []) + [["--cached"]]
+    out = None
+    for extra in attempts:
+        try:
+            out = subprocess.run(["git", "-C", str(top), "check-attr", "-z", "--stdin", *extra,
+                                  *_ATTRS], input=data, capture_output=True, check=True).stdout
+            break
+        except subprocess.CalledProcessError:
+            continue
+    found: dict[str, dict] = {}
+    if out is None:
+        return found
+    parts = out.split(b"\0")
+    for i in range(0, len(parts) - 2, 3):
+        value = _attr_value(parts[i + 2].decode())
+        if value is not None:
+            found.setdefault(parts[i].decode("utf-8", "surrogateescape"), {})[
+                parts[i + 1].decode()] = value
+    return found
+
+
+def _folder_attrs(root: Path):
+    """``linguist-vendored`` / ``linguist-generated`` from the folder's own .gitattributes
+    (outside Git: the top-level file only). Returns path -> {attr: bool}."""
+    import pathspec
+
+    rules = []
+    f = root / ".gitattributes"
+    if f.is_file():
+        for line in f.read_text(encoding="utf-8", errors="replace").splitlines():
+            bits = line.strip().split()
+            if not bits or bits[0].startswith("#"):
+                continue
+            attrs = {}
+            for a in bits[1:]:
+                name, _, val = a.lstrip("-!").partition("=")
+                if name in _ATTRS:
+                    attrs[name] = (False if a.startswith("-") or val == "false" else
+                                   None if a.startswith("!") else True)
+            if attrs:
+                rules.append((pathspec.GitIgnoreSpec.from_lines([bits[0]]), attrs))
+
+    def lookup(path: str) -> dict:
+        out: dict = {}
+        for spec, attrs in rules:                     # the last matching line wins
+            if spec.match_file(path):
+                out.update(attrs)
+        return {k: v for k, v in out.items() if v is not None}
+    return lookup
+
+
+def _excluded(path: str, attrs: dict) -> bool:
+    if any(attrs.get(a) is True for a in _ATTRS):
+        return True
+    if any(attrs.get(a) is False for a in _ATTRS):
+        return binary_path(path)                      # explicitly counted, unless binary
+    return binary_path(path) or path_excluded(path)
+
+
+def _counted(path: str, attrs: dict, lines: int, kind: str) -> bool:
+    if kind == "binary":
+        return False
+    return not (kind == "generated" and attrs.get("linguist-generated") is not False)
+
+
+# -- counting a commit ----------------------------------------------------------------------
+def _cache_path(top: Path) -> Path | None:
+    try:
+        out = subprocess.run(["git", "-C", str(top), "rev-parse", "--git-dir"],
+                             capture_output=True, check=True).stdout.decode().strip()
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        return None
+    git_dir = Path(out) if Path(out).is_absolute() else top / out
+    return git_dir / "codeimprint-lines.json"
+
+
+def _load_cache(path: Path | None) -> dict:
+    try:
+        doc = json.loads(path.read_text()) if path else {}
+        return doc.get("blobs", {}) if doc.get("rules") == RULES else {}
+    except (OSError, ValueError, AttributeError):
+        return {}
+
+
+def _save_cache(path: Path | None, blobs: dict) -> None:
+    if not path:
+        return
+    try:
+        tmp = path.with_suffix(".tmp")
+        tmp.write_text(json.dumps({"rules": RULES, "blobs": blobs}, separators=(",", ":")))
+        os.replace(tmp, path)
+    except OSError:
+        pass                                          # a cache only: never fails a seal
+
+
+def _stream_blobs(top: Path, blob_ids: list[str]):
+    """(blob, data) for each ID through one ``git cat-file --batch``, written by a thread so
+    a large repository never waits on a round trip per file."""
+    proc = subprocess.Popen(["git", "-C", str(top), "cat-file", "--batch"],
+                            stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+
+    def feed():
+        try:
+            for blob in blob_ids:
+                proc.stdin.write(blob.encode() + b"\n")
+        finally:
+            proc.stdin.close()
+
+    writer = threading.Thread(target=feed, daemon=True)
+    writer.start()
+    try:
+        for blob in blob_ids:
+            header = proc.stdout.readline().split()
+            if len(header) != 3:
+                raise RuntimeError(f"blob {blob} is missing from the repository")
+            data = proc.stdout.read(int(header[2]))
+            proc.stdout.read(1)
+            yield blob, data
+    finally:
+        writer.join()
+        proc.stdout.close()
+        proc.wait()
+
+
+def count_commit(top: Path, commit: str, files: list[dict], under: str = "") -> Count:
+    """Count the files of a commit (``gitrepo.ls_tree`` entries, paths relative to ``under``,
+    the sealed folder inside the repository)."""
+    full = [under + f["path"] for f in files]
+    attrs = _git_attrs(top, commit, full)
+    candidates = [(f, path) for f, path in zip(files, full)
+                  if not _excluded(path, attrs.get(path, {}))
+                  and int(f.get("size") or 0) <= MAX_FILE_BYTES]
+    cache_file = _cache_path(top)
+    cache = _load_cache(cache_file)
+    wanted = sorted({f["blob"] for f, _ in candidates if f["blob"] not in cache})
+    for blob, data in _stream_blobs(top, wanted) if wanted else ():
+        cache[blob] = list(measure(data))
+    count = Count()
+    for f, path in candidates:
+        lines, kind = cache[f["blob"]]
+        if _counted(path, attrs.get(path, {}), lines, kind):
+            count.add(path, lines)
+    used = {f["blob"] for f in files}
+    _save_cache(cache_file, {b: v for b, v in cache.items() if b in used})
+    return count
+
+
+def count_folder(root: Path, paths: list[str]) -> Count:
+    """Count the files of a folder outside Git (or sealed with --zip): ``paths`` are the
+    folder-relative files the client would send, every type (``project.select_files``)."""
+    lookup = _folder_attrs(root)
+    count = Count()
+    for path in paths:
+        attrs = lookup(path)
+        if _excluded(path, attrs):
+            continue
+        file = root / path
+        try:
+            if file.stat().st_size > MAX_FILE_BYTES:
+                continue
+            lines, kind = measure(file.read_bytes())
+        except OSError:
+            continue
+        if _counted(path, attrs, lines, kind):
+            count.add(path, lines)
+    return count