codendium 1.0.0__py3-none-any.whl

copyright_deposit/core/deposit.py

@@ -0,0 +1,117 @@
+"""The Compendium section 721.6 page-selection rule.
+
+For a first registration:
+
+* 50 pages or fewer  -> deposit the entire program, and tell the Office the
+  complete code is included.
+* more than 50 pages -> deposit the first 25 and the last 25 pages.
+
+Selection operates on the laid-out pages, so the deposit is assembled from
+the same grid as the full copy and page numbers continue to refer to the
+complete program.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from ..config import DepositOptions
+
+MODE_ENTIRE = "entire"
+MODE_HEAD_TAIL = "head_tail"
+
+
+@dataclass
+class DepositSelection:
+    mode: str
+    total_pages: int
+    page_numbers: list[int] = field(default_factory=list)  # 1-based, in order
+    omitted: tuple[int, int] | None = None  # inclusive range of skipped pages
+
+    @property
+    def deposited_pages(self) -> int:
+        return len(self.page_numbers)
+
+    @property
+    def omitted_count(self) -> int:
+        if self.omitted is None:
+            return 0
+        return self.omitted[1] - self.omitted[0] + 1
+
+    def separator_notice(self) -> str:
+        if self.omitted is None:
+            return ""
+        first, last = self.omitted
+        return (
+            f"[ Pages {first}-{last} intentionally omitted. "
+            f"Pursuant to Compendium (Third) sec. 721.6, this deposit contains "
+            f"the first 25 and the last 25 pages of the program. ]"
+        )
+
+    def filing_statement(self) -> str:
+        """The sentence to put in the application's note to the Office."""
+        if self.mode == MODE_ENTIRE:
+            return (
+                f"The entire source code for this program is {self.total_pages} "
+                "page(s) and is included in full in this deposit."
+            )
+        return (
+            f"The complete program is {self.total_pages} pages. In accordance with "
+            "Compendium (Third) sec. 721.6, this deposit contains the first 25 pages "
+            f"(pages 1-{self.page_numbers[24] if len(self.page_numbers) > 24 else 25}) "
+            f"and the last 25 pages (pages {self.page_numbers[-25]}-{self.total_pages}) "
+            "of the source code."
+        )
+
+
+def select_pages(total_pages: int, options: DepositOptions | None = None) -> DepositSelection:
+    """Apply the 50-page rule to a laid-out document."""
+    options = options or DepositOptions()
+    total = max(0, int(total_pages))
+
+    if not options.apply_rule or total <= options.threshold:
+        return DepositSelection(MODE_ENTIRE, total, list(range(1, total + 1)))
+
+    head = max(0, options.head_pages)
+    tail = max(0, options.tail_pages)
+    if head + tail >= total:
+        return DepositSelection(MODE_ENTIRE, total, list(range(1, total + 1)))
+
+    head_pages = list(range(1, head + 1))
+    tail_pages = list(range(total - tail + 1, total + 1))
+    return DepositSelection(
+        mode=MODE_HEAD_TAIL,
+        total_pages=total,
+        page_numbers=head_pages + tail_pages,
+        omitted=(head + 1, total - tail),
+    )
+
+
+def files_entirely_omitted(selection: DepositSelection, file_ranges) -> list[str]:
+    """Files that fall wholly inside the omitted middle.
+
+    Worth surfacing: this is code the Office will never see, which may
+    change how the operator orders the deposit.
+    """
+    if selection.omitted is None:
+        return []
+    low, high = selection.omitted
+    return [
+        r.rel_path
+        for r in file_ranges
+        if r.first_page >= low and r.last_page <= high
+    ]
+
+
+def files_partially_shown(selection: DepositSelection, file_ranges) -> list[str]:
+    """Files that straddle the omission boundary."""
+    if selection.omitted is None:
+        return []
+    low, high = selection.omitted
+    out = []
+    for r in file_ranges:
+        inside = r.first_page >= low and r.last_page <= high
+        overlaps = r.first_page <= high and r.last_page >= low
+        if overlaps and not inside:
+            out.append(r.rel_path)
+    return out

copyright_deposit/core/discovery.py

@@ -0,0 +1,260 @@
+"""Recursive source discovery.
+
+Every exclusion is recorded rather than silently applied: a deposit is a
+legal filing, so the operator must be able to see exactly what the tool
+decided to leave out and why.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import hashlib
+import os
+import subprocess
+from dataclasses import dataclass, field
+from pathlib import Path, PurePosixPath
+
+from ..config import DiscoveryOptions
+from . import languages
+
+SKIP_EXTENSION = "extension not selected"
+SKIP_IGNORED_DIR = "inside an ignored directory"
+SKIP_IGNORE_GLOB = "matches an ignore pattern"
+SKIP_GITIGNORE = "ignored by .gitignore"
+SKIP_TOO_LARGE = "larger than the size limit"
+SKIP_BINARY = "binary content"
+SKIP_MINIFIED = "appears to be minified or generated"
+SKIP_EMPTY = "empty file"
+SKIP_UNREADABLE = "could not be read"
+
+
+@dataclass
+class DiscoveredFile:
+    rel_path: str  # posix-style, relative to the source root
+    abs_path: str
+    size: int
+    sha256: str
+    language: str
+    family: str
+    raw_line_count: int
+
+    @property
+    def name(self) -> str:
+        return PurePosixPath(self.rel_path).name
+
+
+@dataclass
+class SkippedFile:
+    rel_path: str
+    reason: str
+    size: int = 0
+
+
+@dataclass
+class DiscoveryResult:
+    root: str
+    files: list[DiscoveredFile] = field(default_factory=list)
+    skipped: list[SkippedFile] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+
+    def by_path(self) -> dict[str, DiscoveredFile]:
+        return {f.rel_path: f for f in self.files}
+
+
+# ---------------------------------------------------------------------------
+# .gitignore support
+# ---------------------------------------------------------------------------
+
+
+class _GitIgnoreStack:
+    """Nested .gitignore matching.
+
+    Each .gitignore governs its own directory subtree, so patterns are
+    matched against the path relative to the file that declared them.
+    """
+
+    def __init__(self) -> None:
+        self._specs: list[tuple[str, object]] = []
+
+    def add_dir(self, dir_rel: str, dir_abs: Path) -> None:
+        gi = dir_abs / ".gitignore"
+        if not gi.is_file():
+            return
+        try:
+            import pathspec
+
+            lines = gi.read_text(encoding="utf-8", errors="replace").splitlines()
+            spec = pathspec.PathSpec.from_lines("gitwildmatch", lines)
+        except Exception:
+            return
+        self._specs.append((dir_rel, spec))
+
+    def matches(self, rel_path: str, is_dir: bool = False) -> bool:
+        candidate = rel_path + "/" if is_dir else rel_path
+        for base, spec in self._specs:
+            if base and not candidate.startswith(base + "/"):
+                continue
+            sub = candidate[len(base) + 1 :] if base else candidate
+            if sub and spec.match_file(sub):  # type: ignore[attr-defined]
+                return True
+        return False
+
+
+# ---------------------------------------------------------------------------
+# Discovery
+# ---------------------------------------------------------------------------
+
+
+def _matches_any_glob(name: str, rel_path: str, patterns: list[str]) -> bool:
+    for pattern in patterns:
+        if fnmatch.fnmatch(name, pattern) or fnmatch.fnmatch(rel_path, pattern):
+            return True
+    return False
+
+
+def _inspect(data: bytes) -> tuple[bool, int, float]:
+    """Return (is_binary, line_count, mean_line_length) without decoding."""
+    is_binary = b"\x00" in data[:8192]
+    line_count = data.count(b"\n") + (1 if data and not data.endswith(b"\n") else 0)
+    mean = len(data) / line_count if line_count else float(len(data))
+    return is_binary, line_count, mean
+
+
+def discover(root: str | Path, options: DiscoveryOptions | None = None) -> DiscoveryResult:
+    options = options or DiscoveryOptions()
+    root_path = Path(root).resolve()
+    result = DiscoveryResult(root=str(root_path))
+
+    if not root_path.is_dir():
+        result.warnings.append(f"Source folder does not exist: {root_path}")
+        return result
+
+    ignore_dirs = {d.lower() for d in options.ignore_dirs}
+    gitignore = _GitIgnoreStack() if options.respect_gitignore else None
+
+    for dirpath, dirnames, filenames in os.walk(root_path, followlinks=options.follow_symlinks):
+        current = Path(dirpath)
+        dir_rel = current.relative_to(root_path).as_posix()
+        dir_rel = "" if dir_rel == "." else dir_rel
+
+        if gitignore is not None:
+            gitignore.add_dir(dir_rel, current)
+
+        # Prune in place so os.walk never descends into excluded trees.
+        kept: list[str] = []
+        for d in sorted(dirnames):
+            child_rel = f"{dir_rel}/{d}" if dir_rel else d
+            if d.lower() in ignore_dirs:
+                continue
+            if gitignore is not None and gitignore.matches(child_rel, is_dir=True):
+                continue
+            kept.append(d)
+        dirnames[:] = kept
+
+        for name in sorted(filenames):
+            rel = f"{dir_rel}/{name}" if dir_rel else name
+            abs_path = current / name
+
+            if not languages.is_supported_extension(name, options.extensions):
+                continue  # not a source file; not worth reporting as "skipped"
+
+            if _matches_any_glob(name, rel, options.ignore_globs):
+                result.skipped.append(SkippedFile(rel, SKIP_IGNORE_GLOB))
+                continue
+            if gitignore is not None and gitignore.matches(rel):
+                result.skipped.append(SkippedFile(rel, SKIP_GITIGNORE))
+                continue
+
+            try:
+                size = abs_path.stat().st_size
+            except OSError:
+                result.skipped.append(SkippedFile(rel, SKIP_UNREADABLE))
+                continue
+
+            if options.max_file_bytes and size > options.max_file_bytes:
+                result.skipped.append(SkippedFile(rel, SKIP_TOO_LARGE, size))
+                continue
+            if options.skip_empty and size == 0:
+                result.skipped.append(SkippedFile(rel, SKIP_EMPTY, size))
+                continue
+
+            try:
+                data = abs_path.read_bytes()
+            except OSError:
+                result.skipped.append(SkippedFile(rel, SKIP_UNREADABLE, size))
+                continue
+
+            is_binary, line_count, mean_len = _inspect(data)
+            if is_binary:
+                result.skipped.append(SkippedFile(rel, SKIP_BINARY, size))
+                continue
+            if options.skip_minified and mean_len > 200 and line_count > 0:
+                result.skipped.append(SkippedFile(rel, SKIP_MINIFIED, size))
+                continue
+            if options.skip_empty and not data.strip():
+                result.skipped.append(SkippedFile(rel, SKIP_EMPTY, size))
+                continue
+
+            lang = languages.detect(name)
+            result.files.append(
+                DiscoveredFile(
+                    rel_path=rel,
+                    abs_path=str(abs_path),
+                    size=size,
+                    sha256=hashlib.sha256(data).hexdigest(),
+                    language=lang.name,
+                    family=lang.family,
+                    raw_line_count=line_count,
+                )
+            )
+
+    result.files.sort(key=default_sort_key)
+    result.skipped.sort(key=lambda s: s.rel_path)
+    return result
+
+
+def default_sort_key(f: DiscoveredFile) -> tuple:
+    """Deterministic fallback order: shallower directories first, then path.
+
+    Shallow-first puts entry points and top-level modules near the front,
+    which is usually what a reader expects at the beginning of a program.
+    """
+    parts = PurePosixPath(f.rel_path).parts
+    return (len(parts) - 1, f.rel_path.lower())
+
+
+# ---------------------------------------------------------------------------
+# Git metadata (prefills the identification block)
+# ---------------------------------------------------------------------------
+
+
+def git_revision(root: str | Path) -> str:
+    """Short commit hash for the source tree, or '' if not a git repo."""
+    try:
+        out = subprocess.run(
+            ["git", "-C", str(root), "rev-parse", "--short", "HEAD"],
+            capture_output=True,
+            text=True,
+            timeout=5,
+            check=False,
+        )
+    except (OSError, subprocess.SubprocessError):
+        return ""
+    if out.returncode != 0:
+        return ""
+    rev = out.stdout.strip()
+    if not rev:
+        return ""
+    try:
+        dirty = subprocess.run(
+            ["git", "-C", str(root), "status", "--porcelain"],
+            capture_output=True,
+            text=True,
+            timeout=5,
+            check=False,
+        )
+        if dirty.returncode == 0 and dirty.stdout.strip():
+            rev += " (uncommitted changes present)"
+    except (OSError, subprocess.SubprocessError):
+        pass
+    return rev

copyright_deposit/core/encoding.py

@@ -0,0 +1,136 @@
+"""Reading source files as text, deterministically.
+
+A deposit PDF must be reproducible, so decoding must never depend on the
+machine's locale. The order is: BOM -> UTF-8 -> charset_normalizer ->
+cp1252 -> latin-1 (which cannot fail). Whatever happens, we return text.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+_BOMS: tuple[tuple[bytes, str], ...] = (
+    (b"\xef\xbb\xbf", "utf-8-sig"),
+    (b"\xff\xfe\x00\x00", "utf-32-le"),
+    (b"\x00\x00\xfe\xff", "utf-32-be"),
+    (b"\xff\xfe", "utf-16-le"),
+    (b"\xfe\xff", "utf-16-be"),
+)
+
+# Control characters that would corrupt the fixed line grid. Form feed in
+# particular is a real page-break marker in older C sources.
+_CONTROL_TRANSLATE = {
+    0x00: None, 0x01: None, 0x02: None, 0x03: None, 0x04: None, 0x05: None,
+    0x06: None, 0x07: None, 0x08: None, 0x0B: None, 0x0C: None, 0x0E: None,
+    0x0F: None, 0x10: None, 0x11: None, 0x12: None, 0x13: None, 0x14: None,
+    0x15: None, 0x16: None, 0x17: None, 0x18: None, 0x19: None, 0x1A: None,
+    0x1B: None, 0x1C: None, 0x1D: None, 0x1E: None, 0x1F: None, 0x7F: None,
+}
+
+
+@dataclass
+class DecodedFile:
+    text: str
+    encoding: str
+    had_bom: bool = False
+    line_ending: str = "lf"  # lf | crlf | cr | mixed
+    warnings: list[str] = field(default_factory=list)
+
+
+def detect_line_ending(raw: str) -> str:
+    crlf = raw.count("\r\n")
+    cr = raw.count("\r") - crlf
+    lf = raw.count("\n") - crlf
+    present = [name for name, count in (("crlf", crlf), ("cr", cr), ("lf", lf)) if count]
+    if not present:
+        return "lf"
+    if len(present) > 1:
+        return "mixed"
+    return present[0]
+
+
+def is_probably_binary(data: bytes) -> bool:
+    """NUL byte in the first block is the classic, reliable sniff."""
+    return b"\x00" in data[:8192]
+
+
+def decode_bytes(data: bytes) -> DecodedFile:
+    warnings: list[str] = []
+    encoding = ""
+    had_bom = False
+    text: str | None = None
+
+    for bom, enc in _BOMS:
+        if data.startswith(bom):
+            had_bom = True
+            encoding = enc
+            try:
+                text = data.decode(enc)
+            except UnicodeDecodeError:
+                text = None
+            break
+
+    if text is None and not had_bom:
+        try:
+            text = data.decode("utf-8")
+            encoding = "utf-8"
+        except UnicodeDecodeError:
+            pass
+
+    if text is None:
+        try:
+            from charset_normalizer import from_bytes
+
+            best = from_bytes(data).best()
+            if best is not None:
+                text = str(best)
+                encoding = best.encoding
+                warnings.append(f"Decoded with detected encoding '{encoding}'.")
+        except Exception:  # detector is best-effort, never fatal
+            pass
+
+    if text is None:
+        for enc in ("cp1252", "latin-1"):
+            try:
+                text = data.decode(enc)
+                encoding = enc
+                warnings.append(f"Fell back to '{enc}'; characters may be approximate.")
+                break
+            except UnicodeDecodeError:
+                continue
+
+    if text is None:  # latin-1 cannot fail, but be explicit
+        text = data.decode("latin-1", errors="replace")
+        encoding = "latin-1"
+        warnings.append("Undecodable bytes were replaced.")
+
+    line_ending = detect_line_ending(text)
+    if line_ending == "mixed":
+        warnings.append("Mixed line endings were normalised.")
+
+    # Normalise newlines, then remove control characters that break the grid.
+    text = text.replace("\r\n", "\n").replace("\r", "\n")
+    if "\x0c" in text:
+        warnings.append("Form-feed page breaks were removed.")
+    cleaned = text.translate(_CONTROL_TRANSLATE)
+    if cleaned != text:
+        text = cleaned
+
+    return DecodedFile(
+        text=text,
+        encoding=encoding,
+        had_bom=had_bom,
+        line_ending=line_ending,
+        warnings=warnings,
+    )
+
+
+def read_source(path: str | Path) -> DecodedFile:
+    data = Path(path).read_bytes()
+    return decode_bytes(data)
+
+
+def expand_tabs(text: str, width: int) -> str:
+    """Tab expansion must be per-line: str.expandtabs already is."""
+    return text.expandtabs(width) if width > 0 else text

copyright_deposit/core/languages.py

@@ -0,0 +1,190 @@
+"""Language identification and comment syntax.
+
+Drives three things: which stripper handles a file, how the third-party
+scanner recognises a header comment, and which pygments lexer the
+universal fallback should use.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+# Stripper families.
+FAMILY_PYTHON = "python"
+FAMILY_C = "c"  # C-like: // and /* */ with string/char literals
+FAMILY_HASH = "hash"  # shell-style: # to end of line, no block comments
+FAMILY_GENERIC = "generic"  # anything else -> pygments fallback
+
+
+@dataclass(frozen=True)
+class Language:
+    name: str
+    family: str
+    line_comments: tuple[str, ...] = ()
+    block_comments: tuple[tuple[str, str], ...] = ()
+    pygments_alias: str = ""
+    # C++11 raw strings R"tag(...)tag" and friends need special lexing.
+    raw_strings: bool = False
+    # Languages whose single-quote is a character literal, not a string.
+    char_literals: bool = False
+
+
+_C_LINE = ("//",)
+_C_BLOCK = (("/*", "*/"),)
+
+
+def _c_like(name: str, alias: str, *, raw: bool = False, chars: bool = True) -> Language:
+    return Language(
+        name=name,
+        family=FAMILY_C,
+        line_comments=_C_LINE,
+        block_comments=_C_BLOCK,
+        pygments_alias=alias,
+        raw_strings=raw,
+        char_literals=chars,
+    )
+
+
+def _hash_like(name: str, alias: str) -> Language:
+    return Language(
+        name=name,
+        family=FAMILY_HASH,
+        line_comments=("#",),
+        pygments_alias=alias,
+    )
+
+
+PYTHON = Language(
+    name="Python",
+    family=FAMILY_PYTHON,
+    line_comments=("#",),
+    pygments_alias="python",
+)
+
+# Extension -> Language. Lower-cased keys; lookup lower-cases the suffix.
+EXTENSION_MAP: dict[str, Language] = {
+    ".py": PYTHON,
+    ".pyi": PYTHON,
+    ".pyx": PYTHON,
+    ".pyw": PYTHON,
+    ".c": _c_like("C", "c"),
+    ".h": _c_like("C/C++ Header", "c"),
+    ".cpp": _c_like("C++", "cpp", raw=True),
+    ".cc": _c_like("C++", "cpp", raw=True),
+    ".cxx": _c_like("C++", "cpp", raw=True),
+    ".c++": _c_like("C++", "cpp", raw=True),
+    ".hpp": _c_like("C++ Header", "cpp", raw=True),
+    ".hh": _c_like("C++ Header", "cpp", raw=True),
+    ".hxx": _c_like("C++ Header", "cpp", raw=True),
+    ".inl": _c_like("C++ Inline", "cpp", raw=True),
+    ".java": _c_like("Java", "java"),
+    ".cs": _c_like("C#", "csharp"),
+    ".js": _c_like("JavaScript", "javascript", chars=False),
+    ".jsx": _c_like("JavaScript (JSX)", "jsx", chars=False),
+    ".mjs": _c_like("JavaScript (ESM)", "javascript", chars=False),
+    ".cjs": _c_like("JavaScript (CJS)", "javascript", chars=False),
+    ".ts": _c_like("TypeScript", "typescript", chars=False),
+    ".tsx": _c_like("TypeScript (TSX)", "tsx", chars=False),
+    ".go": _c_like("Go", "go", raw=True),
+    ".rs": _c_like("Rust", "rust", raw=True),
+    ".swift": _c_like("Swift", "swift"),
+    ".kt": _c_like("Kotlin", "kotlin"),
+    ".kts": _c_like("Kotlin Script", "kotlin"),
+    ".m": _c_like("Objective-C", "objective-c"),
+    ".mm": _c_like("Objective-C++", "objective-c++"),
+    ".scala": _c_like("Scala", "scala"),
+    ".dart": _c_like("Dart", "dart"),
+    ".php": _c_like("PHP", "php"),
+    ".glsl": _c_like("GLSL", "glsl"),
+    ".hlsl": _c_like("HLSL", "hlsl"),
+    ".cu": _c_like("CUDA", "cuda", raw=True),
+    ".sh": _hash_like("Shell", "bash"),
+    ".bash": _hash_like("Bash", "bash"),
+    ".zsh": _hash_like("Zsh", "bash"),
+    ".rb": _hash_like("Ruby", "ruby"),
+    ".pl": _hash_like("Perl", "perl"),
+    ".pm": _hash_like("Perl Module", "perl"),
+    ".r": _hash_like("R", "r"),
+    ".jl": _hash_like("Julia", "julia"),
+    ".yaml": _hash_like("YAML", "yaml"),
+    ".yml": _hash_like("YAML", "yaml"),
+    ".toml": _hash_like("TOML", "toml"),
+    ".cmake": _hash_like("CMake", "cmake"),
+    ".ps1": Language(
+        name="PowerShell",
+        family=FAMILY_GENERIC,
+        line_comments=("#",),
+        block_comments=(("<#", "#>"),),
+        pygments_alias="powershell",
+    ),
+    ".sql": Language(
+        name="SQL",
+        family=FAMILY_GENERIC,
+        line_comments=("--",),
+        block_comments=_C_BLOCK,
+        pygments_alias="sql",
+    ),
+    ".lua": Language(
+        name="Lua",
+        family=FAMILY_GENERIC,
+        line_comments=("--",),
+        block_comments=(("--[[", "]]"),),
+        pygments_alias="lua",
+    ),
+    ".vb": Language(
+        name="Visual Basic",
+        family=FAMILY_GENERIC,
+        line_comments=("'",),
+        pygments_alias="vbnet",
+    ),
+    ".f90": Language(
+        name="Fortran",
+        family=FAMILY_GENERIC,
+        line_comments=("!",),
+        pygments_alias="fortran",
+    ),
+    ".html": Language(
+        name="HTML",
+        family=FAMILY_GENERIC,
+        block_comments=(("<!--", "-->"),),
+        pygments_alias="html",
+    ),
+    ".css": Language(
+        name="CSS",
+        family=FAMILY_GENERIC,
+        block_comments=_C_BLOCK,
+        pygments_alias="css",
+    ),
+    ".xml": Language(
+        name="XML",
+        family=FAMILY_GENERIC,
+        block_comments=(("<!--", "-->"),),
+        pygments_alias="xml",
+    ),
+}
+
+# Files without a useful suffix.
+FILENAME_MAP: dict[str, Language] = {
+    "makefile": _hash_like("Makefile", "make"),
+    "dockerfile": _hash_like("Dockerfile", "docker"),
+    "cmakelists.txt": _hash_like("CMake", "cmake"),
+}
+
+UNKNOWN = Language(name="Unknown", family=FAMILY_GENERIC)
+
+
+def detect(path: str | Path) -> Language:
+    """Best-effort language for a path. Never raises."""
+    p = Path(path)
+    by_name = FILENAME_MAP.get(p.name.lower())
+    if by_name is not None:
+        return by_name
+    return EXTENSION_MAP.get(p.suffix.lower(), UNKNOWN)
+
+
+def is_supported_extension(path: str | Path, extensions: list[str]) -> bool:
+    p = Path(path)
+    if p.name.lower() in FILENAME_MAP:
+        return True
+    return p.suffix.lower() in {e.lower() for e in extensions}