codendium 1.0.0__py3-none-any.whl

copyright_deposit/core/render.py

@@ -0,0 +1,242 @@
+"""PDF rendering.
+
+Two properties are load-bearing:
+
+* **Deterministic.** The canvas runs in reportlab's invariant mode with a
+  fixed document date, so the same settings and the same sources produce a
+  byte-identical PDF. A re-run therefore proves what was filed.
+* **Redaction is real.** Blocked-out text is never written to the content
+  stream at all - only a filled rectangle is drawn - so it cannot be
+  recovered with copy/paste or a text extractor.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from .. import DISPLAY_NAME, __version__
+from ..config import HeaderInfo, LayoutOptions
+from .deposit import MODE_HEAD_TAIL, DepositSelection
+from .layout import (
+    KIND_ELISION,
+    KIND_HEADER,
+    KIND_NOTICE,
+    LayoutResult,
+    RenderLine,
+)
+from .metrics import PageGeometry
+
+GUTTER_GRAY = 0.45
+HEADER_GRAY = 0.35
+ELISION_GRAY = 0.40
+
+
+@dataclass
+class RenderReport:
+    output_path: str
+    page_count: int
+    font: str
+    embedded: bool
+
+
+def _canvas(path: str, geometry: PageGeometry, header: HeaderInfo, fingerprint: str):
+    from reportlab.pdfgen import canvas as rl_canvas
+
+    c = rl_canvas.Canvas(
+        path,
+        pagesize=(geometry.width, geometry.height),
+        invariant=1,          # stable document id and creation date
+        pageCompression=1,
+    )
+    c.setTitle(header.title_line())
+    c.setAuthor(header.copyright_owner or "")
+    c.setSubject(header.deposit_label or "Source code deposit")
+    c.setCreator(f"{DISPLAY_NAME} {__version__}")
+    c.setKeywords(f"settings-fingerprint:{fingerprint[:16]}")
+    return c
+
+
+def _draw_text_with_redactions(c, geometry: PageGeometry, line: RenderLine, x0: float, y: float) -> None:
+    font = geometry.font
+    if not line.redactions:
+        c.drawString(x0, y, line.text)
+        return
+
+    char_w = font.char_width
+    bar_bottom = y - font.size * 0.22
+    bar_height = font.size * 1.02
+
+    position = 0
+    for start, end in line.redactions:
+        start = max(0, min(start, len(line.text)))
+        end = max(start, min(end, len(line.text)))
+        if start > position:
+            c.drawString(x0 + position * char_w, y, line.text[position:start])
+        c.saveState()
+        c.setFillGray(0.0)
+        c.rect(x0 + start * char_w, bar_bottom, (end - start) * char_w, bar_height, fill=1, stroke=0)
+        c.restoreState()
+        position = end
+    if position < len(line.text):
+        c.drawString(x0 + position * char_w, y, line.text[position:])
+
+
+def _draw_page(
+    c,
+    geometry: PageGeometry,
+    lines: list[RenderLine],
+    *,
+    printed_number: int,
+    total_label: int,
+    title: str,
+    sheet_label: str = "",
+    show_number: bool = True,
+) -> None:
+    font = geometry.font
+    code_x = geometry.column_x(geometry.gutter_columns)
+
+    if geometry.running_header:
+        c.saveState()
+        c.setFont(font.regular, max(6.5, font.size * 0.72))
+        c.setFillGray(HEADER_GRAY)
+        header_y = geometry.height - geometry.margin - font.size * 0.9
+        c.drawString(geometry.margin, header_y, title[:70])
+        current_file = next((ln.file_path for ln in lines if ln.file_path), "")
+        if current_file:
+            c.drawRightString(geometry.width - geometry.margin, header_y, current_file[-70:])
+        c.restoreState()
+
+    for row, line in enumerate(lines):
+        y = geometry.baseline(row)
+        if not line.text and not line.redactions:
+            continue
+
+        if line.number is not None and geometry.gutter_columns:
+            c.saveState()
+            c.setFont(font.regular, font.size)
+            c.setFillGray(GUTTER_GRAY)
+            c.drawRightString(
+                geometry.column_x(geometry.gutter_columns - 1),
+                y,
+                str(line.number),
+            )
+            c.restoreState()
+
+        c.setFont(font.bold if line.bold else font.regular, font.size)
+        # Elision rules are not source text, so they are set in grey: a
+        # reader can tell at a glance that nothing was written there.
+        c.setFillGray(ELISION_GRAY if line.kind == KIND_ELISION else 0.0)
+        x0 = geometry.margin if line.kind in (KIND_HEADER, KIND_NOTICE) else code_x
+        _draw_text_with_redactions(c, geometry, line, x0, y)
+
+    if geometry.page_numbers:
+        c.saveState()
+        c.setFont(font.regular, max(6.5, font.size * 0.78))
+        c.setFillGray(HEADER_GRAY)
+        footer_y = geometry.margin * 0.55
+        if show_number:
+            c.drawCentredString(
+                geometry.width / 2.0,
+                footer_y,
+                f"Page {printed_number} of {total_label}",
+            )
+        if sheet_label:
+            c.drawRightString(geometry.width - geometry.margin, footer_y, sheet_label)
+        c.restoreState()
+
+    c.showPage()
+
+
+def _separator_lines(text: str, geometry: PageGeometry) -> list[RenderLine]:
+    """Center the omission notice on its own sheet."""
+    from .layout import wrap_source_line
+
+    width = min(geometry.code_columns, 72)
+    chunks = [body for _s, _p, body in wrap_source_line(text, width, "")]
+    top_pad = max(0, (geometry.lines_per_page - len(chunks)) // 2)
+    lines = [RenderLine() for _ in range(top_pad)]
+    lines.extend(RenderLine(text=chunk, kind=KIND_NOTICE, bold=True) for chunk in chunks)
+    return lines
+
+
+def render_pdf(
+    layout: LayoutResult,
+    output_path: str | Path,
+    header: HeaderInfo,
+    options: LayoutOptions,
+    *,
+    selection: DepositSelection | None = None,
+    fingerprint: str = "",
+    include_separator: bool = True,
+) -> RenderReport:
+    """Write `layout` to a PDF, optionally restricted to a deposit selection."""
+    geometry = layout.geometry
+    if geometry is None:
+        raise ValueError("layout has no geometry; build it with build_layout()")
+
+    output_path = str(output_path)
+    Path(output_path).parent.mkdir(parents=True, exist_ok=True)
+
+    pages = layout.pages
+    total_label = len(pages)
+    title = header.title_line()
+
+    if selection is None:
+        chosen = list(range(1, len(pages) + 1))
+        separator_after = None
+        sheets = len(chosen)
+    else:
+        chosen = list(selection.page_numbers)
+        total_label = selection.total_pages
+        sheets = len(chosen)
+        separator_after = None
+        if selection.mode == MODE_HEAD_TAIL and include_separator:
+            separator_after = min(len(chosen), max(0, selection_head_count(selection)))
+
+    c = _canvas(output_path, geometry, header, fingerprint)
+
+    emitted = 0
+    for position, page_number in enumerate(chosen):
+        page = pages[page_number - 1]
+        emitted += 1
+        _draw_page(
+            c,
+            geometry,
+            page.lines,
+            printed_number=page_number,
+            total_label=total_label,
+            title=title,
+            sheet_label=(
+                f"Deposit sheet {emitted} of {sheets}" if selection and selection.mode == MODE_HEAD_TAIL else ""
+            ),
+        )
+        if separator_after is not None and position + 1 == separator_after:
+            notice = selection.separator_notice() if selection else ""
+            if notice:
+                _draw_page(
+                    c,
+                    geometry,
+                    _separator_lines(notice, geometry),
+                    printed_number=page_number,
+                    total_label=total_label,
+                    title=title,
+                    sheet_label="omission notice",
+                    show_number=False,
+                )
+
+    c.save()
+    return RenderReport(
+        output_path=output_path,
+        page_count=emitted,
+        font=geometry.font.regular,
+        embedded=geometry.font.embedded,
+    )
+
+
+def selection_head_count(selection: DepositSelection) -> int:
+    """How many leading pages precede the omission gap."""
+    if selection.omitted is None:
+        return len(selection.page_numbers)
+    gap_start = selection.omitted[0]
+    return sum(1 for n in selection.page_numbers if n < gap_start)

copyright_deposit/core/scanning/__init__.py

@@ -0,0 +1,61 @@
+"""Pre-flight scanners.
+
+A deposit becomes part of a public record that anyone may inspect, and a
+registration may only claim material the applicant actually owns. These
+scanners exist to catch both mistakes before the PDF is filed.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+SEVERITY_HIGH = "high"
+SEVERITY_MEDIUM = "medium"
+SEVERITY_LOW = "low"
+
+_SEVERITY_ORDER = {SEVERITY_HIGH: 0, SEVERITY_MEDIUM: 1, SEVERITY_LOW: 2}
+
+
+@dataclass
+class Finding:
+    id: str
+    rel_path: str
+    line_number: int
+    rule: str
+    detail: str
+    severity: str = SEVERITY_MEDIUM
+    excerpt: str = ""
+    category: str = "secret"
+
+    def location(self) -> str:
+        return f"{self.rel_path}:{self.line_number}"
+
+
+@dataclass
+class ScanReport:
+    secrets: list[Finding] = field(default_factory=list)
+    third_party: list[Finding] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+
+    @property
+    def all_findings(self) -> list[Finding]:
+        return self.secrets + self.third_party
+
+    def blocking(self, ignored: set[str]) -> list[Finding]:
+        return [
+            f
+            for f in self.secrets
+            if f.severity == SEVERITY_HIGH and f.id not in ignored
+        ]
+
+    def sorted_secrets(self) -> list[Finding]:
+        return sorted(
+            self.secrets,
+            key=lambda f: (_SEVERITY_ORDER.get(f.severity, 9), f.rel_path, f.line_number),
+        )
+
+    def sorted_third_party(self) -> list[Finding]:
+        return sorted(
+            self.third_party,
+            key=lambda f: (_SEVERITY_ORDER.get(f.severity, 9), f.rel_path, f.line_number),
+        )

copyright_deposit/core/scanning/secrets.py

@@ -0,0 +1,181 @@
+"""Credential and PII detection.
+
+Anything deposited can be inspected by the public, so a hardcoded key in
+the source is not just a security bug - it is a disclosure. Findings are
+reported with a masked excerpt, and each carries a stable id so the
+operator's decision to ignore or redact it survives into the next run.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import math
+import re
+
+from . import SEVERITY_HIGH, SEVERITY_LOW, SEVERITY_MEDIUM, Finding
+
+# (rule name, pattern, severity, human explanation)
+_RULES: tuple[tuple[str, re.Pattern[str], str, str], ...] = (
+    (
+        "aws-access-key",
+        re.compile(r"\b((?:AKIA|ASIA|ABIA|ACCA)[0-9A-Z]{16})\b"),
+        SEVERITY_HIGH,
+        "AWS access key id",
+    ),
+    (
+        "private-key",
+        re.compile(r"-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----"),
+        SEVERITY_HIGH,
+        "embedded private key block",
+    ),
+    (
+        "github-token",
+        re.compile(r"\b(gh[pousr]_[A-Za-z0-9]{16,})\b"),
+        SEVERITY_HIGH,
+        "GitHub token",
+    ),
+    (
+        "slack-token",
+        re.compile(r"\b(xox[baprs]-[A-Za-z0-9-]{10,})\b"),
+        SEVERITY_HIGH,
+        "Slack token",
+    ),
+    (
+        "google-api-key",
+        re.compile(r"\b(AIza[0-9A-Za-z_\-]{35})\b"),
+        SEVERITY_HIGH,
+        "Google API key",
+    ),
+    (
+        "stripe-key",
+        re.compile(r"\b((?:sk|rk)_(?:live|test)_[0-9A-Za-z]{16,})\b"),
+        SEVERITY_HIGH,
+        "Stripe secret key",
+    ),
+    (
+        "openai-key",
+        re.compile(r"\b(sk-[A-Za-z0-9_\-]{20,})\b"),
+        SEVERITY_HIGH,
+        "OpenAI-style API key",
+    ),
+    (
+        "jwt",
+        re.compile(r"\beyJ[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]{10,}\b"),
+        SEVERITY_MEDIUM,
+        "JSON web token",
+    ),
+    (
+        "connection-string",
+        re.compile(r"\b[a-z][a-z0-9+.\-]*://[^\s:/@]+:[^\s:/@]+@[^\s/]+", re.IGNORECASE),
+        SEVERITY_HIGH,
+        "URL containing credentials",
+    ),
+    (
+        "email",
+        re.compile(r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b"),
+        SEVERITY_LOW,
+        "email address",
+    ),
+)
+
+# name = "value" style assignments worth entropy-checking
+# The prefix is optional so that a name which *is* the keyword - `api_key`,
+# `token` - matches as readily as `service_api_key`.
+_ASSIGNMENT = re.compile(
+    r"""(?ix)
+    \b(?P<name>[A-Za-z0-9_]*
+        (?:pass(?:wd|word)?|secret|token|api[_-]?key|apikey|auth|credential|private[_-]?key)
+        [A-Za-z0-9_]*)
+    \s*[:=]\s*
+    (?P<quote>["'])(?P<value>[^"']{8,})(?P=quote)
+    """
+)
+
+# Values that are obviously not real credentials. The tails accept hyphens
+# and dots so that "your-api-key-here" is recognised as a placeholder.
+_PLACEHOLDER = re.compile(
+    r"(?i)^(?:\s*|x{3,}|\.{3,}|none|null|true|false|changeme|placeholder|redacted|"
+    r"(?:your|my|the|some|example|dummy|test|sample|fake|todo|insert|enter|put)"
+    r"[\w\-. ]*|"
+    r"\$\{.*\}|\{\{.*\}\}|<.*>|%\w+%|\*+)$"
+)
+
+_NON_SECRET_EXT = (".md", ".rst", ".txt")
+
+
+def _shannon_entropy(value: str) -> float:
+    if not value:
+        return 0.0
+    counts: dict[str, int] = {}
+    for ch in value:
+        counts[ch] = counts.get(ch, 0) + 1
+    length = len(value)
+    return -sum((c / length) * math.log2(c / length) for c in counts.values())
+
+
+def _mask(value: str) -> str:
+    if len(value) <= 8:
+        return value[0] + "*" * (len(value) - 1) if value else ""
+    return f"{value[:4]}{'*' * 8}{value[-2:]}"
+
+
+def _finding_id(rel_path: str, line_number: int, rule: str, value: str) -> str:
+    digest = hashlib.sha256(f"{rel_path}|{rule}|{value}".encode("utf-8")).hexdigest()
+    return f"{rule}-{digest[:12]}"
+
+
+def scan_text(rel_path: str, text: str) -> list[Finding]:
+    """Find credentials and PII in one file's original text."""
+    findings: list[Finding] = []
+    seen: set[tuple[int, str]] = set()
+    allow_low = not rel_path.lower().endswith(_NON_SECRET_EXT)
+
+    for number, line in enumerate(text.split("\n"), start=1):
+        if len(line) > 4000:
+            line = line[:4000]
+
+        for rule, pattern, severity, detail in _RULES:
+            if severity == SEVERITY_LOW and not allow_low:
+                continue
+            for match in pattern.finditer(line):
+                value = match.group(1) if match.groups() else match.group(0)
+                if _PLACEHOLDER.match(value):
+                    continue
+                key = (number, rule)
+                if key in seen:
+                    continue
+                seen.add(key)
+                findings.append(
+                    Finding(
+                        id=_finding_id(rel_path, number, rule, value),
+                        rel_path=rel_path,
+                        line_number=number,
+                        rule=rule,
+                        detail=detail,
+                        severity=severity,
+                        excerpt=_mask(value),
+                        category="secret",
+                    )
+                )
+
+        match = _ASSIGNMENT.search(line)
+        if match:
+            value = match.group("value")
+            name = match.group("name")
+            if not _PLACEHOLDER.match(value) and _shannon_entropy(value) >= 3.0:
+                key = (number, "hardcoded-credential")
+                if key not in seen:
+                    seen.add(key)
+                    findings.append(
+                        Finding(
+                            id=_finding_id(rel_path, number, "hardcoded-credential", value),
+                            rel_path=rel_path,
+                            line_number=number,
+                            rule="hardcoded-credential",
+                            detail=f"high-entropy value assigned to '{name}'",
+                            severity=SEVERITY_HIGH,
+                            excerpt=_mask(value),
+                            category="secret",
+                        )
+                    )
+    return findings

copyright_deposit/core/scanning/thirdparty.py

@@ -0,0 +1,190 @@
+"""Detection of code the applicant may not own.
+
+A registration covers the applicant's own authorship. Vendored libraries,
+generated files and headers naming a different copyright holder all need a
+decision - exclude them, or disclaim them in the application - before the
+deposit is filed.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+
+from . import SEVERITY_HIGH, SEVERITY_LOW, SEVERITY_MEDIUM, Finding
+
+HEADER_LINES = 60
+
+# A real notice carries a (c)/copyright symbol or a year. Without that
+# requirement the word "copyright" in ordinary prose - or in a variable
+# name - produces a flood of false positives.
+_COPYRIGHT = re.compile(
+    r"(?i)\bcopyright\b\s*(?P<sym>\(c\)|©|&copy;)?\s*"
+    r"(?P<years>\d{4}(?:\s*[-,]\s*\d{4})*)?\s*"
+    r"(?:by\s+)?(?P<holder>[^\n<]{2,80})?"
+)
+
+# A plausible human or organisation name, not a fragment of source code.
+_HOLDER_OK = re.compile(r"^[A-Za-z][A-Za-z0-9 .,&'’à-ÿ-]{1,60}$")
+_SPDX = re.compile(r"SPDX-License-Identifier:\s*(?P<id>[A-Za-z0-9.\-+ ()]+)")
+
+_LICENSE_PHRASES: tuple[tuple[str, str], ...] = (
+    ("apache license", "Apache License"),
+    ("mit license", "MIT License"),
+    ("bsd license", "BSD License"),
+    ("redistribution and use in source and binary forms", "BSD-style licence text"),
+    ("gnu general public license", "GNU GPL"),
+    ("gnu lesser general public license", "GNU LGPL"),
+    ("mozilla public license", "Mozilla Public License"),
+    ("permission is hereby granted, free of charge", "MIT-style permission notice"),
+)
+
+_GENERATED_PHRASES: tuple[str, ...] = (
+    "do not edit",
+    "do not modify",
+    "@generated",
+    "autogenerated",
+    "auto-generated",
+    "automatically generated",
+    "generated by",
+    "this file was generated",
+    "code generated by",
+)
+
+_VENDOR_SEGMENTS = (
+    "vendor", "third_party", "thirdparty", "external", "extern", "deps",
+    "node_modules", "site-packages", "lib/external",
+)
+
+_CORP_SUFFIXES = re.compile(
+    r"(?i)\b(inc|inc\.|llc|ltd|ltd\.|limited|gmbh|corp|corporation|co|company|"
+    r"foundation|project|team|authors|contributors|and contributors|all rights reserved)\b"
+)
+_NON_WORD = re.compile(r"[^a-z0-9]+")
+
+
+def _normalise_holder(name: str) -> str:
+    name = name.strip().strip(".,;:-* \t")
+    name = _CORP_SUFFIXES.sub(" ", name.lower())
+    return _NON_WORD.sub(" ", name).strip()
+
+
+def _finding_id(rel_path: str, rule: str, detail: str) -> str:
+    digest = hashlib.sha256(f"{rel_path}|{rule}|{detail}".encode("utf-8")).hexdigest()
+    return f"{rule}-{digest[:12]}"
+
+
+def _same_owner(holder: str, owner: str) -> bool:
+    left, right = _normalise_holder(holder), _normalise_holder(owner)
+    if not left or not right:
+        return False
+    return left == right or left in right or right in left
+
+
+def scan_text(
+    rel_path: str,
+    comment_lines: list[str],
+    declared_owner: str = "",
+) -> list[Finding]:
+    """Inspect a file's header comments (and path) for third-party markers.
+
+    `comment_lines` is the file with all non-comment characters blanked out
+    (see ``strip.comments_only_lines``), so line numbers stay accurate while
+    the scanner never reads a character of actual code.
+    """
+    findings: list[Finding] = []
+    lines = list(comment_lines)
+    head = "\n".join(lines[:HEADER_LINES])
+    lowered = head.lower()
+
+    lowered_path = rel_path.lower()
+    for segment in _VENDOR_SEGMENTS:
+        if f"/{segment}/" in f"/{lowered_path}":
+            findings.append(
+                Finding(
+                    id=_finding_id(rel_path, "vendored-path", segment),
+                    rel_path=rel_path,
+                    line_number=1,
+                    rule="vendored-path",
+                    detail=f"path contains '{segment}', which usually holds third-party code",
+                    severity=SEVERITY_MEDIUM,
+                    category="third-party",
+                )
+            )
+            break
+
+    for number, line in enumerate(lines[:HEADER_LINES], start=1):
+        match = _COPYRIGHT.search(line)
+        if not match:
+            continue
+        if not (match.group("sym") or match.group("years")):
+            continue  # the word alone is not a notice
+        holder = (match.group("holder") or "").strip(" .,;:*/-#")
+        holder = re.sub(r"(?i)\s*all rights reserved.*$", "", holder).strip(" .,;:*/-")
+        if not _HOLDER_OK.match(holder):
+            continue
+        if declared_owner and _same_owner(holder, declared_owner):
+            continue
+        findings.append(
+            Finding(
+                id=_finding_id(rel_path, "foreign-copyright", holder),
+                rel_path=rel_path,
+                line_number=number,
+                rule="foreign-copyright",
+                detail=(
+                    f"header names '{holder}'"
+                    + (f", not '{declared_owner}'" if declared_owner else "")
+                ),
+                severity=SEVERITY_HIGH if declared_owner else SEVERITY_MEDIUM,
+                excerpt=line.strip()[:120],
+                category="third-party",
+            )
+        )
+        break
+
+    spdx = _SPDX.search(head)
+    if spdx:
+        identifier = spdx.group("id").strip()
+        findings.append(
+            Finding(
+                id=_finding_id(rel_path, "spdx-license", identifier),
+                rel_path=rel_path,
+                line_number=1,
+                rule="spdx-license",
+                detail=f"declares licence '{identifier}'",
+                severity=SEVERITY_LOW,
+                category="third-party",
+            )
+        )
+    else:
+        for needle, label in _LICENSE_PHRASES:
+            if needle in lowered:
+                findings.append(
+                    Finding(
+                        id=_finding_id(rel_path, "license-text", label),
+                        rel_path=rel_path,
+                        line_number=1,
+                        rule="license-text",
+                        detail=f"header contains {label} text",
+                        severity=SEVERITY_MEDIUM,
+                        category="third-party",
+                    )
+                )
+                break
+
+    for needle in _GENERATED_PHRASES:
+        if needle in lowered:
+            findings.append(
+                Finding(
+                    id=_finding_id(rel_path, "generated-file", needle),
+                    rel_path=rel_path,
+                    line_number=1,
+                    rule="generated-file",
+                    detail=f"header says '{needle}'; generated code is usually not registrable authorship",
+                    severity=SEVERITY_MEDIUM,
+                    category="third-party",
+                )
+            )
+            break
+
+    return findings