codendium 1.0.0__py3-none-any.whl

copyright_deposit/core/strip/__init__.py

@@ -0,0 +1,337 @@
+"""Comment and docstring removal.
+
+Design
+------
+Strippers never rewrite code. They *classify* byte ranges of the original
+text as comment/docstring, and a single shared routine removes those
+ranges. Removal is therefore provably subtractive: nothing that was not
+classified as a comment can ever be altered.
+
+Dispatch, in order of trustworthiness:
+
+* Python   -> ``tokenize`` + ``ast``. Authoritative, and the result is
+  checked by comparing the ASTs of the original and stripped source.
+* Others   -> pygments, whose token stream is verified to reconstruct the
+  source exactly before it is trusted.
+* Fallback -> a hand-written C-family state machine (also used when no
+  lexer exists or the pygments round-trip fails).
+
+The pygments path deliberately keeps ``Comment.Preproc`` tokens: C lexers
+classify ``#include`` and ``#define`` as comments, and deleting those
+would silently gut the deposit.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+from ...config import LEGAL_HEADER_PATTERN, TransformOptions
+from .. import languages
+from ..languages import FAMILY_C, FAMILY_HASH, FAMILY_PYTHON, Language
+
+KIND_COMMENT = "comment"
+KIND_DOCSTRING = "docstring"
+KIND_SHEBANG = "shebang"
+
+Span = tuple[int, int, str]  # (start offset, end offset, kind)
+
+_LEGAL_RE = re.compile(LEGAL_HEADER_PATTERN, re.IGNORECASE)
+
+
+@dataclass
+class SourceLine:
+    """One physical line of the stripped source."""
+
+    number: int  # 1-based line number in the ORIGINAL file
+    text: str
+
+
+@dataclass
+class StripResult:
+    lines: list[SourceLine] = field(default_factory=list)
+    method: str = "none"
+    removed_lines: int = 0
+    warnings: list[str] = field(default_factory=list)
+    # Every classified comment/docstring span, before the keep-policy
+    # toggles are applied. The third-party scanner reads these so it only
+    # inspects real comments instead of matching prose inside code.
+    comment_spans: list[Span] = field(default_factory=list)
+
+    @property
+    def line_count(self) -> int:
+        return len(self.lines)
+
+
+# ---------------------------------------------------------------------------
+# Offset helpers
+# ---------------------------------------------------------------------------
+
+
+def line_start_offsets(text: str) -> list[int]:
+    """Offset of the first character of each 1-based line (index 0 unused)."""
+    starts = [0, 0]
+    for i, ch in enumerate(text):
+        if ch == "\n":
+            starts.append(i + 1)
+    return starts
+
+
+def rowcol_to_offset(starts: list[int], row: int, col: int) -> int:
+    if row < 1 or row >= len(starts):
+        return len(starts) and starts[-1]
+    return starts[row] + col
+
+
+# ---------------------------------------------------------------------------
+# Span post-processing
+# ---------------------------------------------------------------------------
+
+
+def merge_spans(spans: list[Span]) -> list[Span]:
+    """Sort and coalesce overlapping spans."""
+    if not spans:
+        return []
+    spans = sorted(spans, key=lambda s: (s[0], s[1]))
+    merged: list[Span] = [spans[0]]
+    for start, end, kind in spans[1:]:
+        last_start, last_end, last_kind = merged[-1]
+        if start <= last_end:
+            merged[-1] = (last_start, max(last_end, end), last_kind or kind)
+        else:
+            merged.append((start, end, kind))
+    return merged
+
+
+def _filter_spans(text: str, spans: list[Span], options: TransformOptions) -> list[Span]:
+    """Apply the keep-policy toggles to classified spans."""
+    kept: list[Span] = []
+    for start, end, kind in spans:
+        if kind == KIND_SHEBANG:
+            if options.preserve_shebang:
+                continue
+            if not options.strip_comments:
+                continue
+            kept.append((start, end, kind))
+            continue
+        if kind == KIND_DOCSTRING:
+            if options.strip_docstrings:
+                kept.append((start, end, kind))
+            continue
+        if options.strip_comments:
+            kept.append((start, end, kind))
+    if not options.preserve_legal_headers:
+        return kept
+    return _keep_legal_header(text, kept)
+
+
+def _keep_legal_header(text: str, spans: list[Span]) -> list[Span]:
+    """Drop the leading comment block from the kill list if it reads as legal.
+
+    Only comments appearing before the first line of real code qualify: a
+    licence notice buried mid-file is not the header the Office asks for.
+    """
+    if not spans:
+        return spans
+    result: list[Span] = []
+    for index, (start, end, kind) in enumerate(spans):
+        if kind == KIND_DOCSTRING and index > 0:
+            result.append((start, end, kind))
+            continue
+        before = text[:start]
+        # "Leading" means only whitespace and other comments precede it.
+        stripped_before = before
+        for prev_start, prev_end, _ in spans[:index]:
+            stripped_before = (
+                stripped_before[:prev_start] + " " * (prev_end - prev_start) + stripped_before[prev_end:]
+            )
+        if stripped_before.strip():
+            result.append((start, end, kind))
+            continue
+        if _LEGAL_RE.search(text[start:end]):
+            continue  # preserved
+        result.append((start, end, kind))
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Removal
+# ---------------------------------------------------------------------------
+
+
+def apply_spans(text: str, spans: list[Span]) -> list[SourceLine]:
+    """Remove the spans and rebuild lines, tracking original line numbers.
+
+    Removal joins fragments across a multi-line comment, which is what a
+    reader expects: ``int x = /* note\\n more */ 5;`` becomes ``int x =  5;``.
+    """
+    spans = merge_spans(spans)
+    lines: list[SourceLine] = []
+    buf: list[str] = []
+    orig_line = 1
+    line_started_at = 1
+    span_index = 0
+    i = 0
+    length = len(text)
+
+    while i < length:
+        if span_index < len(spans) and i == spans[span_index][0]:
+            _, end, _ = spans[span_index]
+            # Count newlines swallowed by the comment so numbering stays true.
+            orig_line += text.count("\n", i, end)
+            i = end
+            span_index += 1
+            continue
+        ch = text[i]
+        if ch == "\n":
+            lines.append(SourceLine(line_started_at, "".join(buf)))
+            buf = []
+            orig_line += 1
+            line_started_at = orig_line
+            i += 1
+            continue
+        buf.append(ch)
+        i += 1
+
+    if buf:
+        lines.append(SourceLine(line_started_at, "".join(buf)))
+    return lines
+
+
+def comments_only_lines(text: str, spans: list[Span]) -> list[str]:
+    """The file with every non-comment character blanked out.
+
+    Line numbering is preserved exactly, so a scanner can report a hit at
+    its true location while never seeing a character of actual code.
+    """
+    keep = bytearray(len(text))
+    for start, end, _kind in spans:
+        for i in range(max(0, start), min(len(text), end)):
+            keep[i] = 1
+    out = [
+        ch if (ch == "\n" or keep[i]) else " "
+        for i, ch in enumerate(text)
+    ]
+    return "".join(out).split("\n")
+
+
+def _blank_original_lines(text: str) -> set[int]:
+    return {n for n, raw in enumerate(text.split("\n"), start=1) if not raw.strip()}
+
+
+def postprocess(
+    lines: list[SourceLine],
+    original_text: str,
+    options: TransformOptions,
+) -> list[SourceLine]:
+    """Tabs, trailing whitespace, comment-only line removal, blank collapsing."""
+    originally_blank = _blank_original_lines(original_text)
+
+    staged: list[SourceLine] = []
+    for line in lines:
+        text = line.text
+        if options.expand_tabs:
+            text = text.expandtabs(options.tab_width)
+        if options.trim_trailing_whitespace:
+            text = text.rstrip()
+        if not text.strip() and line.number not in originally_blank:
+            # The line held nothing but a comment; drop it entirely.
+            continue
+        staged.append(SourceLine(line.number, text))
+
+    if options.collapse_blank_runs:
+        collapsed: list[SourceLine] = []
+        run = 0
+        for line in staged:
+            if line.text.strip():
+                run = 0
+                collapsed.append(line)
+                continue
+            run += 1
+            if run <= max(0, options.max_blank_run):
+                collapsed.append(line)
+        staged = collapsed
+
+    while staged and not staged[0].text.strip():
+        staged.pop(0)
+    while staged and not staged[-1].text.strip():
+        staged.pop()
+    return staged
+
+
+# ---------------------------------------------------------------------------
+# Dispatch
+# ---------------------------------------------------------------------------
+
+
+def find_spans(text: str, language: Language) -> tuple[list[Span], str, list[str]]:
+    """Classify comment/docstring ranges. Returns (spans, method, warnings)."""
+    from . import cfamily_strip, pygments_strip, python_strip
+
+    if language.family == FAMILY_PYTHON:
+        spans, warnings = python_strip.find_spans(text)
+        if spans is not None:
+            return spans, "python-tokenize", warnings
+
+    spans, warnings = pygments_strip.find_spans(text, language)
+    if spans is not None:
+        return spans, "pygments", warnings
+
+    if language.family == FAMILY_C:
+        spans, w2 = cfamily_strip.find_spans(text, language)
+        return spans, "c-state-machine", warnings + w2
+
+    if language.family in (FAMILY_HASH, FAMILY_PYTHON) or language.line_comments:
+        spans, w2 = cfamily_strip.find_spans(text, language)
+        return spans, "line-scanner", warnings + w2
+
+    return [], "none", warnings + ["No comment syntax known; source kept verbatim."]
+
+
+def strip_source(
+    text: str,
+    path: str,
+    options: TransformOptions,
+    language: Language | None = None,
+) -> StripResult:
+    """Strip `text` according to `options`, returning renderable lines."""
+    language = language or languages.detect(path)
+    result = StripResult()
+
+    # Spans are always classified, even when nothing will be removed: the
+    # third-party scanner needs to know which regions are comments.
+    all_spans, method, warnings = find_spans(text, language)
+    all_spans = merge_spans(all_spans)
+    result.comment_spans = all_spans
+    result.method = method
+    result.warnings.extend(warnings)
+
+    if not (options.strip_comments or options.strip_docstrings):
+        result.method = "verbatim"
+        raw_lines = [SourceLine(n, t) for n, t in enumerate(text.split("\n"), start=1)]
+        result.lines = postprocess(raw_lines, text, options)
+        return result
+
+    spans = _filter_spans(text, all_spans, options)
+    stripped_lines = apply_spans(text, spans)
+
+    # Safety net: for Python the transform must not change the parse tree.
+    if language.family == FAMILY_PYTHON and spans:
+        from . import python_strip
+
+        rebuilt = "\n".join(line.text for line in stripped_lines)
+        ok, message = python_strip.verify_equivalent(text, rebuilt, options)
+        if not ok:
+            result.warnings.append(
+                f"Comment removal changed the parse tree ({message}); "
+                "the original source was kept for this file."
+            )
+            raw_lines = [SourceLine(n, t) for n, t in enumerate(text.split("\n"), start=1)]
+            result.lines = postprocess(raw_lines, text, options)
+            result.method = "verbatim (verification failed)"
+            return result
+
+    original_count = text.count("\n") + 1
+    result.lines = postprocess(stripped_lines, text, options)
+    result.removed_lines = max(0, original_count - len(result.lines))
+    return result

copyright_deposit/core/strip/cfamily_strip.py

@@ -0,0 +1,235 @@
+"""Hand-written comment scanner used when pygments cannot be trusted.
+
+A single pass over the text tracks whether we are in code, a string, a
+character literal, a raw/verbatim string, or a comment. Only regions
+entered through a comment token are reported, so the scanner can never
+classify code as a comment by accident.
+
+The awkward cases it exists to survive:
+
+* ``"/* not a comment */"`` and ``"// not a comment"`` inside strings
+* C++11 raw strings ``R"tag( */ )tag"``
+* C# verbatim strings ``@"C:\\path"`` with ``""`` escapes
+* Rust ``r#"..."#``, Go and JavaScript backtick strings
+* C line-splicing, where a ``\\`` continues a ``//`` comment onto the next line
+* JavaScript regex literals such as ``/a\\/\\/b/`` that contain ``//``
+"""
+
+from __future__ import annotations
+
+from ..languages import FAMILY_C, Language
+from . import KIND_COMMENT, KIND_SHEBANG, Span
+
+# After one of these, a '/' in JavaScript begins a regex literal rather
+# than a division. This is the standard disambiguation heuristic.
+_REGEX_PRECEDERS = set("(,=:[!&|?{};+-*%^~<>") | {""}
+_REGEX_KEYWORDS = ("return", "typeof", "instanceof", "in", "of", "new", "delete", "case", "do", "else", "yield", "await")
+
+_RAW_PREFIXES = ("u8R", "LR", "uR", "UR", "R")
+
+
+def _line_comment_end(text: str, start: int, splice: bool) -> int:
+    """End offset of a line comment (exclusive of the newline)."""
+    i = start
+    n = len(text)
+    while i < n:
+        ch = text[i]
+        if ch == "\n":
+            if splice and i > start and text[i - 1] == "\\":
+                i += 1
+                continue
+            return i
+        i += 1
+    return n
+
+
+def _skip_quoted(text: str, start: int, quote: str, escapes: bool) -> int:
+    """End offset (exclusive) of a quoted run beginning at `start`."""
+    i = start + 1
+    n = len(text)
+    while i < n:
+        ch = text[i]
+        if escapes and ch == "\\":
+            i += 2
+            continue
+        if ch == quote:
+            return i + 1
+        i += 1
+    return n
+
+
+def _skip_cpp_raw(text: str, quote_pos: int) -> int | None:
+    """Handle R"delim( ... )delim" starting at the quote character."""
+    i = quote_pos + 1
+    n = len(text)
+    delim_start = i
+    while i < n and text[i] not in "( \t\n\\":
+        i += 1
+    if i >= n or text[i] != "(":
+        return None
+    delim = text[delim_start:i]
+    closer = ")" + delim + '"'
+    end = text.find(closer, i + 1)
+    return n if end == -1 else end + len(closer)
+
+
+def _has_raw_prefix(text: str, quote_pos: int) -> bool:
+    for prefix in _RAW_PREFIXES:
+        start = quote_pos - len(prefix)
+        if start < 0 or text[start:quote_pos] != prefix:
+            continue
+        before = text[start - 1] if start > 0 else ""
+        if before.isalnum() or before == "_":
+            continue
+        return True
+    return False
+
+
+def _skip_rust_raw(text: str, r_pos: int) -> int | None:
+    """Handle r"..." and r#"..."# starting at the 'r'."""
+    i = r_pos + 1
+    hashes = 0
+    while i < len(text) and text[i] == "#":
+        hashes += 1
+        i += 1
+    if i >= len(text) or text[i] != '"':
+        return None
+    closer = '"' + "#" * hashes
+    end = text.find(closer, i + 1)
+    return len(text) if end == -1 else end + len(closer)
+
+
+def _skip_regex(text: str, start: int) -> int:
+    """End offset of a JavaScript regex literal beginning at '/'."""
+    i = start + 1
+    n = len(text)
+    in_class = False
+    while i < n:
+        ch = text[i]
+        if ch == "\\":
+            i += 2
+            continue
+        if ch == "\n":
+            return start + 1  # unterminated: treat the '/' as ordinary
+        if ch == "[":
+            in_class = True
+        elif ch == "]":
+            in_class = False
+        elif ch == "/" and not in_class:
+            return i + 1
+        i += 1
+    return start + 1
+
+
+def _regex_allowed(text: str, index: int) -> bool:
+    j = index - 1
+    while j >= 0 and text[j] in " \t\r\n":
+        j -= 1
+    if j < 0:
+        return True
+    ch = text[j]
+    if ch in _REGEX_PRECEDERS:
+        return True
+    if ch.isalnum() or ch == "_":
+        end = j + 1
+        while j >= 0 and (text[j].isalnum() or text[j] == "_"):
+            j -= 1
+        return text[j + 1 : end] in _REGEX_KEYWORDS
+    return False
+
+
+def find_spans(text: str, language: Language) -> tuple[list[Span], list[str]]:
+    alias = language.pygments_alias
+    is_js = alias in {"javascript", "jsx", "typescript", "tsx"}
+    is_cpp = language.raw_strings and alias in {"cpp", "cuda"}
+    is_rust = alias == "rust"
+    is_csharp = alias == "csharp"
+    backticks = language.raw_strings or is_js or alias == "go"
+    splice = language.family == FAMILY_C
+
+    line_tokens = sorted(language.line_comments, key=len, reverse=True)
+    block_tokens = language.block_comments
+
+    spans: list[Span] = []
+    warnings: list[str] = []
+    i = 0
+    n = len(text)
+
+    while i < n:
+        ch = text[i]
+
+        # --- comments -----------------------------------------------------
+        matched = False
+        for open_tok, close_tok in block_tokens:
+            if text.startswith(open_tok, i):
+                end = text.find(close_tok, i + len(open_tok))
+                if end == -1:
+                    end = n
+                    warnings.append("Unterminated block comment; stripped to end of file.")
+                else:
+                    end += len(close_tok)
+                spans.append((i, end, KIND_COMMENT))
+                i = end
+                matched = True
+                break
+        if matched:
+            continue
+
+        for tok in line_tokens:
+            if not text.startswith(tok, i):
+                continue
+            end = _line_comment_end(text, i, splice)
+            kind = KIND_SHEBANG if i == 0 and text.startswith("#!") else KIND_COMMENT
+            spans.append((i, end, kind))
+            i = end
+            matched = True
+            break
+        if matched:
+            continue
+
+        # --- literals that must not be scanned for comment tokens ---------
+        if ch == '"':
+            if is_cpp and _has_raw_prefix(text, i):
+                end = _skip_cpp_raw(text, i)
+                i = end if end is not None else _skip_quoted(text, i, '"', True)
+                continue
+            if is_csharp and i > 0 and text[i - 1] == "@":
+                j = i + 1
+                while j < n:
+                    if text[j] == '"':
+                        if j + 1 < n and text[j + 1] == '"':
+                            j += 2
+                            continue
+                        j += 1
+                        break
+                    j += 1
+                i = j
+                continue
+            if text.startswith('"""', i):  # Java/Kotlin/Swift text blocks
+                end = text.find('"""', i + 3)
+                i = n if end == -1 else end + 3
+                continue
+            i = _skip_quoted(text, i, '"', True)
+            continue
+
+        if ch == "'" and language.char_literals:
+            i = _skip_quoted(text, i, "'", True)
+            continue
+
+        if ch == "`" and backticks:
+            i = _skip_quoted(text, i, "`", alias != "go")
+            continue
+
+        if is_rust and ch == "r":
+            end = _skip_rust_raw(text, i)
+            if end is not None:
+                i = end
+                continue
+
+        if is_js and ch == "/" and _regex_allowed(text, i):
+            i = _skip_regex(text, i)
+            continue
+
+        i += 1
+
+    return spans, warnings

copyright_deposit/core/strip/pygments_strip.py

@@ -0,0 +1,85 @@
+"""Universal comment classification via pygments.
+
+Two safeguards make this trustworthy enough to be the default path for
+non-Python languages:
+
+1. **Round-trip verification.** The concatenated token values must equal
+   the source exactly. If a lexer drops or rewrites anything, we refuse
+   the result and the caller falls back to the state machine.
+2. **Preprocessor tokens are never treated as comments.** pygments maps
+   ``#include`` / ``#define`` to ``Comment.Preproc``; removing those would
+   quietly delete real code from the deposit.
+"""
+
+from __future__ import annotations
+
+from pygments.token import Comment, String, Token
+
+from ..languages import Language
+from . import KIND_COMMENT, KIND_DOCSTRING, KIND_SHEBANG, Span
+
+_PREPROC = (Token.Comment.Preproc, Token.Comment.PreprocFile)
+
+
+def _lexer_for(language: Language):
+    from pygments.lexers import get_lexer_by_name
+
+    if not language.pygments_alias:
+        return None
+    try:
+        return get_lexer_by_name(language.pygments_alias, stripnl=False, ensurenl=False)
+    except Exception:
+        return None
+
+
+def _is_preproc(ttype) -> bool:
+    return any(ttype in p for p in _PREPROC)
+
+
+def _classify(ttype) -> str | None:
+    if ttype in Token.Comment.Hashbang:
+        return KIND_SHEBANG
+    if ttype in String.Doc:
+        return KIND_DOCSTRING
+    if ttype in Comment and not _is_preproc(ttype):
+        return KIND_COMMENT
+    return None
+
+
+def find_spans(text: str, language: Language) -> tuple[list[Span] | None, list[str]]:
+    """Return classified spans, or ``None`` if pygments cannot be trusted."""
+    lexer = _lexer_for(language)
+    if lexer is None:
+        return None, []
+
+    try:
+        tokens = list(lexer.get_tokens_unprocessed(text))
+    except Exception as exc:  # a lexer crash must never abort a build
+        return None, [f"Lexer for {language.name} failed ({exc.__class__.__name__})."]
+
+    # Round-trip check: the token stream must reconstruct the source.
+    rebuilt_length = 0
+    for index, _ttype, value in tokens:
+        if index != rebuilt_length:
+            return None, [
+                f"Lexer output for {language.name} did not reconstruct the source; "
+                "using the fallback comment scanner."
+            ]
+        rebuilt_length += len(value)
+    if rebuilt_length != len(text):
+        return None, [
+            f"Lexer output for {language.name} was truncated; using the fallback scanner."
+        ]
+
+    spans: list[Span] = []
+    for index, ttype, value in tokens:
+        kind = _classify(ttype)
+        if kind is None or not value:
+            continue
+        end = index + len(value)
+        # A trailing newline belongs to the layout, not to the comment.
+        while end > index and text[end - 1] == "\n":
+            end -= 1
+        if end > index:
+            spans.append((index, end, kind))
+    return spans, []