codetool-shell 0.1.1__py3-none-win_amd64.whl

codetool_shell/filters/generic_log/summary.py

@@ -0,0 +1,161 @@
+"""Line parsing and classification for generic application logs."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class ParsedLogLine:
+    """A log line with a recognized timestamp/level prefix."""
+
+    level: str
+    message: str
+    had_timestamp: bool
+
+
+LOW_SEVERITY_LEVELS = frozenset({"INFO", "DEBUG", "TRACE"})
+KNOWN_LEVELS = frozenset(
+    {
+        "TRACE",
+        "DEBUG",
+        "INFO",
+        "NOTICE",
+        "WARN",
+        "WARNING",
+        "ERROR",
+        "FATAL",
+        "PANIC",
+        "CRITICAL",
+        "ALERT",
+        "EMERG",
+    }
+)
+
+_TIMESTAMP_PREFIX_RE = re.compile(
+    r"^\s*\[?(?P<timestamp>\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}"
+    r"(?:[.,]\d+)?(?:Z|[+-]\d{2}:?\d{2})?)\]?\s+(?P<rest>.+)$"
+)
+_LEVEL_WITH_TIMESTAMP_RE = re.compile(
+    r"^\s*(?:\[(?P<bracket>[A-Za-z]{3,8})\]|"
+    r"(?P<plain>[A-Za-z]{3,8})\b|"
+    r"level=(?P<kv>[A-Za-z]{3,8})\b)\s*[:\]-]?\s*(?P<message>.*)$",
+    re.IGNORECASE,
+)
+_STANDALONE_LEVEL_RE = re.compile(
+    r"^\s*(?:\[(?P<bracket>[A-Za-z]{3,8})\]\s*|"
+    r"(?P<plain>[A-Za-z]{3,8})\s*[:-]\s*|"
+    r"level=(?P<kv>[A-Za-z]{3,8})\b\s*)"
+    r"(?P<message>.*)$",
+    re.IGNORECASE,
+)
+_NUMBER_RE = re.compile(r"\d+(?:\.\d+)?")
+_WHITESPACE_RE = re.compile(r"\s+")
+_EXCEPTION_RE = re.compile(
+    r"\b(?:exception|traceback|stack trace|caused by|panic|segmentation fault)\b",
+    re.IGNORECASE,
+)
+_FAILURE_RE = re.compile(
+    r"\b(?:failed|failure|exit code|exited with|status: failed|fatal error)\b",
+    re.IGNORECASE,
+)
+_URL_RE = re.compile(r"\b[A-Za-z][A-Za-z0-9+.-]*://\S+")
+_FILE_PATH_RE = re.compile(
+    r"(^|[\s=:\"'(\[])"
+    r"(?:\.{0,2}/|/|~/|[A-Za-z]:[\\/]|[A-Za-z0-9_.@+-]+/)"
+    r"[A-Za-z0-9_.@+/\-\\]+"
+    r"(?:\.(?:py|rs|ts|tsx|js|jsx|json|ya?ml|toml|log|txt|html|xml|zip|png|jpg|jpeg|csv|md)"
+    r"|:\d+)"
+)
+
+
+def parse_log_line(line: str) -> ParsedLogLine | None:
+    """Parse a timestamp/level or bracketed-level application log line."""
+
+    stripped = line.strip()
+    if not stripped:
+        return None
+
+    timestamp_match = _TIMESTAMP_PREFIX_RE.match(stripped)
+    if timestamp_match is not None:
+        return _parse_level_prefix(
+            timestamp_match.group("rest"),
+            had_timestamp=True,
+            allow_bare_level=True,
+        )
+
+    return _parse_level_prefix(
+        stripped,
+        had_timestamp=False,
+        allow_bare_level=False,
+    )
+
+
+def is_low_severity(parsed: ParsedLogLine) -> bool:
+    return parsed.level in LOW_SEVERITY_LEVELS
+
+
+def is_important_line(line: str, parsed: ParsedLogLine | None) -> bool:
+    """Return whether a line must be preserved from a generic log."""
+
+    if parsed is not None and parsed.level not in LOW_SEVERITY_LEVELS:
+        return True
+    return (
+        _EXCEPTION_RE.search(line) is not None
+        or _FAILURE_RE.search(line) is not None
+        or _URL_RE.search(line) is not None
+        or _FILE_PATH_RE.search(line) is not None
+    )
+
+
+def normalize_selected_line(line: str) -> str:
+    return line.strip()
+
+
+def normalized_message_pattern(parsed: ParsedLogLine) -> str:
+    """Return a compact pattern key for repeated low-severity messages."""
+
+    message = _NUMBER_RE.sub("<num>", parsed.message.strip())
+    message = _WHITESPACE_RE.sub(" ", message).strip()
+    if not message:
+        message = "<empty message>"
+    if len(message) > 140:
+        message = f"{message[:137]}..."
+    return message
+
+
+def normalize_level(value: str | None) -> str | None:
+    if value is None:
+        return None
+    level = value.upper()
+    if level == "WARNING":
+        return "WARN"
+    if level not in KNOWN_LEVELS:
+        return None
+    return level
+
+
+def _parse_level_prefix(
+    text: str,
+    *,
+    had_timestamp: bool,
+    allow_bare_level: bool,
+) -> ParsedLogLine | None:
+    regex = _LEVEL_WITH_TIMESTAMP_RE if allow_bare_level else _STANDALONE_LEVEL_RE
+    match = regex.match(text)
+    if match is None:
+        return None
+
+    level = normalize_level(
+        match.groupdict().get("bracket")
+        or match.groupdict().get("plain")
+        or match.groupdict().get("kv")
+    )
+    if level is None:
+        return None
+    return ParsedLogLine(
+        level=level,
+        message=match.group("message").strip(),
+        had_timestamp=had_timestamp,
+    )

codetool_shell/filters/git.py

@@ -0,0 +1,514 @@
+"""Compression for common git command output."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from .text import (
+    choose_smaller_lines,
+    indent_lines,
+    join_preserving_final_newline,
+    score,
+    split_preserving_final_newline,
+)
+from .tree import TreeRow, format_tree
+
+
+def compress_git_output(text: str) -> str:
+    """Compress common git command output shapes."""
+
+    current = text
+    for transform in (
+        _compress_git_diff_blocks,
+        _compress_git_stat_blocks,
+        _compress_git_name_status_blocks,
+        _compress_git_long_status_blocks,
+        _compress_git_status_blocks,
+    ):
+        candidate = transform(current)
+        if score(candidate) < score(current):
+            current = candidate
+    return current
+
+
+@dataclass(frozen=True)
+class _GitStatusEntry:
+    code: str
+    path: str
+
+
+_GIT_STATUS_CHARS = frozenset(" MADRCTU!?")
+
+
+def _parse_git_status_line(line: str) -> _GitStatusEntry | None:
+    if len(line) < 4 or line[2] != " ":
+        return None
+    raw_code = line[:2]
+    if raw_code != "??" and (
+        not any(char != " " for char in raw_code)
+        or any(char not in _GIT_STATUS_CHARS for char in raw_code)
+    ):
+        return None
+    path = line[3:]
+    if not path:
+        return None
+    return _GitStatusEntry(raw_code.strip() or raw_code, path)
+
+
+def _is_git_status_block_line(line: str) -> bool:
+    return line.startswith("## ") or _parse_git_status_line(line) is not None
+
+
+def _compress_git_status_blocks(text: str) -> str:
+    lines, final_newline = split_preserving_final_newline(text)
+    output: list[str] = []
+    i = 0
+
+    while i < len(lines):
+        if not _is_git_status_block_line(lines[i]):
+            output.append(lines[i])
+            i += 1
+            continue
+
+        block: list[str] = []
+        while i < len(lines) and _is_git_status_block_line(lines[i]):
+            block.append(lines[i])
+            i += 1
+        output.extend(_compress_git_status_block(block))
+
+    return join_preserving_final_newline(output, final_newline)
+
+
+def _compress_git_status_block(block: list[str]) -> list[str]:
+    branches = [line for line in block if line.startswith("## ")]
+    entries = [
+        parsed
+        for line in block
+        if (parsed := _parse_git_status_line(line)) is not None
+    ]
+    if not entries:
+        return block
+
+    groups: list[tuple[str, list[TreeRow]]] = []
+    for entry in entries:
+        for code, rows in groups:
+            if code == entry.code:
+                rows.append(TreeRow(entry.path))
+                break
+        else:
+            groups.append((entry.code, [TreeRow(entry.path)]))
+
+    summary = "git status: " + " ".join(
+        f"{code}={len(rows)}" for code, rows in groups
+    )
+    candidate: list[str] = [*branches, summary]
+    for code, rows in groups:
+        candidate.append(f"{code}:")
+        candidate.extend(indent_lines(format_tree(rows)))
+
+    return choose_smaller_lines(block, candidate)
+
+
+@dataclass(frozen=True)
+class _GitNameStatusEntry:
+    code: str
+    path: str
+
+
+_GIT_NAME_STATUS_CODE_RE = re.compile(r"^(?:[MADTUXB]|R\d{0,3}|C\d{0,3})$")
+_GIT_PATHLIKE_RE = re.compile(
+    r"[/\\]|^\.\.?/|(?:^|/)\.[^/]+|\.[A-Za-z0-9]{1,12}$|"
+    r"^(?:README|Makefile|Dockerfile|Cargo\.lock|uv\.lock)$"
+)
+
+
+def _looks_git_path_like(value: str) -> bool:
+    clean = value.strip().strip('"')
+    if not clean or len(clean) > 300 or "://" in clean:
+        return False
+    return bool(_GIT_PATHLIKE_RE.search(clean))
+
+
+def _parse_git_name_status_line(line: str) -> _GitNameStatusEntry | None:
+    parts = line.split("\t")
+    if len(parts) < 2:
+        return None
+    code = parts[0]
+    if _GIT_NAME_STATUS_CODE_RE.match(code) is None:
+        return None
+
+    if code.startswith(("R", "C")):
+        if len(parts) != 3 or not all(_looks_git_path_like(path) for path in parts[1:]):
+            return None
+        return _GitNameStatusEntry(code, f"{parts[1]} -> {parts[2]}")
+
+    if len(parts) != 2 or not _looks_git_path_like(parts[1]):
+        return None
+    return _GitNameStatusEntry(code, parts[1])
+
+
+def _compress_git_name_status_blocks(text: str) -> str:
+    lines, final_newline = split_preserving_final_newline(text)
+    output: list[str] = []
+    i = 0
+
+    while i < len(lines):
+        first = _parse_git_name_status_line(lines[i])
+        if first is None:
+            output.append(lines[i])
+            i += 1
+            continue
+
+        block = [lines[i]]
+        entries = [first]
+        i += 1
+        while i < len(lines):
+            parsed = _parse_git_name_status_line(lines[i])
+            if parsed is None:
+                break
+            block.append(lines[i])
+            entries.append(parsed)
+            i += 1
+
+        if len(entries) < 3:
+            output.extend(block)
+            continue
+        output.extend(choose_smaller_lines(block, _compress_git_name_status_block(entries)))
+
+    return join_preserving_final_newline(output, final_newline)
+
+
+def _compress_git_name_status_block(entries: list[_GitNameStatusEntry]) -> list[str]:
+    groups: list[tuple[str, list[str]]] = []
+    for entry in entries:
+        for code, paths in groups:
+            if code == entry.code:
+                paths.append(entry.path)
+                break
+        else:
+            groups.append((entry.code, [entry.path]))
+
+    summary = "git name-status: " + " ".join(
+        f"{code}={len(paths)}" for code, paths in groups
+    )
+    candidate = [summary]
+    for code, paths in groups:
+        candidate.append(f"{code}:")
+        candidate.extend(_format_git_paths(paths))
+    return candidate
+
+
+@dataclass
+class _GitLongStatusSection:
+    title: str
+    entries: list[tuple[str, str]]
+
+
+_GIT_LONG_STATUS_CATEGORY_RE = re.compile(
+    r"^(Changes to be committed|Changes not staged for commit|Unmerged paths|"
+    r"Untracked files|Ignored files):$"
+)
+_GIT_LONG_STATUS_ENTRY_RE = re.compile(
+    r"^(?:(?P<label>[a-z][a-z ]+):\s+)?(?P<path>\S.*)$"
+)
+
+
+def _compress_git_long_status_blocks(text: str) -> str:
+    lines, final_newline = split_preserving_final_newline(text)
+    if not _looks_like_git_long_status(lines):
+        return text
+
+    candidate = _compress_git_long_status_block(lines)
+    chosen = choose_smaller_lines(lines, candidate)
+    return join_preserving_final_newline(chosen, final_newline)
+
+
+def _looks_like_git_long_status(lines: list[str]) -> bool:
+    nonempty = [line for line in lines if line.strip()]
+    if not nonempty:
+        return False
+    has_status_start = nonempty[0].startswith(("On branch ", "HEAD detached "))
+    has_category = any(_GIT_LONG_STATUS_CATEGORY_RE.match(line) for line in nonempty)
+    return has_status_start and has_category
+
+
+def _compress_git_long_status_block(block: list[str]) -> list[str]:
+    preamble: list[str] = []
+    footer: list[str] = []
+    sections: list[_GitLongStatusSection] = []
+    current_section: _GitLongStatusSection | None = None
+
+    for line in block:
+        if not line.strip() or _is_git_long_status_hint(line):
+            continue
+
+        category = _GIT_LONG_STATUS_CATEGORY_RE.match(line)
+        if category is not None:
+            current_section = _GitLongStatusSection(category.group(1), [])
+            sections.append(current_section)
+            continue
+
+        if current_section is not None:
+            entry = _parse_git_long_status_entry(line)
+            if entry is not None:
+                current_section.entries.append(entry)
+                continue
+            footer.append(line.strip())
+            continue
+
+        preamble.append(line.strip())
+
+    if not any(section.entries for section in sections):
+        return block
+
+    candidate = ["git status:", *preamble]
+    for section in sections:
+        if not section.entries:
+            candidate.append(f"{section.title}: 0")
+            continue
+        groups: list[tuple[str, list[str]]] = []
+        for label, path in section.entries:
+            for group_label, paths in groups:
+                if group_label == label:
+                    paths.append(path)
+                    break
+            else:
+                groups.append((label, [path]))
+
+        summary_parts = [
+            f"{'files' if label == 'path' else label}={len(paths)}"
+            for label, paths in groups
+        ]
+        candidate.append(f"{section.title}: {' '.join(summary_parts)}")
+        for label, paths in groups:
+            heading = "files" if label == "path" else label
+            candidate.append(f" {heading}:")
+            candidate.extend(_format_git_paths(paths))
+
+    candidate.extend(footer)
+    return candidate
+
+
+def _is_git_long_status_hint(line: str) -> bool:
+    stripped = line.strip()
+    return stripped.startswith("(") and stripped.endswith(")")
+
+
+def _parse_git_long_status_entry(line: str) -> tuple[str, str] | None:
+    if not line.startswith((" ", "\t")):
+        return None
+    stripped = line.strip()
+    if not stripped or _is_git_long_status_hint(stripped):
+        return None
+    match = _GIT_LONG_STATUS_ENTRY_RE.match(stripped)
+    if match is None:
+        return None
+    label = (match.group("label") or "path").strip()
+    path = match.group("path").strip()
+    if not path:
+        return None
+    return label, path
+
+
+def _format_git_paths(paths: list[str]) -> list[str]:
+    tree_rows: list[TreeRow] = []
+    literal_paths: list[str] = []
+    for path in paths:
+        if " -> " in path or "\t" in path:
+            literal_paths.append(path)
+        else:
+            tree_rows.append(TreeRow(path))
+
+    lines: list[str] = []
+    if tree_rows:
+        lines.extend(indent_lines(format_tree(tree_rows)))
+    lines.extend(f" {path}" for path in literal_paths)
+    return lines
+
+
+def _parse_git_stat_line(line: str) -> TreeRow | None:
+    if "|" not in line:
+        return None
+    path_part, summary_part = line.rsplit("|", 1)
+    path = path_part.strip()
+    summary = summary_part.strip()
+    if not path or not summary:
+        return None
+    if path.startswith(("---", "+++")):
+        return None
+    if not (
+        re.match(r"^(?:\d+|-)\s*(?:[+\-]+)?$", summary)
+        or summary.startswith("Bin ")
+    ):
+        return None
+    return TreeRow(path, f" | {summary}")
+
+
+_GIT_STAT_SUMMARY_RE = re.compile(r"^\s*\d+ files? changed(?:, .*)?$")
+
+
+def _compress_git_stat_blocks(text: str) -> str:
+    lines, final_newline = split_preserving_final_newline(text)
+    output: list[str] = []
+    i = 0
+
+    while i < len(lines):
+        first = _parse_git_stat_line(lines[i])
+        if first is None:
+            output.append(lines[i])
+            i += 1
+            continue
+
+        block = [lines[i]]
+        rows = [first]
+        i += 1
+        while i < len(lines):
+            parsed = _parse_git_stat_line(lines[i])
+            if parsed is None:
+                break
+            block.append(lines[i])
+            rows.append(parsed)
+            i += 1
+
+        summaries: list[str] = []
+        while i < len(lines) and _GIT_STAT_SUMMARY_RE.match(lines[i]):
+            block.append(lines[i])
+            summaries.append(lines[i].strip())
+            i += 1
+
+        candidate = summaries[:] if summaries else [f"git diff --stat: files={len(rows)}"]
+        candidate.extend(format_tree(rows).splitlines())
+        output.extend(choose_smaller_lines(block, candidate))
+
+    return join_preserving_final_newline(output, final_newline)
+
+
+def _compress_git_diff_blocks(text: str) -> str:
+    if "diff --git " not in text:
+        return text
+
+    lines, final_newline = split_preserving_final_newline(text)
+    output: list[str] = []
+    i = 0
+
+    while i < len(lines):
+        if not lines[i].startswith("diff --git "):
+            output.append(lines[i])
+            i += 1
+            continue
+
+        block: list[str] = []
+        while i < len(lines):
+            if block and lines[i].startswith("diff --git "):
+                break
+            block.append(lines[i])
+            i += 1
+
+        candidate = _compress_git_diff_block(block)
+        output.extend(choose_smaller_lines(block, candidate))
+
+    return join_preserving_final_newline(output, final_newline)
+
+
+def _compress_git_diff_block(block: list[str]) -> list[str]:
+    output: list[str] = []
+    omitted_context = 0
+    omitted_binary_payload = 0
+    in_hunk = False
+    in_binary_payload = False
+
+    def flush_context() -> None:
+        nonlocal omitted_context
+        if omitted_context:
+            plural = "" if omitted_context == 1 else "s"
+            output.append(f"… {omitted_context} context line{plural}")
+            omitted_context = 0
+
+    def flush_binary_payload() -> None:
+        nonlocal omitted_binary_payload
+        if omitted_binary_payload:
+            plural = "" if omitted_binary_payload == 1 else "s"
+            output.append(f"… {omitted_binary_payload} binary patch payload line{plural}")
+            omitted_binary_payload = 0
+
+    for line in block:
+        if line.startswith("@@ "):
+            flush_context()
+            flush_binary_payload()
+            output.append(line)
+            in_hunk = True
+            in_binary_payload = False
+            continue
+
+        if line.startswith(("literal ", "delta ")):
+            flush_context()
+            flush_binary_payload()
+            output.append(line)
+            in_binary_payload = True
+            in_hunk = False
+            continue
+
+        if in_binary_payload:
+            if _is_diff_metadata_line(line):
+                flush_binary_payload()
+                output.append(line)
+                continue
+            omitted_binary_payload += 1
+            continue
+
+        if in_hunk and line.startswith(" "):
+            omitted_context += 1
+            continue
+
+        if _is_diff_metadata_line(line):
+            flush_context()
+            flush_binary_payload()
+            output.append(line)
+            in_binary_payload = line.startswith("GIT binary patch")
+            continue
+
+        if in_hunk and (line.startswith("+") or line.startswith("-")):
+            flush_context()
+            output.append(line)
+            continue
+
+        if line.startswith("\\ No newline"):
+            flush_context()
+            output.append(line)
+            continue
+
+        if line.startswith("index "):
+            continue
+
+        flush_context()
+        flush_binary_payload()
+        output.append(line)
+
+    flush_context()
+    flush_binary_payload()
+    return output
+
+
+def _is_diff_metadata_line(line: str) -> bool:
+    return line.startswith(
+        (
+            "diff --git ",
+            "--- ",
+            "+++ ",
+            "new file mode ",
+            "deleted file mode ",
+            "old mode ",
+            "new mode ",
+            "similarity index ",
+            "dissimilarity index ",
+            "rename from ",
+            "rename to ",
+            "copy from ",
+            "copy to ",
+            "Binary files ",
+            "GIT binary patch",
+            "literal ",
+            "delta ",
+        )
+    )

codetool_shell/filters/html_cleanup/__init__.py

@@ -0,0 +1,7 @@
+"""Conservative full-document HTML cleanup."""
+
+from __future__ import annotations
+
+from .reducer import compress_html_cleanup_output
+
+__all__ = ["compress_html_cleanup_output"]