cli-usereq 0.68.0__py3-none-any.whl

usereq/compress.py

@@ -0,0 +1,393 @@
+#!/usr/bin/env python3
+"""!
+@file compress.py
+@brief Source code compressor for LLM context optimization.
+@details Parses a source file and removes all comments (inline, single-line, multi-line), blank lines, trailing whitespace, and redundant spacing while preserving language semantics (e.g. Python indentation). Leverages LanguageSpec from source_analyzer to correctly identify comment syntax for each supported language.
+@author GitHub Copilot
+@version 0.0.70
+"""
+
+import os
+import sys
+
+from .source_analyzer import build_language_specs
+
+# Extension-to-language map (mirrors generate_markdown.py)
+EXT_LANG_MAP = {
+    ".py": "python", ".js": "javascript", ".mjs": "javascript", ".ts": "typescript",
+    ".rs": "rust", ".go": "go", ".c": "c", ".cpp": "cpp",
+    ".h": "c", ".hpp": "cpp", ".java": "java", ".rb": "ruby",
+    ".php": "php", ".swift": "swift", ".kt": "kotlin",
+    ".scala": "scala", ".lua": "lua", ".sh": "shell",
+    ".pl": "perl", ".hs": "haskell", ".zig": "zig",
+    ".ex": "elixir", ".cs": "csharp",
+}
+"""! @brief Extension-to-language normalization map for compression input."""
+
+# Languages where indentation is semantically significant
+INDENT_SIGNIFICANT = {"python", "haskell", "elixir"}
+"""! @brief Languages requiring indentation-preserving compression behavior."""
+
+_specs_cache = None
+"""! @brief Cached language specification dictionary initialized lazily."""
+
+
+def _get_specs():
+    """! @brief Return cached language specifications, initializing once.
+    @return Dictionary mapping normalized language keys to language specs.
+    @details If cache is empty, calls `build_language_specs()` to populate it.
+    """
+    global _specs_cache
+    if _specs_cache is None:
+        _specs_cache = build_language_specs()
+    return _specs_cache
+
+
+def detect_language(filepath: str) -> str | None:
+    """! @brief Detect language key from file extension.
+    @param filepath Source file path.
+    @return Normalized language key, or None when extension is unsupported.
+    @details Uses `EXT_LANG_MAP` for lookup. Case-insensitive extension matching.
+    """
+    _, ext = os.path.splitext(filepath)
+    return EXT_LANG_MAP.get(ext.lower())
+
+
+def _is_in_string(line: str, pos: int, string_delimiters: tuple) -> bool:
+    """! @brief Check if position `pos` in `line` is inside a string literal.
+    @param line The code line string.
+    @param pos The character index to check.
+    @param string_delimiters Tuple of string delimiter characters/sequences.
+    @return True if `pos` is inside a string, False otherwise.
+    @details iterates through the line handling escaped delimiters.
+    """
+    in_string = None
+    i = 0
+    sorted_delims = sorted(string_delimiters, key=len, reverse=True)
+    while i < pos:
+        if in_string is not None:
+            if line[i:].startswith(in_string):
+                # Check for escaped delimiter (single-char only)
+                if len(in_string) == 1 and i > 0 and line[i - 1] == '\\':
+                    # Count consecutive backslashes
+                    bs = 0
+                    j = i - 1
+                    while j >= 0 and line[j] == '\\':
+                        bs += 1
+                        j -= 1
+                    if bs % 2 == 1:
+                        i += 1
+                        continue
+                i += len(in_string)
+                in_string = None
+                continue
+            i += 1
+        else:
+            matched = False
+            for delim in sorted_delims:
+                if line[i:].startswith(delim):
+                    in_string = delim
+                    i += len(delim)
+                    matched = True
+                    break
+            if not matched:
+                i += 1
+    return in_string is not None
+
+
+def _remove_inline_comment(line: str, single_comment: str,
+                           string_delimiters: tuple) -> str:
+    """! @brief Remove trailing single-line comment from a code line.
+    @param line The code line string.
+    @param single_comment The single-line comment marker (e.g., "//", "#").
+    @param string_delimiters Tuple of string delimiters to respect.
+    @return The line content before the comment starts.
+    @details Respects string literals; does not remove comments inside strings.
+    """
+    if not single_comment:
+        return line
+    sc_len = len(single_comment)
+    i = 0
+    sorted_delims = sorted(string_delimiters, key=len, reverse=True)
+    in_string = None
+    while i < len(line):
+        if in_string is not None:
+            if line[i:].startswith(in_string):
+                if len(in_string) == 1 and i > 0 and line[i - 1] == '\\':
+                    bs = 0
+                    j = i - 1
+                    while j >= 0 and line[j] == '\\':
+                        bs += 1
+                        j -= 1
+                    if bs % 2 == 1:
+                        i += 1
+                        continue
+                i += len(in_string)
+                in_string = None
+                continue
+            i += 1
+        else:
+            if line[i:i + sc_len] == single_comment:
+                return line[:i]
+            for delim in sorted_delims:
+                if line[i:].startswith(delim):
+                    in_string = delim
+                    i += len(delim)
+                    break
+            else:
+                i += 1
+    return line
+
+
+def _is_python_docstring_line(line: str) -> bool:
+    """!
+    @brief Check if a line is a standalone Python docstring (triple-quote only).
+        @param line The code line string.
+        @return True if the line appears to be a standalone triple-quoted string.
+    @details Implements the _is_python_docstring_line function behavior with deterministic control flow.
+    """
+    stripped = line.strip()
+    for q in ('"""', "'''"):
+        if stripped.startswith(q) and stripped.endswith(q) and len(stripped) >= 6:
+            return True
+    return False
+
+
+def _format_result(entries: list[tuple[int, str]],
+                   include_line_numbers: bool) -> str:
+    """!
+    @brief Format compressed entries, optionally prefixing original line numbers.
+        @param entries List of tuples (line_number, text).
+        @param include_line_numbers Boolean flag to enable line prefixes.
+        @return Formatted string.
+    @details Implements the _format_result function behavior with deterministic control flow.
+    """
+    if not include_line_numbers:
+        return '\n'.join(text for _, text in entries)
+    return '\n'.join(f"{lineno}: {text}" for lineno, text in entries)
+
+
+def compress_source(source: str, language: str,
+                    include_line_numbers: bool = True) -> str:
+    """! @brief Compress source code by removing comments, blank lines, and extra whitespace.
+    @param source The source code string.
+    @param language Language identifier (e.g. "python", "javascript").
+    @param include_line_numbers If True (default), prefix each line with <n>: format.
+    @return Compressed source code string.
+    @throws ValueError If language is unsupported.
+    @details Preserves indentation for indent-significant languages (Python, Haskell, Elixir).
+    """
+    specs = _get_specs()
+    lang_key = language.lower().strip().lstrip(".")
+    if lang_key not in specs:
+        raise ValueError(f"Unsupported language: {language}")
+
+    spec = specs[lang_key]
+    preserve_indent = lang_key in INDENT_SIGNIFICANT
+    lines = source.split('\n')
+    result = []        # list of (original_line_number, text)
+
+    in_multi_comment = False
+    mc_end = spec.multi_comment_end
+    mc_start = spec.multi_comment_start
+    string_delims = spec.string_delimiters
+
+    # Python: also handle ''' as multi-comment
+    is_python = lang_key == "python"
+    in_python_docstring = False
+    python_docstring_delim = None
+
+    i = 0
+    while i < len(lines):
+        line = lines[i]
+
+        # --- Handle multi-line comment continuation ---
+        if in_multi_comment:
+            if mc_end and mc_end in line:
+                # End of multi-line comment found
+                end_pos = line.index(mc_end) + len(mc_end)
+                remainder = line[end_pos:]
+                in_multi_comment = False
+                # Process remainder as a new line
+                if remainder.strip():
+                    lines[i] = remainder
+                    continue
+            i += 1
+            continue
+
+        # --- Python docstrings (""" / ''') used as standalone comments ---
+        if is_python and in_python_docstring:
+            if python_docstring_delim and python_docstring_delim in line:
+                end_pos = line.index(python_docstring_delim) + len(python_docstring_delim)
+                remainder = line[end_pos:]
+                in_python_docstring = False
+                python_docstring_delim = None
+                if remainder.strip():
+                    lines[i] = remainder
+                    continue
+            i += 1
+            continue
+
+        stripped = line.strip()
+
+        # Skip empty lines
+        if not stripped:
+            i += 1
+            continue
+
+        # --- Detect multi-line comment start ---
+        if mc_start:
+            # For Python, triple-quotes can be strings or docstrings.
+            # We only strip standalone docstrings (line starts with triple-quote
+            # after optional whitespace).
+            if is_python:
+                for q in ('"""', "'''"):
+                    if stripped.startswith(q):
+                        # Single-line docstring: """..."""
+                        if stripped.count(q) >= 2 and stripped.endswith(q) and len(stripped) > 3:
+                            # Check it's not a variable assignment like x = """..."""
+                            leading = line[:len(line) - len(line.lstrip())]
+                            code_before = line[:line.index(q)].strip()
+                            if not code_before or code_before.endswith('='):
+                                # Standalone docstring or assigned — skip if standalone
+                                if not code_before:
+                                    line = None
+                                    break
+                            # If code before and not assignment, keep line
+                        elif stripped.startswith(q) and stripped.count(q) == 1:
+                            # Multi-line docstring start
+                            code_before = line[:line.index(q)].strip()
+                            if not code_before:
+                                in_python_docstring = True
+                                python_docstring_delim = q
+                                line = None
+                                break
+                if line is None:
+                    i += 1
+                    continue
+            else:
+                # Non-Python: check for multi-line comment start
+                mc_pos = stripped.find(mc_start)
+                if mc_pos != -1:
+                    # Check if inside a string
+                    full_pos = line.find(mc_start)
+                    if not _is_in_string(line, full_pos, string_delims):
+                        # Check for same-line close
+                        after_start = line[full_pos + len(mc_start):]
+                        close_pos = after_start.find(mc_end) if mc_end else -1
+                        if close_pos != -1 and mc_start != mc_end:
+                            # Single-line block comment: remove it
+                            before = line[:full_pos]
+                            after = after_start[close_pos + len(mc_end):]
+                            line = before + after
+                            if not line.strip():
+                                i += 1
+                                continue
+                            # Re-process this reconstructed line
+                            lines[i] = line
+                            continue
+                        else:
+                            # Multi-line comment starts here
+                            before = line[:full_pos]
+                            in_multi_comment = True
+                            if before.strip():
+                                line = before
+                            else:
+                                i += 1
+                                continue
+
+        # --- Full-line single-line comment ---
+        if spec.single_comment and stripped.startswith(spec.single_comment):
+            # Special: keep shebangs
+            if stripped.startswith('#!') and i == 0:
+                result.append((i + 1, stripped))
+            i += 1
+            continue
+
+        # --- Remove inline comment ---
+        if spec.single_comment:
+            line = _remove_inline_comment(line, spec.single_comment, string_delims)
+
+        # --- Clean whitespace ---
+        if preserve_indent:
+            # Keep leading whitespace, strip trailing
+            leading = line[:len(line) - len(line.lstrip())]
+            content = line.strip()
+            if not content:
+                i += 1
+                continue
+            # Collapse internal multiple spaces (but not in strings)
+            line = leading + content
+        else:
+            line = line.strip()
+            if not line:
+                i += 1
+                continue
+
+        # Remove trailing whitespace
+        line = line.rstrip()
+
+        if line:
+            result.append((i + 1, line))
+
+        i += 1
+
+    return _format_result(result, include_line_numbers)
+
+
+def compress_file(filepath: str, language: str | None = None,
+                  include_line_numbers: bool = True) -> str:
+    """!
+    @brief Compress a source file by removing comments and extra whitespace.
+        @param filepath Path to the source file.
+        @param language Optional language override. Auto-detected if None.
+        @param include_line_numbers If True (default), prefix each line with <n>: format.
+        @return Compressed source code string.
+        @throws ValueError If language cannot be detected.
+    @details Implements the compress_file function behavior with deterministic control flow.
+    """
+    if language is None:
+        language = detect_language(filepath)
+        if language is None:
+            raise ValueError(
+                f"Cannot detect language for '{filepath}'. "
+                "Use --lang to specify explicitly.")
+
+    with open(filepath, 'r', encoding='utf-8', errors='replace') as f:
+        source = f.read()
+
+    return compress_source(source, language, include_line_numbers)
+
+
+def main():
+    """!
+    @brief Execute the standalone compression CLI.
+        @details Parses command-line arguments and invokes `compress_file`, printing the result to stdout or errors to stderr.
+    @return {None} Function return value.
+    """
+    import argparse
+    parser = argparse.ArgumentParser(
+        description="Compress source code for LLM context optimization.")
+    parser.add_argument("file", help="Source file to compress.")
+    parser.add_argument("--lang", default=None,
+                        help="Language override (auto-detected from extension).")
+    parser.add_argument("--enable-line-numbers", action="store_true",
+                        default=False,
+                        help="Enable line number prefixes (<n>:) in output.")
+    args = parser.parse_args()
+
+    if not os.path.isfile(args.file):
+        print(f"Error: file not found: {args.file}", file=sys.stderr)
+        sys.exit(1)
+
+    try:
+        compressed = compress_file(args.file, args.lang,
+                                   args.enable_line_numbers)
+        print(compressed)
+    except ValueError as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

usereq/compress_files.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""!
+@file compress_files.py
+@brief Compress and concatenate multiple source files.
+@details Uses the compress module to strip comments and whitespace from each input file, then concatenates results with a compact header per file for unique identification by an LLM agent.
+@author GitHub Copilot
+@version 0.0.70
+"""
+
+import os
+import sys
+from pathlib import Path
+
+from .compress import compress_file, detect_language
+
+
+def _extract_line_range(compressed_with_line_numbers: str) -> tuple[int, int]:
+    """! @brief Extract source line interval from compressed output with <n>: prefixes.
+    @param compressed_with_line_numbers Compressed payload generated with include_line_numbers=True.
+    @return Tuple (line_start, line_end) derived from preserved <n>: prefixes; returns (0, 0) when no prefixed lines exist.
+    @details Parses the first token of each line as an integer line number.
+    """
+    line_numbers: list[int] = []
+    for line in compressed_with_line_numbers.splitlines():
+        marker, sep, _ = line.partition(":")
+        if sep and marker.isdigit():
+            line_numbers.append(int(marker))
+
+    if not line_numbers:
+        return 0, 0
+
+    return line_numbers[0], line_numbers[-1]
+
+
+def _format_output_path(filepath: str, output_base: Path | None) -> str:
+    """!
+    @brief Build the header-visible path for one compressed source file.
+        @param filepath Absolute or relative source file path.
+        @param output_base Project-home base used to relativize output paths.
+        @return Original filepath when output_base is None; otherwise POSIX relative path from output_base.
+    @details Implements the _format_output_path function behavior with deterministic control flow.
+    """
+    if output_base is None:
+        return filepath
+    absolute_path = Path(filepath).resolve()
+    return Path(os.path.relpath(absolute_path, output_base)).as_posix()
+
+
+def compress_files(filepaths: list[str],
+                   include_line_numbers: bool = True,
+                   verbose: bool = False,
+                   output_base: Path | None = None) -> str:
+    """! @brief Compress multiple source files and concatenate with identifying headers.
+    @param filepaths List of source file paths.
+    @param include_line_numbers If True (default), keep <n>: prefixes in code block lines.
+    @param verbose If True, emits progress status messages on stderr.
+    @param output_base Project-home base used to render header paths as relative paths.
+    @return Concatenated compressed output string.
+    @throws ValueError If no files could be processed.
+    @details Each file is compressed and emitted as: header line `@@@ <path> | <lang>`, line-range metadata `> Lines: <start>-<end>`, and fenced code block delimited by triple backticks. Line range is derived from the already computed <n>: prefixes to preserve existing numbering logic. Files are separated by a blank line.
+    """
+    parts = []
+    ok_count = 0
+    fail_count = 0
+    resolved_output_base = output_base.resolve() if output_base is not None else None
+
+    for fpath in filepaths:
+        if not os.path.isfile(fpath):
+            if verbose:
+                print(f"  SKIP  {fpath} (not found)", file=sys.stderr)
+            continue
+
+        lang = detect_language(fpath)
+        if not lang:
+            if verbose:
+                print(f"  SKIP  {fpath} (unsupported extension)", file=sys.stderr)
+            continue
+
+        try:
+            compressed_with_line_numbers = compress_file(fpath, lang, True)
+            line_start, line_end = _extract_line_range(compressed_with_line_numbers)
+            compressed = (
+                compressed_with_line_numbers
+                if include_line_numbers
+                else compress_file(fpath, lang, False)
+            )
+            output_path = _format_output_path(fpath, resolved_output_base)
+            header = f"@@@ {output_path} | {lang}"
+            parts.append(
+                f"{header}\n> Lines: {line_start}-{line_end}\n```\n{compressed}\n```"
+            )
+            ok_count += 1
+            if verbose:
+                print(f"  OK    {fpath}", file=sys.stderr)
+        except Exception as e:
+            if verbose:
+                print(f"  FAIL  {fpath} ({e})", file=sys.stderr)
+            fail_count += 1
+
+    if not parts:
+        raise ValueError("No valid source files processed")
+
+    if verbose:
+        print(f"\n  Compressed: {ok_count} ok, {fail_count} failed",
+              file=sys.stderr)
+
+    return "\n\n".join(parts)
+
+
+def main():
+    """!
+    @brief Execute the multi-file compression CLI command.
+        @details Parses command-line arguments, calls `compress_files`, and prints output or errors.
+    @return {None} Function return value.
+    """
+    import argparse
+    parser = argparse.ArgumentParser(
+        description="Compress and concatenate source files for LLM context.")
+    parser.add_argument("files", nargs="+", help="Source files to compress.")
+    parser.add_argument("--enable-line-numbers", action="store_true",
+                        default=False,
+                        help="Enable line number prefixes (<n>:) in output.")
+    args = parser.parse_args()
+
+    try:
+        output = compress_files(args.files, args.enable_line_numbers)
+        print(output)
+    except ValueError as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()