codebatch 0.1.0__py3-none-any.whl

codebatch/tasks/lint.py

@@ -0,0 +1,244 @@
+"""Lint task executor - rule-based diagnostics.
+
+Emits:
+- kind=diagnostic: Lint warnings/errors with severity, code, message, location
+
+Inputs:
+- Parse outputs (kind=ast) via iter_prior_outputs (preferred)
+- Falls back to raw file content for simple text rules
+
+Rules (Phase 2 minimal set):
+- L001: Trailing whitespace
+- L002: Line too long (>120 chars)
+- L003: TODO/FIXME presence
+- L004: Tab indentation (prefer spaces)
+- L005: Missing newline at end of file
+"""
+
+import json
+from typing import Iterable, Optional
+
+from ..runner import ShardRunner
+
+
+# Rule configuration
+DEFAULT_MAX_LINE_LENGTH = 120
+TODO_PATTERNS = ["TODO", "FIXME", "XXX", "HACK"]
+
+
+def lint_trailing_whitespace(lines: list[str], path: str) -> list[dict]:
+    """L001: Detect trailing whitespace."""
+    diagnostics = []
+    for i, line in enumerate(lines, 1):
+        # Don't strip newline, just check for trailing spaces/tabs before it
+        stripped = line.rstrip('\n\r')
+        if stripped != stripped.rstrip():
+            diagnostics.append({
+                "kind": "diagnostic",
+                "path": path,
+                "severity": "warning",
+                "code": "L001",
+                "message": "Trailing whitespace",
+                "line": i,
+                "col": len(stripped.rstrip()) + 1,
+            })
+    return diagnostics
+
+
+def lint_line_too_long(lines: list[str], path: str, max_length: int = DEFAULT_MAX_LINE_LENGTH) -> list[dict]:
+    """L002: Detect lines exceeding max length."""
+    diagnostics = []
+    for i, line in enumerate(lines, 1):
+        stripped = line.rstrip('\n\r')
+        if len(stripped) > max_length:
+            diagnostics.append({
+                "kind": "diagnostic",
+                "path": path,
+                "severity": "warning",
+                "code": "L002",
+                "message": f"Line too long ({len(stripped)} > {max_length})",
+                "line": i,
+                "col": max_length + 1,
+            })
+    return diagnostics
+
+
+def lint_todo_fixme(lines: list[str], path: str) -> list[dict]:
+    """L003: Detect TODO/FIXME/XXX/HACK comments."""
+    diagnostics = []
+    for i, line in enumerate(lines, 1):
+        upper_line = line.upper()
+        for pattern in TODO_PATTERNS:
+            if pattern in upper_line:
+                col = line.upper().find(pattern) + 1
+                diagnostics.append({
+                    "kind": "diagnostic",
+                    "path": path,
+                    "severity": "info",
+                    "code": "L003",
+                    "message": f"Found {pattern} comment",
+                    "line": i,
+                    "col": col,
+                })
+                break  # Only report once per line
+    return diagnostics
+
+
+def lint_tab_indentation(lines: list[str], path: str) -> list[dict]:
+    """L004: Detect tab indentation (prefer spaces)."""
+    diagnostics = []
+    for i, line in enumerate(lines, 1):
+        if line.startswith('\t'):
+            diagnostics.append({
+                "kind": "diagnostic",
+                "path": path,
+                "severity": "warning",
+                "code": "L004",
+                "message": "Tab indentation (prefer spaces)",
+                "line": i,
+                "col": 1,
+            })
+    return diagnostics
+
+
+def lint_missing_final_newline(content: str, path: str) -> list[dict]:
+    """L005: Detect missing newline at end of file."""
+    diagnostics = []
+    if content and not content.endswith('\n'):
+        lines = content.split('\n')
+        diagnostics.append({
+            "kind": "diagnostic",
+            "path": path,
+            "severity": "warning",
+            "code": "L005",
+            "message": "Missing newline at end of file",
+            "line": len(lines),
+            "col": len(lines[-1]) + 1 if lines else 1,
+        })
+    return diagnostics
+
+
+def lint_content(content: str, path: str, config: dict) -> list[dict]:
+    """Run all lint rules on content.
+
+    Args:
+        content: File content as string.
+        path: File path.
+        config: Lint configuration.
+
+    Returns:
+        List of diagnostic records.
+    """
+    diagnostics = []
+    lines = content.split('\n')
+
+    # Get config options
+    max_line_length = config.get("max_line_length", DEFAULT_MAX_LINE_LENGTH)
+    check_trailing = config.get("check_trailing_whitespace", True)
+    check_line_length = config.get("check_line_length", True)
+    check_todo = config.get("check_todo", True)
+    check_tabs = config.get("check_tab_indentation", True)
+    check_final_newline = config.get("check_final_newline", True)
+
+    if check_trailing:
+        diagnostics.extend(lint_trailing_whitespace(lines, path))
+
+    if check_line_length:
+        diagnostics.extend(lint_line_too_long(lines, path, max_line_length))
+
+    if check_todo:
+        diagnostics.extend(lint_todo_fixme(lines, path))
+
+    if check_tabs:
+        diagnostics.extend(lint_tab_indentation(lines, path))
+
+    if check_final_newline:
+        diagnostics.extend(lint_missing_final_newline(content, path))
+
+    return diagnostics
+
+
+def lint_executor(config: dict, files: Iterable[dict], runner: ShardRunner) -> list[dict]:
+    """Execute the lint task.
+
+    Runs lint rules on files in the shard. Prefers AST-based linting but
+    falls back to text-based rules for all files.
+
+    Args:
+        config: Task configuration.
+        files: Iterable of file records for this shard.
+        runner: ShardRunner for CAS access.
+
+    Returns:
+        List of diagnostic output records.
+    """
+    outputs = []
+
+    # Get execution context
+    batch_id = config.get("_batch_id")
+    shard_id = config.get("_shard_id")
+
+    # Track which files we've linted (to avoid duplicates)
+    linted_paths = set()
+
+    # First pass: lint files that have AST (from parse task)
+    if batch_id and shard_id:
+        for ast_output in runner.iter_prior_outputs(batch_id, "01_parse", shard_id, kind="ast"):
+            path = ast_output.get("path")
+            object_ref = ast_output.get("object")
+
+            if not path or not object_ref or path in linted_paths:
+                continue
+
+            # Skip chunked ASTs
+            if ast_output.get("format") == "json+chunks":
+                continue
+
+            try:
+                # Load AST to check for any parse-related issues
+                # (Future: could add AST-based lint rules here)
+
+                # For now, get the original file content and run text rules
+                # We need to find the file's object ref from snapshot
+                pass
+
+            except Exception:
+                pass
+
+    # Second pass: lint all files in shard by reading from CAS
+    file_list = list(files)
+    for file_record in file_list:
+        path = file_record["path"]
+        object_ref = file_record["object"]
+
+        if path in linted_paths:
+            continue
+
+        try:
+            # Get file content from CAS
+            data = runner.object_store.get_bytes(object_ref)
+
+            # Try to decode as text
+            try:
+                content = data.decode("utf-8")
+            except UnicodeDecodeError:
+                # Binary file - skip
+                continue
+
+            # Run lint rules
+            diagnostics = lint_content(content, path, config)
+            outputs.extend(diagnostics)
+            linted_paths.add(path)
+
+        except Exception as e:
+            outputs.append({
+                "kind": "diagnostic",
+                "path": path,
+                "severity": "error",
+                "code": "L999",
+                "message": f"Lint error: {e}",
+                "line": 1,
+                "col": 1,
+            })
+
+    return outputs

codebatch/tasks/parse.py

@@ -0,0 +1,304 @@
+"""Parse task executor - produces AST and diagnostic outputs.
+
+For Phase 1, supports:
+- Python (via ast module)
+- JavaScript/TypeScript (simple tokenization)
+- Text files (line-based tokenization)
+
+Emits:
+- kind=ast: AST objects stored in CAS (format=json or format=json+chunks)
+- kind=diagnostic: Parse errors/warnings
+
+Enforces chunking threshold (default 16MB) with chunk manifest objects.
+"""
+
+import ast
+import json
+import re
+from typing import Iterable, Optional
+
+from ..common import SCHEMA_VERSION, PRODUCER
+from ..runner import ShardRunner
+
+
+# Default chunk size: 16MB
+DEFAULT_CHUNK_SIZE = 16 * 1024 * 1024
+
+
+def parse_python(content: str, path: str) -> tuple[Optional[dict], list[dict]]:
+    """Parse Python source code.
+
+    Args:
+        content: Python source code.
+        path: File path for error reporting.
+
+    Returns:
+        Tuple of (AST dict or None, list of diagnostics).
+    """
+    diagnostics = []
+
+    try:
+        tree = ast.parse(content, filename=path)
+        # Convert AST to dict - summarized mode for reasonable size
+        ast_dict = {
+            "type": "Module",
+            "ast_mode": "summary",  # Explicit about summarization
+            "body": [
+                {
+                    "type": node.__class__.__name__,
+                    "lineno": getattr(node, "lineno", None),
+                    "col_offset": getattr(node, "col_offset", None),
+                }
+                for node in ast.walk(tree)
+                if hasattr(node, "lineno")
+            ][:100],  # Limit for reasonable size
+            "stats": {
+                "total_nodes": len(list(ast.walk(tree))),
+            },
+        }
+        return ast_dict, diagnostics
+
+    except SyntaxError as e:
+        diagnostics.append({
+            "severity": "error",
+            "code": "E0001",
+            "message": str(e.msg) if e.msg else "Syntax error",
+            "line": e.lineno or 1,
+            "column": e.offset or 1,
+        })
+        return None, diagnostics
+
+
+def parse_javascript(content: str, path: str) -> tuple[Optional[dict], list[dict]]:
+    """Simple JavaScript/TypeScript tokenization.
+
+    This is a basic tokenizer, not a full parser.
+    For Phase 1, we just identify tokens and structure.
+
+    Args:
+        content: JS/TS source code.
+        path: File path.
+
+    Returns:
+        Tuple of (token info dict or None, list of diagnostics).
+    """
+    diagnostics = []
+
+    # Simple token patterns
+    patterns = {
+        "keyword": r'\b(function|const|let|var|if|else|for|while|return|class|import|export|async|await)\b',
+        "string": r'(["\'])(?:(?!\1)[^\\]|\\.)*\1',
+        "number": r'\b\d+(?:\.\d+)?\b',
+        "comment": r'//.*|/\*[\s\S]*?\*/',
+        "identifier": r'\b[a-zA-Z_$][a-zA-Z0-9_$]*\b',
+    }
+
+    token_counts = {}
+    for token_type, pattern in patterns.items():
+        matches = re.findall(pattern, content)
+        # Handle tuple returns from capture groups
+        if matches and isinstance(matches[0], tuple):
+            token_counts[token_type] = len(matches)
+        else:
+            token_counts[token_type] = len(matches)
+
+    # Check for common issues
+    # Unbalanced braces
+    open_braces = content.count('{')
+    close_braces = content.count('}')
+    if open_braces != close_braces:
+        diagnostics.append({
+            "severity": "warning",
+            "code": "W0001",
+            "message": f"Unbalanced braces: {open_braces} open, {close_braces} close",
+            "line": 1,
+            "column": 1,
+        })
+
+    ast_dict = {
+        "type": "TokenInfo",
+        "ast_mode": "tokens",
+        "tokens": token_counts,
+        "stats": {
+            "lines": content.count('\n') + 1,
+            "characters": len(content),
+        },
+    }
+
+    return ast_dict, diagnostics
+
+
+def parse_text(content: str, path: str) -> tuple[Optional[dict], list[dict]]:
+    """Simple text file tokenization.
+
+    Args:
+        content: Text content.
+        path: File path.
+
+    Returns:
+        Tuple of (token info dict, empty diagnostics).
+    """
+    lines = content.split('\n')
+    words = content.split()
+
+    ast_dict = {
+        "type": "TextInfo",
+        "ast_mode": "text_stats",
+        "stats": {
+            "lines": len(lines),
+            "words": len(words),
+            "characters": len(content),
+            "non_empty_lines": sum(1 for line in lines if line.strip()),
+        },
+    }
+
+    return ast_dict, []
+
+
+def create_chunk_manifest(
+    data: bytes,
+    kind: str,
+    fmt: str,
+    runner: ShardRunner,
+    chunk_size: int = DEFAULT_CHUNK_SIZE,
+) -> tuple[str, dict]:
+    """Create a chunk manifest for large data.
+
+    Args:
+        data: Raw bytes to chunk.
+        kind: Output kind.
+        fmt: Base format identifier (will be suffixed with +chunks).
+        runner: ShardRunner for CAS access.
+        chunk_size: Target chunk size.
+
+    Returns:
+        Tuple of (manifest object ref, manifest dict).
+    """
+    chunks = []
+    total_bytes = len(data)
+
+    for i in range(0, total_bytes, chunk_size):
+        chunk_data = data[i:i + chunk_size]
+        chunk_ref = runner.object_store.put_bytes(chunk_data)
+        chunks.append({
+            "object": chunk_ref,
+            "size": len(chunk_data),
+            "index": len(chunks),
+        })
+
+    manifest = {
+        "schema_name": "codebatch.chunk_manifest",
+        "schema_version": SCHEMA_VERSION,
+        "producer": PRODUCER,
+        "kind": kind,
+        "format": fmt,
+        "chunks": chunks,
+        "total_bytes": total_bytes,
+        "chunk_size": chunk_size,
+    }
+
+    manifest_bytes = json.dumps(manifest, separators=(",", ":")).encode("utf-8")
+    manifest_ref = runner.object_store.put_bytes(manifest_bytes)
+
+    return manifest_ref, manifest
+
+
+def parse_executor(config: dict, files: Iterable[dict], runner: ShardRunner) -> list[dict]:
+    """Execute the parse task.
+
+    Args:
+        config: Task configuration.
+        files: Iterable of file records for this shard (may be iterator).
+        runner: ShardRunner for CAS access.
+
+    Returns:
+        List of output records.
+    """
+    outputs = []
+    chunk_threshold = config.get("chunk_threshold", DEFAULT_CHUNK_SIZE)
+    emit_ast = config.get("emit_ast", True)
+    emit_diagnostics = config.get("emit_diagnostics", True)
+
+    for file_record in files:
+        path = file_record["path"]
+        object_ref = file_record["object"]
+        lang_hint = file_record.get("lang_hint")
+
+        try:
+            # Get file content from CAS
+            data = runner.object_store.get_bytes(object_ref)
+
+            # Try to decode as text
+            try:
+                content = data.decode("utf-8")
+            except UnicodeDecodeError:
+                # Binary file - skip
+                continue
+
+            # Parse based on language
+            ast_dict = None
+            diagnostics = []
+
+            if lang_hint == "python":
+                ast_dict, diagnostics = parse_python(content, path)
+            elif lang_hint in ("javascript", "typescript"):
+                ast_dict, diagnostics = parse_javascript(content, path)
+            elif lang_hint in ("markdown", "json", "yaml", "xml", "html", "css"):
+                # Text-based formats
+                ast_dict, diagnostics = parse_text(content, path)
+            else:
+                # Default text tokenization for unknown types
+                ast_dict, diagnostics = parse_text(content, path)
+
+            # Emit AST output
+            if emit_ast and ast_dict is not None:
+                ast_bytes = json.dumps(ast_dict, separators=(",", ":")).encode("utf-8")
+
+                if len(ast_bytes) > chunk_threshold:
+                    # Create chunk manifest - kind stays "ast", format becomes "json+chunks"
+                    manifest_ref, _ = create_chunk_manifest(
+                        ast_bytes, "ast", "json", runner, chunk_threshold
+                    )
+                    outputs.append({
+                        "path": path,
+                        "kind": "ast",  # Semantic kind stays ast
+                        "object": manifest_ref,
+                        "format": "json+chunks",  # Format indicates chunking
+                    })
+                else:
+                    # Store directly
+                    ast_ref = runner.object_store.put_bytes(ast_bytes)
+                    outputs.append({
+                        "path": path,
+                        "kind": "ast",
+                        "object": ast_ref,
+                        "format": "json",
+                    })
+
+            # Emit diagnostics
+            if emit_diagnostics:
+                for diag in diagnostics:
+                    outputs.append({
+                        "path": path,
+                        "kind": "diagnostic",
+                        "severity": diag["severity"],
+                        "code": diag["code"],
+                        "message": diag["message"],
+                        "line": diag.get("line"),
+                        "column": diag.get("column"),
+                    })
+
+        except Exception as e:
+            # Emit error diagnostic
+            if emit_diagnostics:
+                outputs.append({
+                    "path": path,
+                    "kind": "diagnostic",
+                    "severity": "error",
+                    "code": "E9999",
+                    "message": f"Parse error: {str(e)}",
+                    "line": 1,
+                    "column": 1,
+                })
+
+    return outputs