codeupipe 0.1.0__py3-none-any.whl

codeupipe/linter/detect_orphans.py

@@ -0,0 +1,106 @@
+"""
+DetectOrphans: Find unreferenced components and stale test files.
+"""
+
+import ast
+from pathlib import Path
+
+from codeupipe import Payload
+
+
+def _extract_imported_names(tree: ast.Module) -> set:
+    """Extract all names imported from relative or absolute imports."""
+    names = set()
+    for node in ast.walk(tree):
+        if isinstance(node, ast.ImportFrom):
+            for alias in (node.names or []):
+                names.add(alias.name)
+        elif isinstance(node, ast.Import):
+            for alias in (node.names or []):
+                names.add(alias.name.split(".")[-1])
+    return names
+
+
+class DetectOrphans:
+    """
+    Filter (sync): Detect orphaned components and orphaned test files.
+
+    Orphaned component = never imported by any other .py file in the directory
+    (excluding __init__.py re-exports and test files).
+
+    Orphaned test = test_*.py file whose stem doesn't match any component.
+
+    Input keys:
+        - components (list[dict]): from ScanComponents
+        - directory (str): component directory path
+        - tests_dir (str, optional): test directory path
+
+    Output keys (added):
+        - orphaned_components (list[dict]): components never imported
+        - orphaned_tests (list[dict]): test files with no matching component
+        - import_map (dict[str, list[str]]): component_name → list of importing files
+    """
+
+    def call(self, payload: Payload) -> Payload:
+        components = payload.get("components", [])
+        directory = payload.get("directory", "")
+        tests_dir_str = payload.get("tests_dir", "tests")
+
+        dir_path = Path(directory)
+        component_names = {c["name"] for c in components}
+        component_stems = {c["stem"] for c in components}
+
+        # Build import map: which files import each component name
+        import_map = {name: [] for name in component_names}
+
+        if dir_path.is_dir():
+            for py_file in sorted(dir_path.glob("*.py")):
+                # Skip __init__.py (re-exports don't count as real usage)
+                if py_file.name == "__init__.py":
+                    continue
+                # Skip test files in the component directory
+                if py_file.name.startswith("test_"):
+                    continue
+
+                try:
+                    source = py_file.read_text()
+                    tree = ast.parse(source, filename=str(py_file))
+                except (SyntaxError, OSError):
+                    continue
+
+                imported = _extract_imported_names(tree)
+                for name in component_names:
+                    if name in imported:
+                        # Don't count self-imports (file that defines the component)
+                        comp = next((c for c in components if c["name"] == name), None)
+                        if comp and Path(comp["file"]).name == py_file.name:
+                            continue
+                        import_map[name].append(py_file.name)
+
+        # Orphaned components: never imported by anyone
+        orphaned_components = []
+        for comp in components:
+            importers = import_map.get(comp["name"], [])
+            if not importers:
+                orphaned_components.append({
+                    "name": comp["name"],
+                    "kind": comp["kind"],
+                    "file": comp["file"],
+                })
+
+        # Orphaned tests: test_*.py files whose stem doesn't match any component
+        orphaned_tests = []
+        tests_dir = Path(tests_dir_str)
+        if tests_dir.is_dir():
+            for test_file in sorted(tests_dir.glob("test_*.py")):
+                stem = test_file.stem[5:]  # strip "test_"
+                if stem not in component_stems:
+                    orphaned_tests.append({
+                        "file": str(test_file),
+                        "stem": stem,
+                    })
+
+        payload = payload.insert("orphaned_components", orphaned_components)
+        payload = payload.insert("orphaned_tests", orphaned_tests)
+        payload = payload.insert("import_map", import_map)
+        return payload

codeupipe/linter/doc_check_pipeline.py

@@ -0,0 +1,28 @@
+"""
+doc_check_pipeline: Wires the doc-code sync check pipeline.
+
+Scans markdown for cup:ref markers, resolves file references,
+checks symbol existence, detects hash drift, validates index
+coverage, and assembles a report.
+"""
+
+from codeupipe import Pipeline
+
+from .scan_docs import ScanDocs
+from .resolve_refs import ResolveRefs
+from .check_symbols import CheckSymbols
+from .detect_drift import DetectDrift
+from .check_index import CheckIndex
+from .assemble_doc_report import AssembleDocReport
+
+
+def build_doc_check_pipeline() -> Pipeline:
+    """Build and return the doc-check pipeline."""
+    pipeline = Pipeline()
+    pipeline.add_filter(ScanDocs(), "scan_docs")
+    pipeline.add_filter(ResolveRefs(), "resolve_refs")
+    pipeline.add_filter(CheckSymbols(), "check_symbols")
+    pipeline.add_filter(DetectDrift(), "detect_drift")
+    pipeline.add_filter(CheckIndex(), "check_index")
+    pipeline.add_filter(AssembleDocReport(), "assemble_doc_report")
+    return pipeline

codeupipe/linter/git_history.py

@@ -0,0 +1,130 @@
+"""
+GitHistory: Retrieve git log data for component and test files.
+"""
+
+import subprocess
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+from codeupipe import Payload
+
+
+def _git_file_info(filepath: str, repo_root: str) -> dict:
+    """Get git log info for a single file.
+
+    Returns dict with last_modified, last_author, commit_count, days_since_change.
+    """
+    try:
+        # Get last commit date + author
+        result = subprocess.run(
+            ["git", "log", "-1", "--format=%aI%n%aN", "--", filepath],
+            cwd=repo_root,
+            capture_output=True,
+            text=True,
+            timeout=10,
+        )
+        lines = result.stdout.strip().split("\n")
+        if result.returncode != 0 or not lines or not lines[0]:
+            return {
+                "last_modified": None,
+                "last_author": None,
+                "commit_count": 0,
+                "days_since_change": None,
+            }
+
+        last_modified_iso = lines[0]
+        last_author = lines[1] if len(lines) > 1 else None
+
+        # Parse date for days_since_change
+        # Python 3.9/3.10 fromisoformat doesn't accept 'Z' suffix
+        if last_modified_iso.endswith("Z"):
+            last_modified_iso = last_modified_iso[:-1] + "+00:00"
+        last_dt = datetime.fromisoformat(last_modified_iso)
+        now = datetime.now(timezone.utc)
+        if last_dt.tzinfo is None:
+            last_dt = last_dt.replace(tzinfo=timezone.utc)
+        days = (now - last_dt).days
+
+        # Get commit count
+        count_result = subprocess.run(
+            ["git", "rev-list", "--count", "HEAD", "--", filepath],
+            cwd=repo_root,
+            capture_output=True,
+            text=True,
+            timeout=10,
+        )
+        commit_count = int(count_result.stdout.strip()) if count_result.returncode == 0 else 0
+
+        # Normalize date to YYYY-MM-DD
+        last_modified = last_dt.strftime("%Y-%m-%d")
+
+        return {
+            "last_modified": last_modified,
+            "last_author": last_author,
+            "commit_count": commit_count,
+            "days_since_change": days,
+        }
+    except (subprocess.TimeoutExpired, OSError, ValueError):
+        return {
+            "last_modified": None,
+            "last_author": None,
+            "commit_count": 0,
+            "days_since_change": None,
+        }
+
+
+def _find_repo_root(directory: str) -> Optional[str]:
+    """Find the git repository root for a directory."""
+    try:
+        result = subprocess.run(
+            ["git", "rev-parse", "--show-toplevel"],
+            cwd=directory,
+            capture_output=True,
+            text=True,
+            timeout=5,
+        )
+        if result.returncode == 0:
+            return result.stdout.strip()
+    except (subprocess.TimeoutExpired, OSError):
+        pass
+    return None
+
+
+class GitHistory:
+    """
+    Filter (sync): Retrieve git history for each component and test file.
+
+    Input keys:
+        - components (list[dict]): from ScanComponents (each has 'file')
+        - test_map (list[dict], optional): from ScanTests (each has 'test_file')
+        - directory (str): component directory path
+
+    Output keys (added):
+        - git_info (dict[str, dict]): filepath → git data
+            Each entry has: last_modified, last_author, commit_count, days_since_change
+    """
+
+    def call(self, payload: Payload) -> Payload:
+        components = payload.get("components", [])
+        test_map = payload.get("test_map", [])
+        directory = payload.get("directory", ".")
+
+        repo_root = _find_repo_root(directory)
+        if repo_root is None:
+            return payload.insert("git_info", {})
+
+        # Collect unique file paths
+        files = set()
+        for comp in components:
+            files.add(comp["file"])
+        for entry in test_map:
+            files.add(entry["test_file"])
+
+        # Query git for each file
+        git_info = {}
+        for filepath in sorted(files):
+            info = _git_file_info(filepath, repo_root)
+            git_info[filepath] = info
+
+        return payload.insert("git_info", git_info)

codeupipe/linter/lint_pipeline.py

@@ -0,0 +1,51 @@
+"""
+LintPipeline: CUP standards linter — built with CUP itself (dogfooding).
+
+Scans a directory and enforces CUP000–CUP008 rules:
+  CUP000: Syntax error in file
+  CUP001: Multiple components in one file
+  CUP002: Missing test file
+  CUP003: Filter missing call()
+  CUP004: Tap missing observe()
+  CUP005: StreamFilter missing stream()
+  CUP006: Hook missing lifecycle methods
+  CUP007: File name not snake_case
+  CUP008: Stale __init__.py bundle
+"""
+
+from codeupipe import Pipeline, Payload
+
+# TODO: update import paths to match your project layout
+from .scan_directory import ScanDirectory
+from .check_naming import CheckNaming
+from .check_structure import CheckStructure
+from .check_protocols import CheckProtocols
+from .check_tests import CheckTests
+from .check_bundle import CheckBundle
+
+
+def build_lint_pipeline() -> Pipeline:
+    """
+    Construct the LintPipeline pipeline.
+
+    Steps:
+        1. ScanDirectory (Filter)
+        2. CheckNaming (Filter)
+        3. CheckStructure (Filter)
+        4. CheckProtocols (Filter)
+        5. CheckTests (Filter)
+        6. CheckBundle (Filter)
+
+    Use pipeline.run(payload) for single-payload execution.
+    Use pipeline.stream(source) for streaming execution.
+    """
+    pipeline = Pipeline()
+
+    pipeline.add_filter(ScanDirectory(), "scan_directory")
+    pipeline.add_filter(CheckNaming(), "check_naming")
+    pipeline.add_filter(CheckStructure(), "check_structure")
+    pipeline.add_filter(CheckProtocols(), "check_protocols")
+    pipeline.add_filter(CheckTests(), "check_tests")
+    pipeline.add_filter(CheckBundle(), "check_bundle")
+
+    return pipeline

codeupipe/linter/map_coverage.py

@@ -0,0 +1,84 @@
+"""
+MapCoverage: Cross-reference components against tests to build coverage map.
+"""
+
+from codeupipe import Payload
+
+
+class MapCoverage:
+    """
+    Filter (sync): Join components and test_map to produce per-component coverage.
+
+    Input keys:
+        - components (list[dict]): from ScanComponents
+        - test_map (list[dict]): from ScanTests
+
+    Output keys (added):
+        - coverage (list[dict]): each with keys:
+            - name (str): component class/function name
+            - kind (str): component type
+            - file (str): source file path
+            - has_test_file (bool): whether a test file exists
+            - test_count (int): number of test_* methods
+            - methods (list[str]): public method names
+            - tested_methods (list[str]): methods referenced in tests
+            - untested_methods (list[str]): methods not referenced in tests
+            - coverage_pct (float): percentage of methods covered (0-100)
+    """
+
+    def call(self, payload: Payload) -> Payload:
+        components = payload.get("components", [])
+        test_map = payload.get("test_map", [])
+
+        # Index test_map by stem for fast lookup
+        test_index = {}
+        for entry in test_map:
+            stem = entry["stem"]
+            if stem not in test_index:
+                test_index[stem] = {
+                    "test_methods": [],
+                    "referenced_methods": set(),
+                    "imports": set(),
+                }
+            test_index[stem]["test_methods"].extend(entry["test_methods"])
+            test_index[stem]["referenced_methods"] |= entry["referenced_methods"]
+            test_index[stem]["imports"] |= entry["imports"]
+
+        coverage = []
+
+        for comp in components:
+            stem = comp["stem"]
+            name = comp["name"]
+            kind = comp["kind"]
+            methods = comp["methods"]
+            test_info = test_index.get(stem)
+
+            has_test = test_info is not None
+            test_count = len(test_info["test_methods"]) if test_info else 0
+            referenced = test_info["referenced_methods"] if test_info else set()
+
+            # For builders (functions), check if the name itself is imported
+            if kind == "builder":
+                tested = [name] if (test_info and name in test_info["imports"]) else []
+                untested = [] if tested else [name]
+                total = 1
+            else:
+                tested = [m for m in methods if m in referenced]
+                untested = [m for m in methods if m not in referenced]
+                total = len(methods)
+
+            pct = (len(tested) / total * 100) if total > 0 else 100.0
+
+            coverage.append({
+                "name": name,
+                "kind": kind,
+                "file": comp["file"],
+                "has_test_file": has_test,
+                "test_count": test_count,
+                "methods": methods if kind != "builder" else [name],
+                "tested_methods": tested,
+                "untested_methods": untested,
+                "coverage_pct": round(pct, 1),
+            })
+
+        return payload.insert("coverage", coverage)

codeupipe/linter/report_gaps.py

@@ -0,0 +1,68 @@
+"""
+ReportGaps: Compute summary statistics and coverage gaps from coverage data.
+"""
+
+from codeupipe import Payload
+
+
+class ReportGaps:
+    """
+    Filter (sync): Produce a human-readable summary from coverage data.
+
+    Input keys:
+        - coverage (list[dict]): from MapCoverage
+
+    Output keys (added):
+        - summary (dict): with keys:
+            - total_components (int)
+            - tested_components (int): components with at least one test
+            - untested_components (int): components with zero tests
+            - total_methods (int)
+            - tested_methods (int)
+            - untested_methods (int)
+            - overall_pct (float): aggregate method coverage %
+        - gaps (list[dict]): components with coverage < 100%, each with:
+            - name (str)
+            - kind (str)
+            - file (str)
+            - coverage_pct (float)
+            - missing (list[str]): untested method names
+    """
+
+    def call(self, payload: Payload) -> Payload:
+        coverage = payload.get("coverage", [])
+
+        total_components = len(coverage)
+        tested_components = sum(1 for c in coverage if c["has_test_file"])
+        untested_components = total_components - tested_components
+
+        total_methods = sum(len(c["methods"]) for c in coverage)
+        tested_methods = sum(len(c["tested_methods"]) for c in coverage)
+        untested_methods = total_methods - tested_methods
+
+        overall_pct = (tested_methods / total_methods * 100) if total_methods > 0 else 100.0
+
+        summary = {
+            "total_components": total_components,
+            "tested_components": tested_components,
+            "untested_components": untested_components,
+            "total_methods": total_methods,
+            "tested_methods": tested_methods,
+            "untested_methods": untested_methods,
+            "overall_pct": round(overall_pct, 1),
+        }
+
+        gaps = []
+        for c in coverage:
+            if c["coverage_pct"] < 100.0:
+                gaps.append({
+                    "name": c["name"],
+                    "kind": c["kind"],
+                    "file": c["file"],
+                    "coverage_pct": c["coverage_pct"],
+                    "missing": c["untested_methods"],
+                })
+
+        payload = payload.insert("summary", summary)
+        payload = payload.insert("gaps", gaps)
+        return payload

codeupipe/linter/report_pipeline.py

@@ -0,0 +1,49 @@
+"""
+ReportPipeline: Full codebase health report — built with CUP.
+
+Composes coverage analysis, orphan detection, git history, and
+report assembly into a single pipeline.
+
+Reuses: ScanComponents, ScanTests, MapCoverage, ReportGaps
+New:    DetectOrphans, GitHistory, AssembleReport
+"""
+
+from codeupipe import Pipeline
+
+from .scan_components import ScanComponents
+from .scan_tests import ScanTests
+from .map_coverage import MapCoverage
+from .report_gaps import ReportGaps
+from .detect_orphans import DetectOrphans
+from .git_history import GitHistory
+from .assemble_report import AssembleReport
+
+
+def build_report_pipeline() -> Pipeline:
+    """
+    Construct the full report pipeline.
+
+    Steps:
+        1. ScanComponents — catalog components + public methods
+        2. ScanTests — parse test files, map tested symbols
+        3. MapCoverage — cross-reference coverage
+        4. ReportGaps — compute summary + gaps
+        5. DetectOrphans — find unreferenced components/tests
+        6. GitHistory — retrieve git log per file
+        7. AssembleReport — merge into unified report
+
+    Use pipeline.run(payload) with:
+        - directory (str): component directory path
+        - tests_dir (str, optional): test directory (default: "tests")
+    """
+    pipeline = Pipeline()
+
+    pipeline.add_filter(ScanComponents(), "scan_components")
+    pipeline.add_filter(ScanTests(), "scan_tests")
+    pipeline.add_filter(MapCoverage(), "map_coverage")
+    pipeline.add_filter(ReportGaps(), "report_gaps")
+    pipeline.add_filter(DetectOrphans(), "detect_orphans")
+    pipeline.add_filter(GitHistory(), "git_history")
+    pipeline.add_filter(AssembleReport(), "assemble_report")
+
+    return pipeline

codeupipe/linter/resolve_refs.py

@@ -0,0 +1,48 @@
+"""
+ResolveRefs: Verify source files exist and compute current content hashes.
+
+Takes the doc_refs from ScanDocs and enriches each with existence checks
+and current file content hashes for drift detection.
+"""
+
+import hashlib
+from pathlib import Path
+
+from codeupipe import Payload
+
+
+class ResolveRefs:
+    """
+    Filter (sync): Resolve file references and compute current hashes.
+
+    Input keys:
+        - directory (str): root directory
+        - doc_refs (list[dict]): from ScanDocs
+
+    Output keys (added):
+        - resolved_refs (list[dict]): enriched refs with:
+            exists (bool), current_hash (str|None), abs_path (str)
+    """
+
+    def call(self, payload: Payload) -> Payload:
+        directory = Path(payload.get("directory", "."))
+        doc_refs = payload.get("doc_refs", [])
+        resolved = []
+
+        for ref in doc_refs:
+            abs_path = directory / ref["file"]
+            exists = abs_path.is_file()
+
+            current_hash = None
+            if exists:
+                content = abs_path.read_bytes()
+                current_hash = hashlib.sha256(content).hexdigest()[:7]
+
+            resolved.append({
+                **ref,
+                "exists": exists,
+                "current_hash": current_hash,
+                "abs_path": str(abs_path),
+            })
+
+        return payload.insert("resolved_refs", resolved)

codeupipe/linter/scan_components.py

@@ -0,0 +1,95 @@
+"""
+ScanComponents: Discover all CUP components and their public methods.
+"""
+
+import ast
+from pathlib import Path
+from typing import Optional
+
+from codeupipe import Payload
+
+from .scan_directory import classify_class
+
+
+def _extract_public_methods(node: ast.ClassDef) -> list:
+    """Extract public method names from a class AST node."""
+    return [
+        n.name
+        for n in ast.iter_child_nodes(node)
+        if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))
+        and not n.name.startswith("_")
+    ]
+
+
+def _extract_public_functions(tree: ast.Module) -> list:
+    """Extract top-level public function names from a module."""
+    return [
+        n.name
+        for n in ast.iter_child_nodes(tree)
+        if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))
+        and not n.name.startswith("_")
+    ]
+
+
+class ScanComponents:
+    """
+    Filter (sync): Parse component directory and catalog each
+    component class with its public methods.
+
+    Input keys:
+        - directory (str): path to the component directory
+
+    Output keys (added):
+        - components (list[dict]): each with keys:
+            - file (str): relative filepath
+            - stem (str): filename without .py
+            - name (str): class or function name
+            - kind (str): component type (filter, tap, hook, stream-filter, builder)
+            - methods (list[str]): public method names
+    """
+
+    def call(self, payload: Payload) -> Payload:
+        directory = payload.get("directory")
+        dir_path = Path(directory)
+
+        if not dir_path.is_dir():
+            raise FileNotFoundError(f"Directory not found: {directory}")
+
+        components = []
+
+        for py_file in sorted(dir_path.glob("*.py")):
+            if py_file.name == "__init__.py":
+                continue
+
+            try:
+                source = py_file.read_text()
+                tree = ast.parse(source, filename=str(py_file))
+            except (SyntaxError, OSError):
+                continue
+
+            rel = str(py_file)
+            stem = py_file.stem
+
+            for node in ast.iter_child_nodes(tree):
+                if isinstance(node, ast.ClassDef) and not node.name.startswith("_"):
+                    ctype = classify_class(node)
+                    if ctype is not None:
+                        methods = _extract_public_methods(node)
+                        components.append({
+                            "file": rel,
+                            "stem": stem,
+                            "name": node.name,
+                            "kind": ctype,
+                            "methods": methods,
+                        })
+                elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                    if not node.name.startswith("_") and node.name.startswith("build_"):
+                        components.append({
+                            "file": rel,
+                            "stem": stem,
+                            "name": node.name,
+                            "kind": "builder",
+                            "methods": [],
+                        })
+
+        return payload.insert("components", components)