codeupipe 0.1.0__py3-none-any.whl

codeupipe/linter/scan_directory.py

@@ -0,0 +1,123 @@
+"""
+ScanDirectory: Discover and AST-analyze all .py files in a directory.
+"""
+
+import ast
+import re
+from pathlib import Path
+from typing import Optional
+
+from codeupipe import Payload
+
+
+_SNAKE_RE = re.compile(r"^[a-z][a-z0-9]*(_[a-z0-9]+)*$")
+
+
+def classify_class(node: ast.ClassDef) -> Optional[str]:
+    """Classify an AST class node as a CUP component type or None."""
+    methods = {
+        n.name
+        for n in ast.iter_child_nodes(node)
+        if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))
+    }
+
+    for base in node.bases:
+        base_name = getattr(base, "id", None) or getattr(
+            getattr(base, "attr", None), "__str__", lambda: ""
+        )()
+        if base_name == "Hook":
+            return "hook"
+
+    if "stream" in methods:
+        return "stream-filter"
+    if "call" in methods:
+        return "filter"
+    if "observe" in methods:
+        return "tap"
+
+    return None
+
+
+def analyze_file(filepath: Path) -> dict:
+    """Analyze a single Python file and return component info.
+
+    Returns dict with:
+        'path': str filepath
+        'stem': filename without .py
+        'classes': list of (name, component_type, methods)
+        'functions': list of function names
+        'error': parse error string or None
+    """
+    try:
+        source = filepath.read_text()
+        tree = ast.parse(source, filename=str(filepath))
+    except SyntaxError as e:
+        return {
+            "path": str(filepath),
+            "stem": filepath.stem,
+            "classes": [],
+            "functions": [],
+            "error": str(e),
+        }
+    except OSError as e:
+        return {
+            "path": str(filepath),
+            "stem": filepath.stem,
+            "classes": [],
+            "functions": [],
+            "error": str(e),
+        }
+
+    classes = []
+    functions = []
+
+    for node in ast.iter_child_nodes(tree):
+        if isinstance(node, ast.ClassDef) and not node.name.startswith("_"):
+            methods = {
+                n.name
+                for n in ast.iter_child_nodes(node)
+                if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))
+            }
+            ctype = classify_class(node)
+            classes.append((node.name, ctype, methods))
+        elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            if not node.name.startswith("_"):
+                functions.append(node.name)
+
+    return {
+        "path": str(filepath),
+        "stem": filepath.stem,
+        "classes": classes,
+        "functions": functions,
+        "error": None,
+    }
+
+
+class ScanDirectory:
+    """
+    Filter (sync): Read a directory and AST-analyze every .py file.
+
+    Input keys:
+        - directory (str): path to the directory to scan
+
+    Output keys (added):
+        - files (list[dict]): analysis of each .py file
+        - issues (list): initialized empty issue list
+    """
+
+    def call(self, payload: Payload) -> Payload:
+        directory = payload.get("directory")
+        dir_path = Path(directory)
+
+        if not dir_path.is_dir():
+            raise FileNotFoundError(f"Directory not found: {directory}")
+
+        files = []
+        for py_file in sorted(dir_path.glob("*.py")):
+            if py_file.name == "__init__.py":
+                continue
+            files.append(analyze_file(py_file))
+
+        payload = payload.insert("files", files)
+        payload = payload.insert("issues", [])
+        return payload

codeupipe/linter/scan_docs.py

@@ -0,0 +1,62 @@
+"""
+ScanDocs: Extract cup:ref markers from markdown files.
+
+Scans all .md files in a directory for <!-- cup:ref ... --> markers
+and produces a list of doc-code references for downstream validation.
+"""
+
+import re
+from pathlib import Path
+
+from codeupipe import Payload
+
+
+_MARKER_RE = re.compile(
+    r"<!--\s*cup:ref\s+(.*?)\s*-->",
+    re.IGNORECASE,
+)
+
+_ATTR_RE = re.compile(r"(\w+)=(\S+)")
+
+
+class ScanDocs:
+    """
+    Filter (sync): Scan .md files for cup:ref markers.
+
+    Input keys:
+        - directory (str): root directory to scan
+
+    Output keys (added):
+        - doc_refs (list[dict]): extracted references, each with:
+            file, symbols, hash, doc_path, line
+    """
+
+    def call(self, payload: Payload) -> Payload:
+        directory = Path(payload.get("directory", "."))
+        refs = []
+
+        for md_path in sorted(directory.glob("*.md")):
+            content = md_path.read_text(encoding="utf-8", errors="replace")
+            for line_num, line in enumerate(content.splitlines(), start=1):
+                match = _MARKER_RE.search(line)
+                if not match:
+                    continue
+
+                attrs_str = match.group(1)
+                attrs = dict(_ATTR_RE.findall(attrs_str))
+
+                if "file" not in attrs:
+                    continue
+
+                symbols_raw = attrs.get("symbols", "")
+                symbols = [s for s in symbols_raw.split(",") if s]
+
+                refs.append({
+                    "file": attrs["file"],
+                    "symbols": symbols,
+                    "hash": attrs.get("hash", None),
+                    "doc_path": str(md_path),
+                    "line": line_num,
+                })
+
+        return payload.insert("doc_refs", refs)

codeupipe/linter/scan_tests.py

@@ -0,0 +1,104 @@
+"""
+ScanTests: Discover test files and extract tested symbols.
+"""
+
+import ast
+import re
+from pathlib import Path
+
+from codeupipe import Payload
+
+
+def _extract_imports(tree: ast.Module) -> set:
+    """Extract all imported names from a module AST."""
+    names = set()
+    for node in ast.walk(tree):
+        if isinstance(node, ast.ImportFrom):
+            for alias in (node.names or []):
+                names.add(alias.name)
+        elif isinstance(node, ast.Import):
+            for alias in (node.names or []):
+                names.add(alias.name.split(".")[-1])
+    return names
+
+
+def _extract_test_methods(tree: ast.Module) -> list:
+    """Extract test method names from test classes and top-level test functions."""
+    methods = []
+    for node in ast.iter_child_nodes(tree):
+        if isinstance(node, ast.ClassDef) and node.name.startswith("Test"):
+            for child in ast.iter_child_nodes(node):
+                if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                    if child.name.startswith("test_"):
+                        methods.append(child.name)
+        elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            if node.name.startswith("test_"):
+                methods.append(node.name)
+    return methods
+
+
+def _extract_tested_methods(tree: ast.Module) -> set:
+    """Extract method names referenced in test bodies via attribute access.
+
+    Looks for patterns like `obj.call(`, `obj.observe(`, `filter.call(` etc.
+    """
+    tested = set()
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Attribute):
+            tested.add(node.attr)
+    return tested
+
+
+class ScanTests:
+    """
+    Filter (sync): Parse test files and map which component symbols are tested.
+
+    Input keys:
+        - components (list[dict]): from ScanComponents
+        - tests_dir (str, optional): path to tests directory (default: "tests")
+
+    Output keys (added):
+        - test_map (list[dict]): each with keys:
+            - test_file (str): path to the test file
+            - stem (str): component stem the test covers
+            - imports (set[str]): imported symbol names
+            - test_methods (list[str]): test_* method names
+            - referenced_methods (set[str]): methods called in test bodies
+    """
+
+    def call(self, payload: Payload) -> Payload:
+        components = payload.get("components", [])
+        tests_dir = Path(payload.get("tests_dir", "tests"))
+
+        test_map = []
+
+        # Build a set of stems we know about
+        known_stems = {c["stem"] for c in components}
+
+        # Scan test files that correspond to known component stems
+        if tests_dir.is_dir():
+            for test_file in sorted(tests_dir.glob("test_*.py")):
+                # test_validate_email.py → validate_email
+                stem = test_file.stem[5:]  # strip "test_"
+                if stem not in known_stems:
+                    continue
+
+                try:
+                    source = test_file.read_text()
+                    tree = ast.parse(source, filename=str(test_file))
+                except (SyntaxError, OSError):
+                    continue
+
+                imports = _extract_imports(tree)
+                test_methods = _extract_test_methods(tree)
+                referenced = _extract_tested_methods(tree)
+
+                test_map.append({
+                    "test_file": str(test_file),
+                    "stem": stem,
+                    "imports": imports,
+                    "test_methods": test_methods,
+                    "referenced_methods": referenced,
+                })
+
+        return payload.insert("test_map", test_map)

codeupipe/py.typed

@@ -0,0 +1 @@
+# This file indicates that the package supports type hints

codeupipe/testing.py

@@ -0,0 +1,344 @@
+"""
+codeupipe.testing — Test utilities optimized for CUP projects.
+
+Provides zero-boilerplate helpers for testing filters, pipelines, taps,
+hooks, and stream filters. One import, no asyncio or Pipeline wiring needed.
+
+Usage:
+    from codeupipe.testing import run_filter, assert_payload, mock_filter
+
+    def test_my_filter():
+        result = run_filter(MyFilter(), {"input": "data"})
+        assert_payload(result, output="expected")
+"""
+
+import asyncio
+import inspect
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Sequence, Union
+
+from codeupipe import Payload, Pipeline
+from codeupipe.core.hook import Hook
+from codeupipe.core.state import State
+
+
+__all__ = [
+    "run_filter",
+    "run_pipeline",
+    "assert_pipeline_streaming",
+    "assert_payload",
+    "assert_keys",
+    "assert_state",
+    "mock_filter",
+    "mock_tap",
+    "mock_hook",
+    "cup_component",
+    "RecordingTap",
+    "RecordingHook",
+]
+
+
+# ── Runners ─────────────────────────────────────────────────────────
+
+def _to_payload(data: Union[dict, Payload]) -> Payload:
+    """Coerce dict or Payload to Payload."""
+    if isinstance(data, Payload):
+        return data
+    return Payload(data)
+
+
+def run_filter(filter_instance, data: Union[dict, Payload]) -> Payload:
+    """Run a single filter with minimal boilerplate.
+
+    Handles sync/async transparently. Accepts dict or Payload.
+    Returns the output Payload.
+    """
+    payload = _to_payload(data)
+    result = filter_instance.call(payload)
+    if inspect.isawaitable(result):
+        result = asyncio.run(result)
+    return result
+
+
+def run_pipeline(
+    pipeline: Pipeline,
+    data: Union[dict, Payload],
+    return_state: bool = False,
+) -> Union[Payload, tuple]:
+    """Run a pipeline with minimal boilerplate.
+
+    Args:
+        pipeline: Wired Pipeline instance.
+        data: Input dict or Payload.
+        return_state: If True, returns (result, state) tuple.
+
+    Returns:
+        Payload, or (Payload, State) if return_state=True.
+    """
+    payload = _to_payload(data)
+    result = asyncio.run(pipeline.run(payload))
+    if return_state:
+        return result, pipeline.state
+    return result
+
+
+def assert_pipeline_streaming(
+    pipeline: Pipeline,
+    chunks: Sequence[Union[dict, Payload]],
+) -> List[Payload]:
+    """Run a pipeline in stream mode and collect all output chunks.
+
+    Name reflects both dimensions: it runs a *pipeline* through its *stream*
+    path, collecting results for assertion.
+
+    Args:
+        pipeline: Pipeline with stream-capable filters.
+        chunks: Input chunks (dicts or Payloads).
+
+    Returns:
+        List of output Payloads.
+    """
+    payloads = [_to_payload(c) for c in chunks]
+
+    async def _source():
+        for p in payloads:
+            yield p
+
+    async def _collect():
+        results = []
+        async for chunk in pipeline.stream(_source()):
+            results.append(chunk)
+        return results
+
+    return asyncio.run(_collect())
+
+
+# ── Assertions ──────────────────────────────────────────────────────
+
+def assert_payload(payload: Payload, **expected: Any) -> None:
+    """Assert payload contains all expected key=value pairs.
+
+    Usage:
+        assert_payload(result, status="ok", count=3)
+    """
+    for key, expected_value in expected.items():
+        actual = payload.get(key)
+        assert actual == expected_value, (
+            f"Payload key '{key}': expected {expected_value!r}, got {actual!r}"
+        )
+
+
+def assert_keys(payload: Payload, *keys: str) -> None:
+    """Assert payload contains all specified keys.
+
+    Usage:
+        assert_keys(result, "user_id", "name", "email")
+    """
+    data = payload.to_dict()
+    for key in keys:
+        assert key in data, f"Payload missing expected key '{key}'. Keys present: {list(data.keys())}"
+
+
+def assert_state(state: State, executed: Optional[List[str]] = None) -> None:
+    """Assert pipeline state after execution.
+
+    Usage:
+        _, state = run_pipeline(pipeline, data, return_state=True)
+        assert_state(state, executed=["step_a", "step_b"])
+    """
+    if executed is not None:
+        for step in executed:
+            assert step in state.executed, (
+                f"Step '{step}' not in state.executed. "
+                f"Executed: {list(state.executed)}"
+            )
+
+
+# ── Mocks ───────────────────────────────────────────────────────────
+
+class _MockFilter:
+    """Filter mock that inserts predefined data and records calls."""
+
+    def __init__(self, **data: Any):
+        self._data = data
+        self.call_count = 0
+        self.last_payload: Optional[Payload] = None
+
+    def call(self, payload: Payload) -> Payload:
+        self.call_count += 1
+        self.last_payload = payload
+        result = payload
+        for key, value in self._data.items():
+            result = result.insert(key, value)
+        return result
+
+
+def mock_filter(**data: Any) -> _MockFilter:
+    """Create a mock filter that inserts the given key-value pairs.
+
+    Usage:
+        f = mock_filter(status="ok", processed=True)
+        result = run_filter(f, {"input": 1})
+    """
+    return _MockFilter(**data)
+
+
+class RecordingTap:
+    """Tap that records every payload it observes."""
+
+    def __init__(self):
+        self.payloads: List[Payload] = []
+        self.call_count = 0
+
+    def observe(self, payload: Payload) -> None:
+        self.call_count += 1
+        self.payloads.append(payload)
+
+
+def mock_tap() -> RecordingTap:
+    """Create a recording tap for testing.
+
+    Usage:
+        tap = mock_tap()
+        pipeline.add_tap(tap, "spy")
+        run_pipeline(pipeline, data)
+        assert tap.call_count == 1
+    """
+    return RecordingTap()
+
+
+class RecordingHook(Hook):
+    """Hook that records all lifecycle events."""
+
+    def __init__(self):
+        self.before_count = 0
+        self.after_count = 0
+        self.error_count = 0
+        self.before_payloads: List[Payload] = []
+        self.after_payloads: List[Payload] = []
+        self.errors: List[Exception] = []
+
+    async def before(self, filter, payload):
+        self.before_count += 1
+        self.before_payloads.append(payload)
+
+    async def after(self, filter, payload):
+        self.after_count += 1
+        self.after_payloads.append(payload)
+
+    async def on_error(self, filter, error, payload):
+        self.error_count += 1
+        self.errors.append(error)
+
+
+def mock_hook() -> RecordingHook:
+    """Create a recording hook for testing.
+
+    Usage:
+        hook = mock_hook()
+        pipeline.use_hook(hook)
+        run_pipeline(pipeline, data)
+        assert hook.before_count > 0
+    """
+    return RecordingHook()
+
+
+# ── Scaffolding ─────────────────────────────────────────────────────
+
+def _to_class_name(snake: str) -> str:
+    """Convert snake_case to PascalCase."""
+    return "".join(word.capitalize() for word in snake.split("_"))
+
+
+_TEMPLATES = {
+    "filter": (
+        "class {cls}:\n"
+        "{methods}\n"
+    ),
+    "tap": (
+        "class {cls}:\n"
+        "    def observe(self, payload):\n"
+        "        pass\n"
+    ),
+    "hook": (
+        "from codeupipe import Hook\n\n\n"
+        "class {cls}(Hook):\n"
+        "    async def before(self, filter, payload):\n"
+        "        pass\n\n"
+        "    async def after(self, filter, payload):\n"
+        "        pass\n\n"
+        "    async def on_error(self, filter, error, payload):\n"
+        "        pass\n"
+    ),
+    "stream-filter": (
+        "class {cls}:\n"
+        "    async def stream(self, chunk):\n"
+        "        yield chunk\n"
+    ),
+    "builder": (
+        "def build_{snake}():\n"
+        "    pass\n"
+    ),
+}
+
+_TEST_TEMPLATE = (
+    "from codeupipe.testing import run_filter\n"
+    "from {module} import {symbol}\n\n\n"
+    "class Test{cls}:\n"
+    "    def test_placeholder(self):\n"
+    "        pass\n"
+)
+
+
+def cup_component(
+    directory: Path,
+    name: str,
+    kind: str,
+    *,
+    with_test: bool = False,
+    methods: Optional[List[str]] = None,
+) -> Path:
+    """Scaffold a CUP component file on disk for analysis tests.
+
+    Args:
+        directory: Where to create the .py file.
+        name: snake_case component name.
+        kind: filter, tap, hook, stream-filter, builder.
+        with_test: Also create tests/test_{name}.py.
+        methods: Custom method list (filter kind only).
+
+    Returns:
+        Path to the created component file.
+    """
+    cls = _to_class_name(name)
+    filepath = directory / f"{name}.py"
+
+    if kind == "filter":
+        if methods:
+            method_lines = []
+            for m in methods:
+                method_lines.append(f"    def {m}(self, payload):\n        pass\n")
+            methods_str = "\n".join(method_lines)
+        else:
+            methods_str = "    def call(self, payload):\n        return payload\n"
+        source = _TEMPLATES["filter"].format(cls=cls, methods=methods_str)
+    elif kind == "builder":
+        source = _TEMPLATES["builder"].format(snake=name)
+    else:
+        source = _TEMPLATES[kind].format(cls=cls)
+
+    filepath.write_text(source)
+
+    if with_test:
+        tests_dir = directory / "tests"
+        tests_dir.mkdir(exist_ok=True)
+        if kind == "builder":
+            symbol = f"build_{name}"
+        else:
+            symbol = cls
+        test_source = _TEST_TEMPLATE.format(
+            module=name, symbol=symbol, cls=cls,
+        )
+        (tests_dir / f"test_{name}.py").write_text(test_source)
+
+    return filepath

codeupipe/utils/__init__.py

@@ -0,0 +1,10 @@
+"""
+Utils Module: Shared Utilities
+
+Common utilities that get reused across projects.
+These are the helpers that make development easier.
+"""
+
+from .error_handling import ErrorHandlingMixin, RetryFilter
+
+__all__ = ["ErrorHandlingMixin", "RetryFilter"]