codeupipe 0.1.0__py3-none-any.whl

codeupipe/core/pipeline.py

@@ -0,0 +1,231 @@
+"""
+Pipeline: The Orchestrator
+
+The Pipeline orchestrates filter execution with hooks, taps, and state tracking.
+Filters run in sequence; Valves provide conditional flow control;
+Taps provide observation points; Hooks provide lifecycle integration.
+"""
+
+import inspect
+from typing import Any, AsyncIterator, Dict, List, Optional, Set, Tuple, TypeVar, Generic, Union
+from .payload import Payload
+from .filter import Filter
+from .stream_filter import StreamFilter
+from .tap import Tap
+from .hook import Hook
+from .state import State
+
+__all__ = ["Pipeline"]
+
+TInput = TypeVar('TInput')
+TOutput = TypeVar('TOutput')
+
+
+class Pipeline(Generic[TInput, TOutput]):
+    """
+    Orchestrator — runs filters in sequence with hooks, taps, and state tracking.
+
+    Build a pipeline by adding filters (.add_filter), taps (.add_tap),
+    and hooks (.use_hook). Run it with .run(payload).
+    After execution, inspect .state for execution metadata.
+    """
+
+    def __init__(self):
+        self._steps: List[Tuple[str, Union[Filter, Tap], str]] = []  # (name, step, type)
+        self._hooks: List[Hook] = []
+        self._state: State = State()
+
+    @property
+    def state(self) -> State:
+        """Access pipeline execution state after run()."""
+        return self._state
+
+    def add_filter(self, filter: Filter[TInput, TOutput], name: Optional[str] = None) -> None:
+        """Add a filter to the pipeline."""
+        filter_name = name or filter.__class__.__name__
+        self._steps.append((filter_name, filter, "filter"))
+
+    def add_tap(self, tap: Tap, name: Optional[str] = None) -> None:
+        """Add a tap (observation point) to the pipeline."""
+        tap_name = name or tap.__class__.__name__
+        self._steps.append((tap_name, tap, "tap"))
+
+    def use_hook(self, hook: Hook) -> None:
+        """Attach a lifecycle hook."""
+        self._hooks.append(hook)
+
+    @staticmethod
+    async def _invoke(fn, *args):
+        """Call fn(*args), awaiting the result only if it is a coroutine."""
+        result = fn(*args)
+        if inspect.isawaitable(result):
+            result = await result
+        return result
+
+    async def run(self, initial_payload: Payload[TInput]) -> Payload[TOutput]:
+        """Execute the pipeline — flow payload through all filters and taps."""
+        # Check for StreamFilters — .run() is 1→1, StreamFilters are 0..N
+        for name, step, step_type in self._steps:
+            if step_type == "filter" and self._is_stream_filter(step):
+                raise ValueError(
+                    f"Pipeline contains StreamFilter '{name}'. "
+                    f"Use pipeline.stream(source) with an async generator instead. "
+                    f"Example: async for result in pipeline.stream(async_generator_of_payloads): ..."
+                )
+        
+        self._state = State()
+        payload = initial_payload
+
+        # Hook: pipeline start
+        for hook in self._hooks:
+            await self._invoke(hook.before, None, payload)
+
+        try:
+            for name, step, step_type in self._steps:
+
+                if step_type == "tap":
+                    await self._invoke(step.observe, payload)  # type: ignore
+                    self._state.mark_executed(name)
+                    continue
+
+                # It's a filter (or valve — valves conform to Filter protocol)
+                for hook in self._hooks:
+                    await self._invoke(hook.before, step, payload)
+
+                payload = await self._invoke(step.call, payload)  # type: ignore
+
+                # Track valve skips via Valve's own tracking
+                if hasattr(step, '_last_skipped') and step._last_skipped:
+                    self._state.mark_skipped(name)
+                else:
+                    self._state.mark_executed(name)
+
+                for hook in self._hooks:
+                    await self._invoke(hook.after, step, payload)
+
+        except Exception as e:
+            for hook in self._hooks:
+                await self._invoke(hook.on_error, None, e, payload)
+            raise
+
+        # Hook: pipeline end
+        for hook in self._hooks:
+            await self._invoke(hook.after, None, payload)
+
+        return payload  # type: ignore
+
+    # ------------------------------------------------------------------
+    # Streaming
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _is_stream_filter(step) -> bool:
+        """Check if a step implements the StreamFilter protocol (has .stream())."""
+        return hasattr(step, 'stream') and callable(getattr(step, 'stream'))
+
+    async def stream(
+        self,
+        source: AsyncIterator[Payload[TInput]],
+    ) -> AsyncIterator[Payload[TOutput]]:
+        """
+        Stream payloads through the pipeline, one chunk at a time.
+
+        source: An async iterable of Payload chunks.
+        Yields: Transformed Payload chunks as they flow out.
+
+        - Regular Filters are auto-adapted: 1 chunk in → 1 chunk out.
+        - StreamFilters can yield 0, 1, or N chunks per input.
+        - Valves gate per-chunk (predicate evaluated on each chunk).
+        - Taps observe each chunk.
+        - Hooks fire once per filter at stream-start and stream-end.
+        - State tracks chunks_processed per step.
+        """
+        self._state = State()
+
+        # Hook: pipeline start (payload=None-ish, use empty payload as sentinel)
+        sentinel = Payload()
+        for hook in self._hooks:
+            await self._invoke(hook.before, None, sentinel)
+
+        try:
+            # Build the processing chain as nested async generators
+            async def _source_gen():
+                async for chunk in source:
+                    yield chunk
+
+            current = _source_gen()
+
+            for name, step, step_type in self._steps:
+                current = self._wrap_step(current, name, step, step_type)
+
+            # Drain the chain, yielding results to the caller
+            async for result in current:
+                yield result
+
+        except Exception as e:
+            for hook in self._hooks:
+                await self._invoke(hook.on_error, None, e, sentinel)
+            raise
+
+        # Hook: pipeline end
+        for hook in self._hooks:
+            await self._invoke(hook.after, None, sentinel)
+
+    async def _wrap_step(
+        self,
+        upstream: AsyncIterator[Payload],
+        name: str,
+        step,
+        step_type: str,
+    ) -> AsyncIterator[Payload]:
+        """Wrap a single step around an upstream async iterator."""
+
+        # --- Tap: observe each chunk, pass through unchanged ---
+        if step_type == "tap":
+            if name not in self._state.executed:
+                self._state.mark_executed(name)
+            async for chunk in upstream:
+                await self._invoke(step.observe, chunk)  # type: ignore
+                self._state.increment_chunks(name)
+                yield chunk
+            return
+
+        # --- Filter or Valve ---
+        # Fire hook.before once at the start of this step's stream
+        for hook in self._hooks:
+            await self._invoke(hook.before, step, Payload())
+
+        is_valve = hasattr(step, '_predicate')
+        is_stream = self._is_stream_filter(step)
+
+        if name not in self._state.executed and name not in self._state.skipped:
+            self._state.mark_executed(name)
+
+        async for chunk in upstream:
+            # Valve gating — per-chunk predicate
+            if is_valve:
+                if not step._predicate(chunk):
+                    self._state.increment_chunks(name)  # counted but skipped
+                    yield chunk
+                    continue
+
+            if is_stream:
+                # StreamFilter: yield 0..N chunks per input
+                stream_result = step.stream(chunk)
+                if inspect.isasyncgen(stream_result):
+                    async for out in stream_result:
+                        self._state.increment_chunks(name)
+                        yield out
+                else:
+                    for out in stream_result:
+                        self._state.increment_chunks(name)
+                        yield out
+            else:
+                # Regular Filter: 1 chunk in → 1 chunk out
+                result = await self._invoke(step.call, chunk)  # type: ignore
+                self._state.increment_chunks(name)
+                yield result
+
+        # Fire hook.after once at the end of this step's stream
+        for hook in self._hooks:
+            await self._invoke(hook.after, step, Payload())

codeupipe/core/state.py

@@ -0,0 +1,78 @@
+"""
+State: Pipeline Execution Metadata
+
+State tracks what happened during pipeline execution — which filters ran,
+which were skipped, timing data, and errors encountered.
+Access it after pipeline.run() via pipeline.state.
+"""
+
+from typing import Any, Dict, List, Optional, Tuple
+
+__all__ = ["State"]
+
+
+class State:
+    """
+    Pipeline execution state — tracks filter execution, timing, and errors.
+
+    Provides visibility into what happened during a pipeline run:
+    - Which filters executed and in what order
+    - Which filters were skipped (by valves)
+    - Errors encountered during execution
+    - Arbitrary metadata for custom tracking
+    """
+
+    def __init__(self):
+        self.executed: List[str] = []
+        self.skipped: List[str] = []
+        self.errors: List[Tuple[str, Exception]] = []
+        self.metadata: Dict[str, Any] = {}
+        self.chunks_processed: Dict[str, int] = {}
+
+    def mark_executed(self, name: str) -> None:
+        """Record that a filter executed."""
+        self.executed.append(name)
+
+    def mark_skipped(self, name: str) -> None:
+        """Record that a filter was skipped."""
+        self.skipped.append(name)
+
+    def increment_chunks(self, name: str, count: int = 1) -> None:
+        """Increment the chunk counter for a streaming step."""
+        self.chunks_processed[name] = self.chunks_processed.get(name, 0) + count
+
+    def record_error(self, name: str, error: Exception) -> None:
+        """Record an error from a filter."""
+        self.errors.append((name, error))
+
+    def set(self, key: str, value: Any) -> None:
+        """Store arbitrary metadata."""
+        self.metadata[key] = value
+
+    def get(self, key: str, default: Any = None) -> Any:
+        """Retrieve metadata."""
+        return self.metadata.get(key, default)
+
+    @property
+    def has_errors(self) -> bool:
+        """Whether any errors were recorded."""
+        return len(self.errors) > 0
+
+    @property
+    def last_error(self) -> Optional[Exception]:
+        """The most recent error, or None."""
+        return self.errors[-1][1] if self.errors else None
+
+    def reset(self) -> None:
+        """Reset state for a fresh run."""
+        self.executed.clear()
+        self.skipped.clear()
+        self.errors.clear()
+        self.metadata.clear()
+        self.chunks_processed.clear()
+
+    def __repr__(self) -> str:
+        return (
+            f"State(executed={self.executed}, skipped={self.skipped}, "
+            f"errors={len(self.errors)}, chunks={self.chunks_processed})"
+        )

codeupipe/core/stream_filter.py

@@ -0,0 +1,33 @@
+"""
+StreamFilter Protocol: Chunk-at-a-Time Processing
+
+A StreamFilter processes one Payload chunk and yields zero or more output chunks.
+This enables filtering (drop), mapping (1→1), and fan-out (1→N) at constant memory.
+
+Regular Filters are auto-adapted for streaming (1 chunk in → 1 chunk out).
+StreamFilters opt in to the richer yield-based interface.
+"""
+
+from typing import AsyncIterator, Protocol, TypeVar
+from .payload import Payload
+
+__all__ = ["StreamFilter"]
+
+TInput = TypeVar('TInput')
+TOutput = TypeVar('TOutput')
+
+
+class StreamFilter(Protocol[TInput, TOutput]):
+    """
+    Streaming processing unit — receives one chunk, yields zero or more output chunks.
+
+    Use for:
+    - Filtering: yield nothing to drop a chunk
+    - Mapping: yield one transformed chunk (same as a regular Filter)
+    - Fan-out: yield multiple chunks from one input
+    - Batching/windowing: accumulate internally, yield when ready
+    """
+
+    async def stream(self, chunk: Payload[TInput]) -> AsyncIterator[Payload[TOutput]]:
+        """Process a single chunk and yield output chunks."""
+        ...

codeupipe/core/tap.py

@@ -0,0 +1,27 @@
+"""
+Tap: Observation Point
+
+A Tap is a non-modifying observation point in the pipeline.
+It receives the payload for inspection (logging, metrics, debugging)
+but never modifies it. Think of it as a pressure gauge on a pipe.
+"""
+
+from typing import Protocol, TypeVar
+from .payload import Payload
+
+__all__ = ["Tap"]
+
+T = TypeVar('T')
+
+
+class Tap(Protocol[T]):
+    """
+    Non-modifying observation point — inspect the payload without changing it.
+
+    Use Taps for logging, metrics, debugging, and auditing.
+    The pipeline calls observe() and discards the return value.
+    """
+
+    async def observe(self, payload: Payload[T]) -> None:
+        """Observe the payload. Must not modify it."""
+        ...

codeupipe/core/valve.py

@@ -0,0 +1,52 @@
+"""
+Valve: Conditional Flow Control
+
+A Valve wraps a Filter with a predicate — the inner filter only executes
+when the predicate evaluates to True. Otherwise the payload passes through unchanged.
+Valves conform to the Filter protocol, so they compose seamlessly into Pipelines.
+"""
+
+import inspect
+from typing import Callable, Generic, TypeVar
+from .payload import Payload
+from .filter import Filter
+
+__all__ = ["Valve"]
+
+TInput = TypeVar('TInput')
+TOutput = TypeVar('TOutput')
+
+
+class Valve(Generic[TInput, TOutput]):
+    """
+    Conditional flow control — gates a Filter with a predicate.
+
+    If the predicate returns True, the inner filter processes the payload.
+    If False, the payload passes through unchanged.
+
+    Conforms to the Filter protocol so it can be used anywhere a Filter is expected.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        inner: Filter[TInput, TOutput],
+        predicate: Callable[[Payload[TInput]], bool],
+    ):
+        self.name = name
+        self._inner = inner
+        self._predicate = predicate
+
+    async def call(self, payload: Payload[TInput]) -> Payload[TOutput]:
+        """Execute the inner filter only if the predicate passes."""
+        if self._predicate(payload):
+            self._last_skipped = False
+            result = self._inner.call(payload)
+            if inspect.isawaitable(result):
+                result = await result
+            return result
+        self._last_skipped = True
+        return payload  # type: ignore — pass through unchanged
+
+    def __repr__(self) -> str:
+        return f"Valve({self.name!r})"

codeupipe/linter/__init__.py

@@ -0,0 +1,57 @@
+"""
+Public API for linter package.
+
+Auto-generated by: cup bundle
+"""
+
+from .assemble_doc_report import AssembleDocReport
+from .assemble_report import AssembleReport
+from .check_bundle import CheckBundle
+from .check_index import CheckIndex
+from .check_naming import CheckNaming
+from .check_protocols import CheckProtocols
+from .check_structure import CheckStructure
+from .check_symbols import CheckSymbols
+from .check_tests import CheckTests
+from .coverage_pipeline import build_coverage_pipeline
+from .detect_drift import DetectDrift
+from .detect_orphans import DetectOrphans
+from .doc_check_pipeline import build_doc_check_pipeline
+from .git_history import GitHistory
+from .lint_pipeline import build_lint_pipeline
+from .map_coverage import MapCoverage
+from .report_gaps import ReportGaps
+from .report_pipeline import build_report_pipeline
+from .resolve_refs import ResolveRefs
+from .scan_components import ScanComponents
+from .scan_directory import ScanDirectory, analyze_file, classify_class
+from .scan_docs import ScanDocs
+from .scan_tests import ScanTests
+
+__all__ = [
+    "AssembleDocReport",
+    "AssembleReport",
+    "CheckBundle",
+    "CheckIndex",
+    "CheckNaming",
+    "CheckProtocols",
+    "CheckStructure",
+    "CheckSymbols",
+    "CheckTests",
+    "DetectDrift",
+    "DetectOrphans",
+    "GitHistory",
+    "MapCoverage",
+    "ReportGaps",
+    "ResolveRefs",
+    "ScanComponents",
+    "ScanDirectory",
+    "ScanDocs",
+    "ScanTests",
+    "analyze_file",
+    "build_coverage_pipeline",
+    "build_doc_check_pipeline",
+    "build_lint_pipeline",
+    "build_report_pipeline",
+    "classify_class",
+]

codeupipe/linter/assemble_doc_report.py

@@ -0,0 +1,97 @@
+"""
+AssembleDocReport: Merge all doc-check findings into a structured report.
+
+Combines results from ScanDocs, ResolveRefs, CheckSymbols, DetectDrift,
+and CheckIndex into a single structured report with summary counts and
+detail items.
+"""
+
+from codeupipe import Payload
+
+
+class AssembleDocReport:
+    """
+    Filter (sync): Assemble the final doc-check report.
+
+    Input keys:
+        - doc_refs (list[dict]): from ScanDocs
+        - resolved_refs (list[dict]): from ResolveRefs
+        - drifted_refs (list[dict]): from DetectDrift
+        - symbol_issues (list[dict]): from CheckSymbols
+        - index_issues (list[dict]): from CheckIndex (optional)
+
+    Output keys (added):
+        - doc_report (dict): structured report with:
+            total_refs, drifted, missing_symbols, missing_files,
+            unmapped_files, status ("ok"|"stale"), details (list)
+    """
+
+    def call(self, payload: Payload) -> Payload:
+        doc_refs = payload.get("doc_refs", [])
+        resolved = payload.get("resolved_refs", [])
+        drifted = payload.get("drifted_refs", [])
+        symbol_issues = payload.get("symbol_issues", [])
+        index_issues = payload.get("index_issues", [])
+
+        missing_files = sum(1 for r in resolved if not r.get("exists", True))
+
+        details = []
+
+        for d in drifted:
+            details.append({
+                "type": "drift",
+                "file": d["file"],
+                "doc_path": d["doc_path"],
+                "line": d["line"],
+                "message": (
+                    f"Hash drift: stored={d['stored_hash']}, "
+                    f"current={d['current_hash']}"
+                ),
+            })
+
+        for s in symbol_issues:
+            details.append({
+                "type": "missing_symbol",
+                "file": s["file"],
+                "doc_path": s["doc_path"],
+                "line": s["line"],
+                "message": f"Symbol '{s['symbol']}' not found in {s['file']}",
+            })
+
+        for r in resolved:
+            if not r.get("exists", True):
+                details.append({
+                    "type": "missing_file",
+                    "file": r["file"],
+                    "doc_path": r["doc_path"],
+                    "line": r["line"],
+                    "message": f"Referenced file '{r['file']}' does not exist",
+                })
+
+        for idx in index_issues:
+            details.append({
+                "type": "unmapped_file",
+                "file": idx["file"],
+                "doc_path": "INDEX.md",
+                "line": 0,
+                "message": idx["message"],
+            })
+
+        has_issues = (
+            len(drifted) > 0
+            or len(symbol_issues) > 0
+            or missing_files > 0
+            or len(index_issues) > 0
+        )
+
+        report = {
+            "total_refs": len(doc_refs),
+            "drifted": len(drifted),
+            "missing_symbols": len(symbol_issues),
+            "missing_files": missing_files,
+            "unmapped_files": len(index_issues),
+            "status": "stale" if has_issues else "ok",
+            "details": details,
+        }
+
+        return payload.insert("doc_report", report)