pydp-engine 0.1.0__py3-none-any.whl

pydp/__init__.py

@@ -0,0 +1,76 @@
+"""PyDP -- an intelligent, declarative Dynamic Programming engine.
+
+Describe the recurrence; the engine handles memoization, dependency analysis,
+execution strategy, memory optimization, statistics, and visualization.
+
+Quick start::
+
+    from pydp import dp
+
+    @dp
+    def fib(solve, n):
+        if n < 2:
+            return n
+        return solve(n - 1) + solve(n - 2)
+
+    print(fib(30))            # 832040
+    print(fib.stats.summary())
+"""
+
+from __future__ import annotations
+
+from .analyze import (
+    BaseCase,
+    RecurrenceAnalysis,
+    RecursiveCall,
+    analyze,
+    analyze_source,
+)
+from .engine import CyclicDependencyError, DPProblem, dp
+from .explain import explain
+from .graph import DependencyGraph
+from .optimize import (
+    ComplexityEstimate,
+    MemoryOptimization,
+    analyze_memory,
+    estimate_complexity,
+    suggestions,
+)
+from .stats import PerformanceStats
+from .storage import StorageAnalysis, analyze_storage
+from .visualize import (
+    dependency_text,
+    heatmap_text,
+    recursion_tree,
+    to_graphviz,
+)
+from . import templates
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "dp",
+    "DPProblem",
+    "CyclicDependencyError",
+    "DependencyGraph",
+    "PerformanceStats",
+    "StorageAnalysis",
+    "analyze_storage",
+    "MemoryOptimization",
+    "ComplexityEstimate",
+    "analyze_memory",
+    "estimate_complexity",
+    "suggestions",
+    "explain",
+    "analyze",
+    "analyze_source",
+    "RecurrenceAnalysis",
+    "RecursiveCall",
+    "BaseCase",
+    "recursion_tree",
+    "dependency_text",
+    "heatmap_text",
+    "to_graphviz",
+    "templates",
+    "__version__",
+]

pydp/analyze.py

@@ -0,0 +1,185 @@
+"""AST-based static analysis of a recurrence function.
+
+This inspects the *source* of a recurrence (before it ever runs) and infers its
+structure: the solve handle, the state parameters, the base cases, and the
+recursive transitions. It complements the runtime dependency graph in
+:mod:`pydp.graph` — that one observes what actually happened, this one reads
+what the code says.
+
+Analysis is best-effort: it degrades gracefully (empty / partial results)
+when source is unavailable (e.g. lambdas defined in the REPL) or when the
+recurrence uses constructs it does not model.
+"""
+
+from __future__ import annotations
+
+import ast
+import inspect
+import textwrap
+from dataclasses import dataclass, field
+from typing import Any, Callable, List, Optional
+
+from .engine import DPProblem
+
+
+@dataclass
+class RecursiveCall:
+    """A call to the solve handle found in the source."""
+    args_src: List[str]           # source text of each argument
+    lineno: int
+
+    def __str__(self) -> str:
+        return f"solve({', '.join(self.args_src)})  @line {self.lineno}"
+
+
+@dataclass
+class BaseCase:
+    """A return statement that contains no recursive call (a terminating case)."""
+    condition_src: Optional[str]  # guarding condition, if any
+    value_src: str
+    lineno: int
+
+    def __str__(self) -> str:
+        if self.condition_src:
+            return f"if {self.condition_src}: return {self.value_src}  @line {self.lineno}"
+        return f"return {self.value_src}  @line {self.lineno}"
+
+
+@dataclass
+class RecurrenceAnalysis:
+    name: str
+    solve_param: Optional[str]
+    state_params: List[str]
+    base_cases: List[BaseCase] = field(default_factory=list)
+    recursive_calls: List[RecursiveCall] = field(default_factory=list)
+    source_available: bool = True
+    notes: List[str] = field(default_factory=list)
+
+    @property
+    def dimensionality(self) -> int:
+        return len(self.state_params)
+
+    def summary(self) -> str:
+        lines = [f"Recurrence analysis: {self.name!r}", "-" * 44]
+        if not self.source_available:
+            lines.append("  (source unavailable — static analysis skipped)")
+            return "\n".join(lines)
+        lines.append(f"  Solve handle       : {self.solve_param}")
+        lines.append(f"  State parameters   : {', '.join(self.state_params) or '(none)'}")
+        lines.append(f"  Dimensionality     : {self.dimensionality}-D")
+        lines.append(f"  Base cases         : {len(self.base_cases)}")
+        for bc in self.base_cases:
+            lines.append(f"    - {bc}")
+        lines.append(f"  Recursive calls    : {len(self.recursive_calls)}")
+        for rc in self.recursive_calls:
+            lines.append(f"    - {rc}")
+        if self.notes:
+            lines.append("  Notes:")
+            for n in self.notes:
+                lines.append(f"    * {n}")
+        return "\n".join(lines)
+
+
+def _unparse(node: ast.AST) -> str:
+    try:
+        return ast.unparse(node)  # py3.9+
+    except Exception:  # pragma: no cover - defensive
+        return "<expr>"
+
+
+class _Visitor(ast.NodeVisitor):
+    def __init__(self, solve_param: str) -> None:
+        self.solve_param = solve_param
+        self.recursive_calls: List[RecursiveCall] = []
+
+    def visit_Call(self, node: ast.Call) -> None:
+        func = node.func
+        if isinstance(func, ast.Name) and func.id == self.solve_param:
+            self.recursive_calls.append(
+                RecursiveCall(args_src=[_unparse(a) for a in node.args],
+                              lineno=getattr(node, "lineno", -1))
+            )
+        self.generic_visit(node)
+
+
+def _contains_solve_call(node: ast.AST, solve_param: str) -> bool:
+    for child in ast.walk(node):
+        if (isinstance(child, ast.Call) and isinstance(child.func, ast.Name)
+                and child.func.id == solve_param):
+            return True
+    return False
+
+
+def _get_function_def(func: Callable[..., Any]):
+    src = textwrap.dedent(inspect.getsource(func))
+    module = ast.parse(src)
+    for node in module.body:
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            return node
+    return None
+
+
+def analyze_source(func: Callable[..., Any], name: Optional[str] = None) -> RecurrenceAnalysis:
+    """Statically analyze a recurrence *function* (not a DPProblem)."""
+    display = name or getattr(func, "__name__", "recurrence")
+    try:
+        fn_def = _get_function_def(func)
+    except (OSError, TypeError):
+        return RecurrenceAnalysis(display, None, [], source_available=False)
+
+    if fn_def is None:
+        return RecurrenceAnalysis(display, None, [], source_available=False,
+                                  notes=["could not locate a function definition"])
+
+    params = [a.arg for a in fn_def.args.args]
+    solve_param = params[0] if params else None
+    state_params = params[1:] if len(params) > 1 else []
+
+    analysis = RecurrenceAnalysis(display, solve_param, state_params)
+
+    if solve_param is None:
+        analysis.notes.append("recurrence has no parameters; expected solve(...) handle first")
+        return analysis
+
+    # Recursive calls anywhere in the body.
+    visitor = _Visitor(solve_param)
+    visitor.visit(fn_def)
+    analysis.recursive_calls = visitor.recursive_calls
+
+    # Base cases: return statements with no recursive call in the returned expr.
+    for node in ast.walk(fn_def):
+        if isinstance(node, ast.Return) and node.value is not None:
+            if not _contains_solve_call(node.value, solve_param):
+                cond = _enclosing_condition(fn_def, node)
+                analysis.base_cases.append(
+                    BaseCase(condition_src=cond,
+                             value_src=_unparse(node.value),
+                             lineno=getattr(node, "lineno", -1))
+                )
+
+    if not analysis.recursive_calls:
+        analysis.notes.append("no recursive solve(...) calls found — is this actually a DP recurrence?")
+    if not analysis.base_cases:
+        analysis.notes.append("no base case detected — recurrence may not terminate")
+
+    return analysis
+
+
+def _enclosing_condition(fn_def: ast.AST, target: ast.Return) -> Optional[str]:
+    """Find the innermost `if` test whose body directly contains `target`."""
+    found: Optional[str] = None
+
+    class Finder(ast.NodeVisitor):
+        def visit_If(self, node: ast.If) -> None:
+            if any(stmt is target for stmt in node.body):
+                nonlocal found
+                found = _unparse(node.test)
+            self.generic_visit(node)
+
+    Finder().visit(fn_def)
+    return found
+
+
+def analyze(problem: DPProblem, name: Optional[str] = None) -> RecurrenceAnalysis:
+    """Statically analyze the recurrence backing a :class:`DPProblem`."""
+    return analyze_source(problem.func, name=name or problem.name)

pydp/engine.py

@@ -0,0 +1,360 @@
+"""The PyDP core engine.
+
+Users declare a recurrence as a plain function whose first parameter is a
+``solve`` handle used to request sub-states::
+
+    @dp
+    def fib(solve, n):
+        if n < 2:
+            return n
+        return solve(n - 1) + solve(n - 2)
+
+    fib(30)          # -> 832040
+    fib.stats        # PerformanceStats for the last run
+    fib.graph        # DependencyGraph for the last run
+
+The engine transparently memoizes states, records the dependency graph and
+recursion tree, collects performance statistics, and can execute the recurrence
+either top-down (default) or via an automatically derived bottom-up order.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+import threading
+import time
+from typing import Any, Callable, Dict, Hashable, List, Optional, Tuple
+
+from .graph import DependencyGraph
+from .state import make_key
+from .stats import PerformanceStats
+from .storage import StorageAnalysis, analyze_storage
+
+# States requested during the current top-level run but not yet computed are
+# pushed onto a stack so we can attribute dependency edges to their parent.
+_SENTINEL = object()
+
+
+class CyclicDependencyError(ValueError):
+    """Raised when a recurrence asks for a state that is still being computed."""
+
+# Deep recurrences can exceed the interpreter's default limit *and* the OS C
+# stack. sys.setrecursionlimit only raises the former; to grow the actual stack
+# we run the computation in a worker thread with an enlarged stack size. The
+# maximum stack size accepted by threading.stack_size is platform-dependent
+# (e.g. Windows rejects >~256 MB), so we probe a descending list of candidates.
+_STACK_CANDIDATES_MB = (200, 128, 64, 32, 16)
+
+
+def _apply_big_stack_size() -> None:
+    for mb in _STACK_CANDIDATES_MB:
+        try:
+            threading.stack_size(mb * 1024 * 1024)
+            return
+        except (ValueError, RuntimeError, OverflowError):
+            continue
+    # No candidate accepted; leave the default stack in place.
+
+
+def _run_with_big_stack(fn: Callable[[], Any]) -> Any:
+    box: Dict[str, Any] = {}
+
+    def target() -> None:
+        try:
+            box["value"] = fn()
+        except BaseException as exc:  # propagate to caller thread
+            box["error"] = exc
+
+    old_stack = threading.stack_size()
+    _apply_big_stack_size()
+    worker = threading.Thread(target=target)
+    worker.start()
+    worker.join()
+    try:
+        threading.stack_size(old_stack)
+    except (ValueError, RuntimeError):
+        pass
+    if "error" in box:
+        raise box["error"]
+    return box.get("value")
+
+
+class _RunContext:
+    """Mutable per-run state shared between the solver and the problem."""
+
+    def __init__(self) -> None:
+        self.memo: Dict[Hashable, Any] = {}
+        self.args_of: Dict[Hashable, Tuple[Any, ...]] = {}
+        self.graph = DependencyGraph()
+        self.stats = PerformanceStats()
+        self.call_stack: List[Hashable] = []
+        # States currently being evaluated (on the recursion stack). Used to
+        # detect cyclic dependencies before they overflow the C stack.
+        self.in_progress: set = set()
+        # Full recursion tree: parent-key -> ordered list of child keys.
+        self.tree_children: Dict[Optional[Hashable], List[Hashable]] = {}
+
+
+class DPProblem:
+    """A declarative dynamic-programming problem.
+
+    Wrap a recurrence function with :func:`dp` (or construct directly). Calling
+    the instance solves for the requested state and caches artifacts from the
+    most recent run on ``.stats``, ``.graph``, ``.table`` and ``.tree``.
+    """
+
+    def __init__(
+        self,
+        func: Callable[..., Any],
+        name: Optional[str] = None,
+        strategy: str = "top-down",
+        recursion_limit: int = 1_000_000,
+    ) -> None:
+        if strategy not in ("top-down", "bottom-up", "auto", "parallel"):
+            raise ValueError(f"unknown strategy: {strategy!r}")
+        self.func = func
+        self.name = name or getattr(func, "__name__", "dp")
+        self.strategy = strategy
+        self.recursion_limit = recursion_limit
+
+        # Artifacts from the most recent run.
+        self.stats: Optional[PerformanceStats] = None
+        self.graph: Optional[DependencyGraph] = None
+        self.table: Dict[Hashable, Any] = {}
+        self.tree: Dict[Optional[Hashable], List[Hashable]] = {}
+        self._last_ctx: Optional[_RunContext] = None
+
+    # ------------------------------------------------------------------ #
+    # Public entry points
+    # ------------------------------------------------------------------ #
+    def __call__(self, *args: Any, strategy: Optional[str] = None,
+                 workers: Optional[int] = None) -> Any:
+        strat = strategy or self.strategy
+        if strat == "auto":
+            strat = "top-down"
+        if strat == "top-down":
+            return self._run_top_down(args)
+        if strat == "parallel":
+            return self._run_bottom_up(args, parallel=True, workers=workers)
+        return self._run_bottom_up(args)
+
+    def solve(self, *args: Any, strategy: Optional[str] = None) -> Any:
+        """Alias for calling the problem; reads naturally in templates."""
+        return self.__call__(*args, strategy=strategy)
+
+    # ------------------------------------------------------------------ #
+    # Top-down execution (demand-driven recursion + memoization)
+    # ------------------------------------------------------------------ #
+    def _run_top_down(self, args: Tuple[Any, ...]) -> Any:
+        ctx = _RunContext()
+        ctx.stats.strategy = "top-down"
+        self._install(ctx)
+        old_limit = sys.getrecursionlimit()
+        sys.setrecursionlimit(max(old_limit, self.recursion_limit))
+        start = time.perf_counter()
+        try:
+            result = _run_with_big_stack(lambda: self._compute(ctx, args))
+        finally:
+            ctx.stats.execution_time = time.perf_counter() - start
+            ctx.stats.peak_states_in_memory = len(ctx.memo)
+            sys.setrecursionlimit(old_limit)
+            self._publish(ctx)
+        return result
+
+    def _compute(self, ctx: _RunContext, args: Tuple[Any, ...]) -> Any:
+        key = make_key(args)
+        parent = ctx.call_stack[-1] if ctx.call_stack else None
+
+        # Record structure regardless of cache state.
+        ctx.graph.add_node(key)
+        if parent is not None:
+            ctx.graph.add_edge(parent, key)
+        ctx.tree_children.setdefault(parent, []).append(key)
+
+        if key in ctx.memo:
+            ctx.stats.cache_hits += 1
+            return ctx.memo[key]
+
+        if key in ctx.in_progress:
+            raise CyclicDependencyError(
+                f"state {key!r} depends on itself (cyclic recurrence); "
+                "a well-formed DP recurrence must be acyclic."
+            )
+
+        ctx.stats.cache_misses += 1
+        ctx.stats.states_explored += 1
+        ctx.args_of[key] = args
+
+        ctx.call_stack.append(key)
+        ctx.in_progress.add(key)
+        depth = len(ctx.call_stack)
+        if depth > ctx.stats.max_recursion_depth:
+            ctx.stats.max_recursion_depth = depth
+        try:
+            value = self.func(ctx.solver, *args)
+        finally:
+            ctx.call_stack.pop()
+            ctx.in_progress.discard(key)
+
+        ctx.memo[key] = value
+        return value
+
+    # ------------------------------------------------------------------ #
+    # Bottom-up execution (topological order over the discovered DAG)
+    # ------------------------------------------------------------------ #
+    def _run_bottom_up(self, args: Tuple[Any, ...], parallel: bool = False,
+                       workers: Optional[int] = None) -> Any:
+        # Phase 1: discover the dependency DAG and the args of every state via
+        # a top-down pass (values memoized so recursion stays shallow after the
+        # first descent).
+        discovery = _RunContext()
+        label = "parallel" if parallel else "bottom-up"
+        discovery.stats.strategy = label
+        self._install(discovery)
+        old_limit = sys.getrecursionlimit()
+        sys.setrecursionlimit(max(old_limit, self.recursion_limit))
+        start = time.perf_counter()
+        try:
+            _run_with_big_stack(lambda: self._compute(discovery, args))
+            if discovery.graph.has_cycle():
+                raise CyclicDependencyError(
+                    f"recurrence has cyclic state dependencies; cannot solve "
+                    f"{label} (use strategy='top-down')."
+                )
+
+            ctx = _RunContext()
+            ctx.stats.strategy = label
+            ctx.args_of = discovery.args_of
+
+            if parallel:
+                self._eval_parallel(ctx, discovery, workers)
+            else:
+                self._eval_sequential(ctx, discovery)
+
+            result = ctx.memo[make_key(args)]
+            ctx.stats.execution_time = time.perf_counter() - start
+            ctx.stats.max_recursion_depth = 1
+            ctx.stats.peak_states_in_memory = len(ctx.memo)
+            for dep_from, dep_to in discovery.graph.edges():
+                ctx.graph.add_edge(dep_from, dep_to)
+            self._publish(ctx)
+        finally:
+            sys.setrecursionlimit(old_limit)
+        return result
+
+    def _eval_sequential(self, ctx: "_RunContext", discovery: "_RunContext") -> None:
+        """Re-evaluate every state in dependency order (single-threaded)."""
+        self._install(ctx)
+        order = discovery.graph.topological_order()
+        for key in order:
+            state_args = discovery.args_of.get(key)
+            if state_args is None:
+                continue
+            ctx.graph.add_node(key)
+            ctx.stats.states_explored += 1
+            ctx.stats.cache_misses += 1
+            ctx.call_stack.append(key)
+            try:
+                value = self.func(ctx.solver, *state_args)
+            finally:
+                ctx.call_stack.pop()
+            ctx.memo[key] = value
+
+    def _eval_parallel(self, ctx: "_RunContext", discovery: "_RunContext",
+                       workers: Optional[int]) -> None:
+        """Evaluate topological layers concurrently.
+
+        Every state in a layer depends only on earlier layers, whose values are
+        already in ``completed``. Each worker gets a read-only solver over that
+        shared dict, so no worker mutates state another worker reads.
+
+        Note: the recurrence body runs in Python, so the GIL bounds CPU-bound
+        speedup; the win is real for recurrences that release the GIL (I/O,
+        NumPy) and the layering is what a genuine multi-process backend would
+        also consume. Determinism and correctness are guaranteed regardless.
+        """
+        import concurrent.futures as _cf
+
+        completed: Dict[Hashable, Any] = {}
+
+        def readonly_solver(*sub_args: Any) -> Any:
+            return completed[make_key(sub_args)]
+
+        layers = discovery.graph.topological_layers()
+        max_workers = workers or min(32, (os.cpu_count() or 2))
+
+        def evaluate(key: Hashable) -> Tuple[Hashable, Any]:
+            state_args = discovery.args_of[key]
+            return key, self.func(readonly_solver, *state_args)
+
+        with _cf.ThreadPoolExecutor(max_workers=max_workers) as pool:
+            for layer in layers:
+                runnable = [k for k in layer if k in discovery.args_of]
+                results = list(pool.map(evaluate, runnable))
+                for key, value in results:
+                    completed[key] = value
+                    ctx.memo[key] = value
+                    ctx.graph.add_node(key)
+                    ctx.stats.states_explored += 1
+                    ctx.stats.cache_misses += 1
+
+    # ------------------------------------------------------------------ #
+    # Solver handle passed into the recurrence
+    # ------------------------------------------------------------------ #
+    def _install(self, ctx: _RunContext) -> None:
+        def solver(*sub_args: Any) -> Any:
+            return self._compute(ctx, sub_args)
+        ctx.solver = solver  # type: ignore[attr-defined]
+
+    def _publish(self, ctx: _RunContext) -> None:
+        self.stats = ctx.stats
+        self.graph = ctx.graph
+        self.table = dict(ctx.memo)
+        self.tree = ctx.tree_children
+        self._last_ctx = ctx
+
+    # ------------------------------------------------------------------ #
+    # Analysis helpers
+    # ------------------------------------------------------------------ #
+    def storage(self) -> StorageAnalysis:
+        """Recommend a storage mechanism for the most recent run's states."""
+        if self.graph is None:
+            raise RuntimeError("run the problem before analyzing storage")
+        return analyze_storage(self.graph.nodes)
+
+    def clear(self) -> None:
+        self.stats = None
+        self.graph = None
+        self.table = {}
+        self.tree = {}
+        self._last_ctx = None
+
+    def __repr__(self) -> str:
+        return f"<DPProblem {self.name!r} strategy={self.strategy!r}>"
+
+
+def dp(
+    func: Optional[Callable[..., Any]] = None,
+    *,
+    strategy: str = "top-down",
+    name: Optional[str] = None,
+    recursion_limit: int = 1_000_000,
+) -> Any:
+    """Decorator turning a recurrence function into a :class:`DPProblem`.
+
+    Usage::
+
+        @dp
+        def fib(solve, n): ...
+
+        @dp(strategy="bottom-up")
+        def knap(solve, i, cap): ...
+    """
+    def wrap(f: Callable[..., Any]) -> DPProblem:
+        return DPProblem(f, name=name, strategy=strategy,
+                         recursion_limit=recursion_limit)
+
+    if func is not None:
+        return wrap(func)
+    return wrap

pydp/explain.py

@@ -0,0 +1,55 @@
+"""Educational mode: narrate how a solved DP problem behaved."""
+
+from __future__ import annotations
+
+from typing import List
+
+from .engine import DPProblem
+from .optimize import estimate_complexity, suggestions
+
+
+def explain(problem: DPProblem, max_states: int = 12) -> str:
+    if problem.graph is None or problem.stats is None:
+        raise RuntimeError("run the problem before explaining it")
+
+    graph = problem.graph
+    stats = problem.stats
+    lines: List[str] = []
+    lines.append(f"How '{problem.name}' was solved")
+    lines.append("=" * 44)
+
+    lines.append("")
+    lines.append("1. State space")
+    lines.append(f"   Explored {len(graph.nodes)} distinct states using the "
+                 f"{stats.strategy} strategy.")
+
+    lines.append("")
+    lines.append("2. Dependency structure")
+    acyclic = not graph.has_cycle()
+    lines.append(f"   The state graph is {'acyclic (a DAG)' if acyclic else 'cyclic'} "
+                 f"with {graph.edge_count()} edges.")
+    if acyclic:
+        try:
+            order = graph.topological_order()
+            shown = ", ".join(str(s) for s in order[:max_states])
+            more = "" if len(order) <= max_states else ", ..."
+            lines.append(f"   Valid evaluation order (dependencies first): {shown}{more}")
+        except ValueError:
+            pass
+
+    lines.append("")
+    lines.append("3. Why memoization helps")
+    lines.append(f"   {stats.cache_hits} of {stats.total_lookups} lookups were cache "
+                 f"hits ({stats.hit_rate:.0%}); each hit avoided recomputing an "
+                 "entire sub-problem.")
+
+    lines.append("")
+    lines.append("4. Complexity")
+    lines.append(f"   {estimate_complexity(problem)}")
+
+    lines.append("")
+    lines.append("5. Optimization suggestions")
+    for s in suggestions(problem):
+        lines.append(f"   - {s}")
+
+    return "\n".join(lines)