pipefy 0.0.0__tar.gz

pipefy-0.0.0/PKG-INFO

@@ -0,0 +1,7 @@
+Metadata-Version: 2.4
+Name: pipefy
+Version: 0.0.0
+Summary: A lightweight pipeline framework with validation, middleware, logging and reports.
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: polars

pipefy-0.0.0/pyproject.toml

@@ -0,0 +1,26 @@
+[project]
+name = "pipefy"
+dynamic = ["version"]
+description = "A lightweight pipeline framework with validation, middleware, logging and reports."
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "polars",
+]
+
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools_scm]
+version_scheme = "guess-next-dev"
+local_scheme = "no-local-version"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+pipefy = ["py.typed"]

pipefy-0.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

pipefy-0.0.0/src/pipefy/__init__.py

@@ -0,0 +1,8 @@
+from .context import Context, ErrorContext
+from .node import Node
+from .pipeline import Pipeline
+from .error import ErrorDecision
+from .report import PipelineErrorReport, PipelineNodeReport, PipelineReport, PipelineRetryReport, PipelineStepReport
+from .middleware.middleware import Middleware
+
+from .sdk import Logger, saveHTML, saveJson, getReference

pipefy-0.0.0/src/pipefy/context.py

@@ -0,0 +1,71 @@
+from pathlib import Path 
+from typing import Any
+import polars as pl 
+from typing import Callable
+
+from .sdk import logger
+
+from .error import ErrorDecision
+
+df_readers: dict[str, Callable[[Path], pl.DataFrame]] = {
+  ".xlsx": pl.read_excel,
+  ".xls": pl.read_excel,
+  ".csv": pl.read_csv,
+  ".parquet": pl.read_parquet,
+}
+
+class Context:
+  def __init__(self, log: logger.Logger):
+    self.log: logger.Logger = log
+    self.data: dict[str, Any] = {}
+
+    self.read_keys: set[str] = set()
+    self.written_keys: set[str] = set()
+
+  def begin_tracking(self) -> None:
+    self.read_keys = set()
+    self.written_keys = set()
+
+  def get(self, key: str) -> Any:
+    self.read_keys.add(key)
+
+    if key not in self.data:
+      raise KeyError(f"Context key not found: {key}")
+
+    return self.data[key]
+
+  def set(self, key: str, value: Any) -> None:
+    self.written_keys.add(key)
+    self.data[key] = value
+
+  def has(self, key: str) -> bool:
+    return key in self.data
+
+class ErrorContext:
+  def __init__(self, ctx: Context, exc: Exception):
+    self.error = exc 
+    self._ctx = ctx
+
+  def get(self, key: str):
+    return self._ctx.get(key)
+
+  def set(self, key: str, value):
+    return self._ctx.set(key, value)
+
+  def has(self, key: str):
+    return self._ctx.has(key)
+
+  def abort(self, reason: str | None = None):
+    return ErrorDecision("abort", reason or str(self.error))
+
+  def retry(self, max_retries: int = 1, reason: str | None = None):
+    return ErrorDecision("retry", reason or str(self.error), max_retries)
+
+  def continue_(self, reason: str | None = None):
+    return ErrorDecision("continue", reason or str(self.error))
+
+  def skip(self, reason: str | None = None):
+    return ErrorDecision("skip", reason or str(self.error))
+  
+  def is_(self, error_type: type[Exception]) -> bool:
+    return isinstance(self.error, error_type)

pipefy-0.0.0/src/pipefy/engine/__init__.py

@@ -0,0 +1,6 @@
+
+from .middleware_engine import MiddlewareEngine
+from .node_registry import NodeRegistry
+from .decision_engine import DecisionEngine
+from .step_runner import StepRunner
+from .node_runner import NodeRunner

pipefy-0.0.0/src/pipefy/engine/decision_engine.py

@@ -0,0 +1,116 @@
+from typing import Callable
+
+from ..context import Context
+from ..error import ErrorDecision
+from ..report import PipelineStepReport, PipelineNodeReport, PipelineRetryReport
+from ..engine.middleware_engine import MiddlewareEngine
+
+class DecisionEngine:
+  def __init__(self, middleware: MiddlewareEngine):
+    self.middleware = middleware
+
+  def finish_step(
+    self,
+    step_report: PipelineStepReport,
+    node_report: PipelineNodeReport,
+    success: bool,
+    fixed: bool | None = None,
+  ) -> bool:
+    step_report.success = success
+    step_report.fixed = fixed
+
+    if not success:
+      node_report.success = False
+
+    node_report.steps.append(step_report)
+    return success
+
+  def handle(
+    self,
+    ctx: Context,
+    action: Callable[[Context], None],
+    decision: ErrorDecision,
+    step_report: PipelineStepReport,
+    node_report: PipelineNodeReport,
+  ) -> bool:
+    step_report.decision = decision.action
+    step_report.decision_reason = decision.reason
+
+    handlers = {
+      "continue": self._continue,
+      "skip": self._skip,
+      "abort": self._abort,
+      "retry": self._retry,
+    }
+
+    handler = handlers.get(decision.action)
+
+    if handler is None:
+      step_report.decision_reason = (
+        step_report.decision_reason
+        or f"Unknown error decision: {decision.action}"
+      )
+      return self.finish_step(step_report, node_report, False, False)
+
+    return handler(ctx, action, decision, step_report, node_report)
+
+  def _continue(self, ctx, action, decision, step_report, node_report):
+    return self.finish_step(step_report, node_report, True, True)
+
+  def _skip(self, ctx, action, decision, step_report, node_report):
+    return self.finish_step(step_report, node_report, True, False)
+
+  def _abort(self, ctx, action, decision, step_report, node_report):
+    return self.finish_step(step_report, node_report, False, False)
+
+  def _retry(self, ctx, action, decision, step_report, node_report):
+    max_retries = decision.max_retries or 1
+
+    for attempt in range(1, max_retries + 1):
+      retry_report = PipelineRetryReport(
+        attempt=attempt,
+        success=True,
+      )
+
+      self.middleware.emit(
+        "beforeRetry",
+        ctx,
+        retry_report,
+        attempt,
+        max_retries,
+        action,
+      )
+
+      try:
+        action(ctx)
+
+        retry_report.success = True
+        step_report.retries.append(retry_report)
+
+        self.middleware.emit(
+          "afterRetry",
+          ctx,
+          retry_report,
+          attempt,
+          max_retries,
+          action,
+        )
+
+        return self.finish_step(step_report, node_report, True, True)
+
+      except Exception as retry_err:
+        retry_report.success = False
+
+        self.middleware.emit(
+          "onRetryError",
+          ctx,
+          retry_report,
+          retry_err,
+          attempt,
+          max_retries,
+          action,
+        )
+
+        step_report.retries.append(retry_report)
+
+    return self.finish_step(step_report, node_report, False, False)

pipefy-0.0.0/src/pipefy/engine/middleware_engine.py

@@ -0,0 +1,42 @@
+from ..middleware.middleware import Middleware, MiddlewareEvent
+
+class MiddlewareEngine:
+  def __init__(self, *mw: Middleware):
+    self.middlewares = mw
+    self.listeners: dict[str, list[Middleware]] = self._index_middlewares(self.middlewares)
+
+  def _index_middlewares(self, middlewares: tuple[Middleware, ...]):
+    listeners = {}
+
+    for mw in middlewares:
+      for event in mw.events:
+        listeners.setdefault(event, []).append(mw)
+
+    return listeners
+
+  def emit(self, hook: MiddlewareEvent, *args) -> None:
+    for mw in self._listeners(hook):
+      getattr(mw, hook)(*args)
+
+  def every(self, hook: MiddlewareEvent, *args) -> bool:
+    for mw in self._listeners(hook):
+      result = getattr(mw, hook)(*args)
+
+      if result is False:
+        return False
+
+    return True
+
+  def emit_transform(self, hook: MiddlewareEvent, value):
+    result = value
+
+    for mw in self._listeners(hook):
+      result = getattr(mw, hook)(result)
+
+      if result is None:
+        raise TypeError(f"{mw.__class__.__name__}.{hook} must return a value")
+
+    return result
+
+  def _listeners(self, hook: MiddlewareEvent):
+    return self.listeners.get(hook, ())

pipefy-0.0.0/src/pipefy/engine/node_registry.py

@@ -0,0 +1,15 @@
+from typing import Self
+
+from ..node import Node
+from .middleware_engine import MiddlewareEngine
+
+class NodeRegistry:
+  def __init__(self, middleware: MiddlewareEngine):
+    self.middleware = middleware
+    self.nodes: list[Node] = []
+
+  def push(self, n: Node) -> Self:
+    self.nodes.append(
+      self.middleware.emit_transform("onPush", n)
+    )
+    return self

pipefy-0.0.0/src/pipefy/engine/node_runner.py

@@ -0,0 +1,75 @@
+from ..context import Context
+from ..node import Node
+from ..report import PipelineNodeReport
+from .step_runner import StepRunner
+from .middleware_engine import MiddlewareEngine
+
+
+class NodeRunner:
+  def __init__(
+    self,
+    middleware: MiddlewareEngine,
+    steps: StepRunner,
+  ):
+    self.middleware = middleware
+    self.steps = steps
+
+  def run(
+    self,
+    ctx: Context,
+    current_node: Node,
+    node_report: PipelineNodeReport,
+  ) -> bool:
+    self.middleware.emit("beforeRunNode", ctx, node_report, current_node)
+
+    prev_keys = set(ctx.data.keys())
+
+    try:
+      if not self.middleware.every(
+        "validateInputs",
+        ctx,
+        current_node,
+        node_report,
+      ):
+        return False
+
+      if not self.steps.run(
+        ctx,
+        "onPreRun",
+        current_node.onPreRun,
+        current_node.onPreRunErr,
+        node_report,
+      ):
+        return False
+
+      if not self.steps.run(
+        ctx,
+        "onRun",
+        current_node.onRun,
+        current_node.onRunErr,
+        node_report,
+      ):
+        return False
+
+      if not self.steps.run(
+        ctx,
+        "onPostRun",
+        current_node.onPostRun,
+        current_node.onPostRunErr,
+        node_report,
+      ):
+        return False
+
+      if not self.middleware.every(
+        "validateOutputs",
+        ctx,
+        current_node,
+        node_report,
+        prev_keys,
+      ):
+        return False
+
+      return True
+
+    finally:
+      self.middleware.emit("afterRunNode", ctx, node_report, current_node)

pipefy-0.0.0/src/pipefy/engine/step_runner.py

@@ -0,0 +1,90 @@
+from typing import Callable
+
+from ..context import Context, ErrorContext
+from ..error import ErrorDecision
+from ..report import PipelineNodeReport, PipelineStepReport
+from ..sdk import reference
+
+from .decision_engine import DecisionEngine
+from .middleware_engine import MiddlewareEngine
+
+class StepRunner:
+  def __init__(
+    self,
+    middleware: MiddlewareEngine,
+    decisions: DecisionEngine,
+  ):
+    self.middleware = middleware
+    self.decisions = decisions
+
+  def run(
+    self,
+    ctx: Context,
+    name: str,
+    action: Callable[[Context], None],
+    error_handler: Callable[[ErrorContext], ErrorDecision],
+    node_report: PipelineNodeReport,
+  ) -> bool:
+    step_report = PipelineStepReport(
+      name=name,
+      reference=reference.getReference(action),
+      success=True,
+    )
+
+    self.middleware.emit("beforeRunStep", ctx, step_report, name, action)
+
+    try:
+      action(ctx)
+
+      result = self.decisions.finish_step(
+        step_report,
+        node_report,
+        True,
+      )
+
+      self.middleware.emit("afterRunStep", ctx, step_report, name, action)
+      return result
+
+    except Exception as err:
+      err_ctx = ErrorContext(ctx, err)
+
+      self.middleware.emit("onStepError", ctx, step_report, err, error_handler)
+
+      try:
+        decision = error_handler(err_ctx)
+
+        if decision is None:
+          decision = err_ctx.abort(f"{name} must return an ErrorDecision")
+
+      except Exception as handler_err:
+        self.middleware.emit(
+          "onStepError",
+          ctx,
+          step_report,
+          handler_err,
+          error_handler,
+        )
+
+        step_report.decision = "abort"
+        step_report.decision_reason = "error handler failed"
+
+        result = self.decisions.finish_step(
+          step_report,
+          node_report,
+          False,
+          False,
+        )
+
+        self.middleware.emit("afterRunStep", ctx, step_report, name, action)
+        return result
+
+      result = self.decisions.handle(
+        ctx=ctx,
+        action=action,
+        decision=decision,
+        step_report=step_report,
+        node_report=node_report,
+      )
+
+      self.middleware.emit("afterRunStep", ctx, step_report, name, action)
+      return result

pipefy-0.0.0/src/pipefy/error.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+from typing import Literal
+
+ErrorAction = Literal["abort", "retry", "skip", "continue"]
+
+@dataclass
+class ErrorDecision:
+  action: ErrorAction 
+  reason: str | None = None 
+  max_retries: int = 0

pipefy-0.0.0/src/pipefy/middleware/__init__.py

@@ -0,0 +1,5 @@
+from .middleware import Middleware
+from .error_report import ErrorReportMiddleware
+from .logger import LoggerMiddleware
+from .report_trace import ReportTraceMiddleware
+from .validation import ValidationMiddleware

pipefy-0.0.0/src/pipefy/middleware/error_report.py

@@ -0,0 +1,31 @@
+from ..report import PipelineErrorReport
+from ..sdk import getReference
+from .middleware import Middleware
+import traceback 
+
+class ErrorReportMiddleware(Middleware):
+  events = (
+    "onStepError",
+    "onRetryError",
+  )
+  def build_error_report(self, err: Exception) -> PipelineErrorReport:
+    cause = err.__cause__ or err.__context__
+
+    return PipelineErrorReport(
+      type=type(err).__name__,
+      message=str(err),
+      traceback="".join(
+        traceback.format_exception(type(err), err, err.__traceback__)
+      ),
+      cause_type=type(cause).__name__ if cause else None,
+      cause_message=str(cause) if cause else None,
+    )
+
+  def onStepError(self, ctx, r, err, error_handler):
+    r.success = False
+    r.error = self.build_error_report(err)
+    r.handler_reference = getReference(error_handler)
+
+  def onRetryError(self, ctx, r, err, attempt, max_retries, action):
+    r.success = False
+    r.error = self.build_error_report(err)

pipefy-0.0.0/src/pipefy/middleware/logger.py

@@ -0,0 +1,47 @@
+from ..sdk import getReference
+from .middleware import Middleware
+
+class LoggerMiddleware(Middleware):
+  events = (
+    "beforeRunPipeline",
+    "afterRunPipeline",
+    "beforeRunNode",
+    "afterRunNode",
+    "beforeRunStep",
+    "afterRunStep",
+    "onStepError",
+    "beforeRetry",
+    "afterRetry",
+    "onRetryError",
+  )
+    
+  def beforeRunPipeline(self, ctx, r):
+    ctx.log.info("pipeline", "started")
+
+  def afterRunPipeline(self, ctx, r):
+    ctx.log.info("pipeline", "success" if r.success else "failed")
+
+  def beforeRunNode(self, ctx, r, n):
+    ctx.log.info(n.id, "started")
+
+  def afterRunNode(self, ctx, r, n):
+    ctx.log.info(n.id, "success" if r.success else "failed")
+
+  def beforeRunStep(self, ctx, r, name, action):
+    ctx.log.info(getReference(action), "started")
+
+  def afterRunStep(self, ctx, r, name, action):
+    ctx.log.info(getReference(action), "success" if r.success else "failed")
+
+  def onStepError(self, ctx, r, err, error_handler):
+    ctx.log.error(r.reference, type(err).__name__, str(err))
+
+  def beforeRetry(self, ctx, r, attempt, max_retries, action):
+    ctx.log.info(getReference(action), f"retry {attempt}/{max_retries}")
+
+  def afterRetry(self, ctx, r, attempt, max_retries, action):
+    ctx.log.info(getReference(action), f"retry {attempt}/{max_retries} success")
+
+  def onRetryError(self, ctx, r, err, attempt, max_retries, action):
+    ctx.log.error(getReference(action), f"retry {attempt}/{max_retries} failed:", err)
+

pipefy-0.0.0/src/pipefy/middleware/middleware.py

@@ -0,0 +1,82 @@
+from abc import ABC
+from typing import Callable, Literal
+
+from ..context import Context
+from ..node import Node
+from ..report import  PipelineNodeReport, PipelineReport, PipelineRetryReport, PipelineStepReport
+
+MiddlewareEvent = Literal[
+  "onPush",
+  "beforeRunPipeline",
+  "afterRunPipeline",
+  "beforeRunNode",
+  "afterRunNode",
+  "beforeRunStep",
+  "afterRunStep",
+  "onStepError",
+  "validateInputs",
+  "validateOutputs",
+  "beforeRetry",
+  "afterRetry",
+  "onRetryError",
+]
+
+class Middleware(ABC):
+  events: tuple[MiddlewareEvent, ...] = (
+    "onPush",
+    "beforeRunPipeline",
+    "afterRunPipeline",
+    "beforeRunNode",
+    "afterRunNode",
+    "beforeRunStep",
+    "afterRunStep",
+    "onStepError",
+    "validateInputs",
+    "validateOutputs",
+    "beforeRetry",
+    "afterRetry",
+    "onRetryError",
+  )
+
+  def listens(self, event: MiddlewareEvent) -> bool:
+    return event in self.events
+
+  def onPush(self, n: Node) -> Node:
+    return n
+
+  def beforeRunPipeline(self, ctx: Context, r: PipelineReport) -> None:
+    pass
+
+  def afterRunPipeline(self, ctx: Context, r: PipelineReport) -> None:
+    pass
+
+  def beforeRunNode(self, ctx: Context, r: PipelineNodeReport, n: Node) -> None:
+    pass
+
+  def afterRunNode(self, ctx: Context, r: PipelineNodeReport, n: Node) -> None:
+    pass
+
+  def beforeRunStep(self, ctx: Context, r: PipelineStepReport, name: str, action: Callable) -> None:
+    pass
+
+  def afterRunStep(self, ctx: Context, r: PipelineStepReport, name: str, action: Callable) -> None:
+    pass
+
+  def onStepError(self, ctx: Context, r: PipelineStepReport, err: Exception, error_handler: Callable) -> None:
+    pass
+
+  def validateInputs(self, ctx: Context, n: Node, r: PipelineNodeReport) -> bool:
+    return True
+
+  def validateOutputs(self, ctx: Context, n: Node, r: PipelineNodeReport, prev_keys: set[str]) -> bool:
+    return True
+
+  def beforeRetry(self, ctx: Context, r: PipelineRetryReport, attempt: int, max_retries: int, action: Callable) -> None:
+    pass
+
+  def afterRetry(self, ctx: Context, r: PipelineRetryReport, attempt: int, max_retries: int, action: Callable) -> None:
+    pass
+
+  def onRetryError(self, ctx: Context, r: PipelineRetryReport, err: Exception, attempt: int, max_retries: int, action: Callable) -> None:
+    pass
+