reflet 0.1.0__py3-none-any.whl

reflet/__init__.py

@@ -0,0 +1,36 @@
+"""reflet: a bidirectional data transformation engine with built-in provenance."""
+
+from reflet.errors import (
+    ConflictError,
+    InsertError,
+    ReadOnlyFieldError,
+    RefletError,
+    SourceConflictError,
+    UnknownFieldError,
+    UnsupportedOperationError,
+    ValidationError,
+)
+from reflet.lens import ConflictPolicy
+from reflet.pipeline import Pipeline, Source
+from reflet.view import Change, FieldProxy, JournalEntry, Row, View
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Source",
+    "Pipeline",
+    "View",
+    "Row",
+    "FieldProxy",
+    "Change",
+    "JournalEntry",
+    "ConflictPolicy",
+    "RefletError",
+    "ReadOnlyFieldError",
+    "ConflictError",
+    "UnknownFieldError",
+    "SourceConflictError",
+    "ValidationError",
+    "InsertError",
+    "UnsupportedOperationError",
+]

reflet/__main__.py

@@ -0,0 +1,3 @@
+from reflet.cli import main
+
+main()

reflet/cli.py

@@ -0,0 +1,67 @@
+"""Command-line interface: `python -m reflet show|why ...`."""
+
+from __future__ import annotations
+
+import argparse
+import runpy
+import sys
+
+from reflet.declarative import pipeline_from_json
+from reflet.pipeline import Pipeline
+from reflet.view import View
+
+
+def _load_pipeline(script: str) -> Pipeline:
+    if script.endswith(".json"):
+        return pipeline_from_json(script)
+    namespace    = runpy.run_path(script)
+    pipeline     = namespace.get("pipeline")
+    if pipeline is None or not isinstance(pipeline, Pipeline):
+        raise SystemExit(f"{script}: no module-level `pipeline = rf.Source(...)...` found")
+    return pipeline
+
+
+def _run(script: str) -> View:
+    return _load_pipeline(script).run()
+
+
+def cmd_show(args: argparse.Namespace) -> None:
+    view = _run(args.script)
+    if len(view) == 0:
+        print("(empty view)")
+        return
+    fields = list(view._table[0].fields.keys())
+    print("\t".join(fields))
+    for row in view:
+        print("\t".join(str(getattr(row, f)) for f in fields))
+
+
+def cmd_why(args: argparse.Namespace) -> None:
+    view     = _run(args.script)
+    row      = view[args.row]
+    field    = getattr(row, args.field)
+    print(field.why())
+
+
+def main(argv: list[str] | None = None) -> None:
+    parser        = argparse.ArgumentParser(prog="python -m reflet")
+    subparsers    = parser.add_subparsers(dest="command", required=True)
+
+    script_help = "path to a .py file defining `pipeline`, or a declarative .json spec"
+
+    show_parser = subparsers.add_parser("show", help="run a pipeline and print its view")
+    show_parser.add_argument("script", help=script_help)
+    show_parser.set_defaults(func=cmd_show)
+
+    why_parser = subparsers.add_parser("why", help="print a field's provenance")
+    why_parser.add_argument("script", help=script_help)
+    why_parser.add_argument("row", type=int, help="row index in the view")
+    why_parser.add_argument("field", help="field name")
+    why_parser.set_defaults(func=cmd_why)
+
+    args = parser.parse_args(argv)
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

reflet/declarative.py

@@ -0,0 +1,133 @@
+"""Build a Pipeline from a JSON spec, without writing Python.
+
+Only the lenses expressible without an arbitrary callable are supported:
+`rename`, `select`, `sort`, `constant`, `aggregate` (with a named agg
+function), `filter` (a single field/comparison/value), and `map` (a named
+numeric transform: scale or offset). `compute`/`validate`, and a `map` with
+custom logic, need real Python — build those with `reflet.Source(...)`
+directly and treat the declarative spec as a starting point, not the whole
+pipeline.
+
+There's no reverse (Pipeline -> spec): a Pipeline built the normal way holds
+arbitrary Python closures for filter/map/compute, which have no general
+serialization. This module only builds pipelines from specs, not the other
+way around.
+
+Spec shape:
+
+    {
+      "source": {"path": "clients.csv", "key": "id"},
+      "steps": [
+        {"op": "filter", "field": "country", "cmp": "==", "value": "FR"},
+        {"op": "rename", "mapping": {"customer": "name"}},
+        {"op": "map", "field": "price", "transform": "scale", "factor": 1.2},
+        {"op": "sort", "field": "customer"}
+      ]
+    }
+"""
+
+from __future__ import annotations
+
+import json
+import operator
+from collections.abc import Callable
+from typing import Any
+
+from reflet.lens import ConflictPolicy
+from reflet.lenses import RowView
+from reflet.pipeline import Pipeline, Source
+
+_CMP_OPS: dict[str, Callable[[Any, Any], bool]] = {
+    "==": operator.eq,
+    "!=": operator.ne,
+    "<": operator.lt,
+    "<=": operator.le,
+    ">": operator.gt,
+    ">=": operator.ge,
+    "in": lambda a, b: a in b,
+    "not in": lambda a, b: a not in b,
+}
+
+_AGG_FNS: dict[str, Callable[[list[Any]], Any]] = {
+    "sum": sum,
+    "min": min,
+    "max": max,
+    "count": len,
+    "avg": lambda xs: sum(xs) / len(xs),
+}
+
+
+def _compile_predicate(step: dict[str, Any]) -> Callable[[RowView], bool]:
+    field = step["field"]
+    try:
+        cmp = _CMP_OPS[step["cmp"]]
+    except KeyError as exc:
+        raise ValueError(f"declarative filter: unknown cmp {step['cmp']!r}") from exc
+    value = step["value"]
+
+    def predicate(row: RowView) -> bool:
+        return cmp(getattr(row, field), value)
+
+    return predicate
+
+
+def _compile_map(step: dict[str, Any]) -> tuple[Callable[[Any], Any], Callable[[Any], Any]]:
+    transform = step["transform"]
+    if transform == "scale":
+        factor = step["factor"]
+        return (lambda x, factor=factor: x * factor, lambda x, factor=factor: x / factor)
+    if transform == "offset":
+        amount = step["amount"]
+        return (lambda x, amount=amount: x + amount, lambda x, amount=amount: x - amount)
+    raise ValueError(f"declarative map: unknown transform {transform!r}")
+
+
+def pipeline_from_dict(spec: dict[str, Any]) -> Pipeline:
+    """Build a `Pipeline` from an already-parsed declarative spec dict."""
+    src = spec["source"]
+    pipeline = Source(
+        src["path"],
+        key                = src.get("key"),
+        tombstone_field    = src.get("tombstone_field"),
+        table              = src.get("table"),
+        rows_at            = src.get("rows_at"),
+    )
+
+    for step in spec.get("steps", []):
+        op = step["op"]
+        if op == "filter":
+            pipeline = pipeline.filter(_compile_predicate(step), defaults=step.get("defaults"))
+        elif op == "rename":
+            pipeline = pipeline.rename(**step["mapping"])
+        elif op == "select":
+            pipeline = pipeline.select(*step["fields"])
+        elif op == "sort":
+            pipeline = pipeline.sort(step["field"], reverse=step.get("reverse", False))
+        elif op == "constant":
+            pipeline = pipeline.constant(step["field"], step["value"])
+        elif op == "map":
+            fwd, bwd    = _compile_map(step)
+            pipeline    = pipeline.map(step["field"], fwd=fwd, bwd=bwd)
+        elif op == "aggregate":
+            try:
+                agg_fn = _AGG_FNS[step["agg_fn"]]
+            except KeyError as exc:
+                name = step["agg_fn"]
+                raise ValueError(f"declarative aggregate: unknown agg_fn {name!r}") from exc
+            pipeline = pipeline.aggregate(
+                step["group_by"],
+                step["agg_field"],
+                agg_fn,
+                source_field    = step.get("source_field"),
+                policy          = ConflictPolicy(step.get("policy", "error")),
+            )
+        else:
+            raise ValueError(f"unknown declarative step: {op!r}")
+
+    return pipeline
+
+
+def pipeline_from_json(path: str) -> Pipeline:
+    """Build a `Pipeline` from a declarative spec file."""
+    with open(path, encoding="utf-8") as fh:
+        return pipeline_from_dict(json.load(fh))

reflet/errors.py

@@ -0,0 +1,72 @@
+"""Exceptions raised by reflet."""
+
+from __future__ import annotations
+
+
+class RefletError(Exception):
+    """Base class for all reflet errors."""
+
+
+class ReadOnlyFieldError(RefletError):
+    """Raised when attempting to write to a field produced by a non-invertible lens."""
+
+    def __init__(self, field: str) -> None:
+        super().__init__(
+            f"field {field!r} is read-only: it was produced by a non-invertible "
+            "operation (e.g. compute/constant/aggregate/join) and has no inverse "
+            "to propagate through"
+        )
+        self.field = field
+
+
+class ConflictError(RefletError):
+    """Raised when an output row maps back to several source rows under the 'error' policy."""
+
+    def __init__(self, count: int) -> None:
+        super().__init__(
+            f"ambiguous write-back: output row maps to {count} source rows; "
+            "pass a conflict policy ('first', 'last') to resolve this explicitly"
+        )
+        self.count = count
+
+
+class UnknownFieldError(RefletError):
+    """Raised when referencing a field that does not exist on a row."""
+
+    def __init__(self, field: str) -> None:
+        super().__init__(f"no such field: {field!r}")
+        self.field = field
+
+
+class SourceConflictError(RefletError):
+    """Raised when the source file changed on disk since it was read."""
+
+    def __init__(self, path: str) -> None:
+        super().__init__(
+            f"{path!r} changed on disk since it was read; pass force=True to "
+            "sync() to overwrite it anyway, or re-run the pipeline to pick up "
+            "the external changes first"
+        )
+        self.path = path
+
+
+class ValidationError(RefletError):
+    """Raised when an edit fails a `.validate()` constraint, before it touches the source."""
+
+    def __init__(
+        self, field: str, value: object, message: str | None = None, why: str | None = None
+    ) -> None:
+        text = message or f"{field}={value!r} failed validation"
+        if why:
+            text = f"{text} ({why})"
+        super().__init__(text)
+        self.field    = field
+        self.value    = value
+
+
+class InsertError(RefletError):
+    """Raised when `View.insert()` can't be resolved back to a valid source row."""
+
+
+class UnsupportedOperationError(RefletError):
+    """Raised when an operation isn't supported for the current lens chain."""

reflet/lens.py

@@ -0,0 +1,113 @@
+"""Core bidirectional lens abstraction, operating on tables of records."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Hashable
+from dataclasses import dataclass, replace
+from enum import Enum
+from typing import Any
+
+from reflet.errors import ConflictError, ReadOnlyFieldError
+from reflet.provenance import Provenance
+
+# A row's stable identity: its position (no key given) or a key field's value.
+RowKey = Hashable
+# A row identifier: a single row key, or a tuple of several when a lens
+# (e.g. aggregate) merges multiple source rows into one output row.
+RowId = RowKey | tuple[RowKey, ...]
+
+
+class ConflictPolicy(str, Enum):
+    """How to resolve a write to a field shared by several source rows."""
+
+    FIRST    = "first"
+    LAST     = "last"
+    ERROR    = "error"
+
+
+@dataclass(frozen=True)
+class Cell:
+    """A single field value together with its provenance."""
+
+    value: Any
+    prov: Provenance
+    readonly: bool                     = False
+    conflict: ConflictPolicy | None    = None
+    conflict_size: int                 = 1
+
+    def transform(self, fn: Any, step: str, *, readonly: bool = False) -> Cell:
+        return Cell(fn(self.value), self.prov.step(step), readonly=readonly)
+
+    def require_writable(self, field: str) -> None:
+        if self.readonly:
+            raise ReadOnlyFieldError(field)
+        if self.conflict == ConflictPolicy.ERROR:
+            raise ConflictError(self.conflict_size)
+
+
+@dataclass(frozen=True)
+class Record:
+    """A single row: its (possibly composite) identity plus its field cells."""
+
+    rid: RowId
+    fields: dict[str, Cell]
+
+    def with_fields(self, fields: dict[str, Cell]) -> Record:
+        return replace(self, fields=fields)
+
+
+Table = list[Record]
+
+
+class Lens(ABC):
+    """A bidirectional transformation from one table shape to another.
+
+    `get` produces the forward view; `put` takes an edited view (`a`, built by
+    editing the result of `get(s)`) together with the original table `s` and
+    reconstructs the updated `s`.
+    """
+
+    name: str = "lens"
+
+    @abstractmethod
+    def get(self, s: Table) -> Table: ...
+
+    @abstractmethod
+    def put(self, a: Table, s: Table) -> Table: ...
+
+    def __rshift__(self, other: Lens) -> Lens:
+        return ComposedLens(self, other)
+
+
+class ComposedLens(Lens):
+    """Sequential composition of two lenses: self first, then other."""
+
+    def __init__(self, first: Lens, second: Lens) -> None:
+        self.first     = first
+        self.second    = second
+        self.name      = f"{first.name}>>{second.name}"
+
+    def get(self, s: Table) -> Table:
+        return self.second.get(self.first.get(s))
+
+    def put(self, a: Table, s: Table) -> Table:
+        mid        = self.first.get(s)
+        new_mid    = self.second.put(a, mid)
+        return self.first.put(new_mid, s)
+
+
+class IdentityLens(Lens):
+    """The no-op lens; used when a pipeline has no transformation steps yet."""
+
+    name = "identity"
+
+    def get(self, s: Table) -> Table:
+        return list(s)
+
+    def put(self, a: Table, s: Table) -> Table:
+        return list(a)
+
+
+def index_by_rid(table: Table) -> dict[RowId, Record]:
+    return {r.rid: r for r in table}