pineapple-pine 0.7.0__py3-none-any.whl

pine/config.py

@@ -0,0 +1,304 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from typing import Any
+
+from pine.errors import ConfigError
+
+RESERVED_KEYS = frozenset({
+    "type_name", "$metadata", "$code_info", "skip", "recall", "sources",
+    "debug", "consumes_row_set", "mutates_row_set",
+    "additive_writes_row_set", "common_defaults", "item_defaults",
+    "for_branch_control", "data_parallel",
+})
+
+
+@dataclass
+class Metadata:
+    common_input: list[str] = field(default_factory=list)
+    common_output: list[str] = field(default_factory=list)
+    item_input: list[str] = field(default_factory=list)
+    item_output: list[str] = field(default_factory=list)
+
+
+@dataclass
+class OperatorConfig:
+    """Per-operator configuration within a pipeline."""
+
+    type_name: str = ""
+    metadata: Metadata = field(default_factory=Metadata)
+    skip: list[str] = field(default_factory=list)
+    recall: bool = False
+    sources: list[str] = field(default_factory=list)
+    debug: bool = False
+    consumes_row_set: bool = False
+    mutates_row_set: bool = False
+    additive_writes_row_set: bool = False
+    for_branch_control: bool = False
+    data_parallel: int = 1
+    common_defaults: dict[str, Any] = field(default_factory=dict)
+    item_defaults: dict[str, Any] = field(default_factory=dict)
+    raw_params: dict[str, Any] = field(default_factory=dict)
+    operator_type: str = ""
+    input_spec: InputFieldSpec | None = None
+
+
+class DefaultedField:
+    __slots__ = ("name", "default")
+
+    def __init__(self, name: str, default: Any):
+        self.name = name
+        self.default = default
+
+
+class InputFieldSpec:
+    """Resolved input field specification with defaults and strict fields."""
+
+    __slots__ = ("strict_common", "defaulted_common", "strict_item", "defaulted_item")
+
+    def __init__(
+        self,
+        strict_common: list[str],
+        defaulted_common: list[DefaultedField],
+        strict_item: list[str],
+        defaulted_item: list[DefaultedField],
+    ):
+        self.strict_common = strict_common
+        self.defaulted_common = defaulted_common
+        self.strict_item = strict_item
+        self.defaulted_item = defaulted_item
+
+    @staticmethod
+    def compute(metadata: "Metadata", common_defaults: dict[str, Any],
+                item_defaults: dict[str, Any], skip: list[str]) -> "InputFieldSpec":
+        skip_set = set(skip)
+        strict_common: list[str] = []
+        defaulted_common: list[DefaultedField] = []
+        for f in metadata.common_input:
+            if f in skip_set:
+                continue
+            if f in common_defaults:
+                defaulted_common.append(DefaultedField(f, common_defaults[f]))
+            else:
+                strict_common.append(f)
+
+        strict_item: list[str] = []
+        defaulted_item: list[DefaultedField] = []
+        for f in metadata.item_input:
+            if f in item_defaults:
+                defaulted_item.append(DefaultedField(f, item_defaults[f]))
+            else:
+                strict_item.append(f)
+
+        return InputFieldSpec(strict_common, defaulted_common, strict_item, defaulted_item)
+
+
+class SubFlowRef:
+    def __init__(self, pipeline: list[str] | None = None):
+        self.pipeline: list[str] = pipeline or []
+
+
+class FlowContract:
+    """Declares the pipeline's required inputs and guaranteed outputs."""
+
+    def __init__(self):
+        self.common_input: list[str] = []
+        self.item_input: list[str] = []
+        self.common_output: list[str] = []
+        self.item_output: list[str] = []
+
+
+class PipelineConfig:
+    """Holds operator configs and sub-flow pipeline map."""
+
+    def __init__(self):
+        self.operators: dict[str, OperatorConfig] = {}
+        self.pipeline_map: dict[str, SubFlowRef] = {}
+
+
+class ExpandResult:
+    def __init__(self, sequence: list[str], op_to_sub_flow: dict[str, str]):
+        self.sequence = sequence
+        self.op_to_sub_flow = op_to_sub_flow
+
+
+class Config:
+    """Pipeline configuration parsed from JSON."""
+
+    def __init__(self):
+        self.pineapple_version: str = ""
+        self.log_prefix: str = ""
+        self.debug: bool = False
+        self.storage_mode: str = "row"
+        self.pipeline_config: PipelineConfig = PipelineConfig()
+        self.pipeline_group: dict[str, SubFlowRef] = {}
+        self.flow_contract: FlowContract = FlowContract()
+
+    @classmethod
+    def load(cls, json_data: bytes) -> "Config":
+        try:
+            root = json.loads(json_data)
+        except Exception as e:
+            raise ConfigError(f"failed to parse config JSON: {e}")
+
+        cfg = Config()
+        cfg.pineapple_version = root.get("_PINEAPPLE_VERSION", "")
+        cfg.log_prefix = root.get("log_prefix", "")
+        cfg.debug = root.get("debug", False)
+        cfg.storage_mode = root.get("storage_mode", "row")
+
+        fc = root.get("flow_contract", {})
+        cfg.flow_contract.common_input = fc.get("common_input", [])
+        cfg.flow_contract.item_input = fc.get("item_input", [])
+        cfg.flow_contract.common_output = fc.get("common_output", [])
+        cfg.flow_contract.item_output = fc.get("item_output", [])
+
+        pg = root.get("pipeline_group", {})
+        for name, val in pg.items():
+            cfg.pipeline_group[name] = SubFlowRef(val.get("pipeline", []))
+
+        pc = root.get("pipeline_config", {})
+        pm = pc.get("pipeline_map", {})
+        for name, val in pm.items():
+            cfg.pipeline_config.pipeline_map[name] = SubFlowRef(val.get("pipeline", []))
+
+        ops = pc.get("operators", {})
+        for name, op_node in ops.items():
+            op_cfg = _parse_operator_config(op_node)
+            cfg.pipeline_config.operators[name] = op_cfg
+
+        _validate(cfg)
+        return cfg
+
+    def expand_operator_sequence_with_sub_flows(self) -> ExpandResult:
+        if "main" in self.pipeline_group:
+            group = self.pipeline_group["main"]
+        elif len(self.pipeline_group) == 1:
+            group = next(iter(self.pipeline_group.values()))
+        else:
+            raise ConfigError(
+                'pipeline_group must contain a "main" entry or exactly one entry'
+            )
+
+        for name in self.pipeline_config.operators:
+            if name in self.pipeline_config.pipeline_map:
+                raise ConfigError(
+                    f'name "{name}" exists in both operators and pipeline_map'
+                )
+
+        sequence: list[str] = []
+        op_to_sub_flow: dict[str, str] = {}
+        visiting: set[str] = set()
+        seen: set[str] = set()
+
+        self._expand_entries(
+            group.pipeline, "", sequence, op_to_sub_flow, visiting, seen
+        )
+        return ExpandResult(sequence, op_to_sub_flow)
+
+    def _expand_entries(
+        self,
+        entries: list[str],
+        parent_path: str,
+        sequence: list[str],
+        op_to_sub_flow: dict[str, str],
+        visiting: set[str],
+        seen: set[str],
+    ):
+        for entry in entries:
+            if entry in self.pipeline_config.operators:
+                if entry in seen:
+                    raise ConfigError(
+                        f'operator "{entry}" referenced more than once in pipeline tree'
+                    )
+                seen.add(entry)
+                sequence.append(entry)
+                op_to_sub_flow[entry] = parent_path
+            elif entry in self.pipeline_config.pipeline_map:
+                if entry in visiting:
+                    raise ConfigError(
+                        f'cycle detected in sub-flow expansion: "{entry}"'
+                    )
+                visiting.add(entry)
+                self._expand_entries(
+                    self.pipeline_config.pipeline_map[entry].pipeline,
+                    entry,
+                    sequence,
+                    op_to_sub_flow,
+                    visiting,
+                    seen,
+                )
+                visiting.discard(entry)
+            else:
+                raise ConfigError(
+                    f'pipeline entry "{entry}" is neither an operator nor a sub-flow'
+                )
+
+
+def _parse_operator_config(node: dict[str, Any]) -> OperatorConfig:
+    op_cfg = OperatorConfig()
+    op_cfg.type_name = node.get("type_name", "")
+    op_cfg.recall = node.get("recall", False)
+    op_cfg.debug = node.get("debug", False)
+    op_cfg.consumes_row_set = node.get("consumes_row_set", False)
+    op_cfg.mutates_row_set = node.get("mutates_row_set", False)
+    op_cfg.additive_writes_row_set = node.get("additive_writes_row_set", False)
+    op_cfg.for_branch_control = node.get("for_branch_control", False)
+    op_cfg.data_parallel = node.get("data_parallel", 1)
+    op_cfg.sources = node.get("sources", [])
+
+    skip = node.get("skip", [])
+    if isinstance(skip, str):
+        op_cfg.skip = [skip] if skip else []
+    elif isinstance(skip, list):
+        op_cfg.skip = [str(s) for s in skip]
+    else:
+        op_cfg.skip = []
+
+    meta = node.get("$metadata", {})
+    op_cfg.metadata.common_input = meta.get("common_input", [])
+    op_cfg.metadata.common_output = meta.get("common_output", [])
+    op_cfg.metadata.item_input = meta.get("item_input", [])
+    op_cfg.metadata.item_output = meta.get("item_output", [])
+
+    op_cfg.common_defaults = node.get("common_defaults", {})
+    op_cfg.item_defaults = node.get("item_defaults", {})
+
+    op_cfg.raw_params = {}
+    for key, value in node.items():
+        if key not in RESERVED_KEYS:
+            op_cfg.raw_params[key] = value
+
+    return op_cfg
+
+
+def _validate(cfg: Config):
+    if not cfg.pipeline_config.operators:
+        raise ConfigError("pipeline_config.operators is empty")
+    if not cfg.pipeline_group:
+        raise ConfigError("pipeline_group is empty")
+
+    for name, op_cfg in cfg.pipeline_config.operators.items():
+        if not op_cfg.type_name:
+            raise ConfigError(f'operator "{name}": missing type_name')
+
+    for name, op_cfg in cfg.pipeline_config.operators.items():
+        for src in op_cfg.sources:
+            if src not in cfg.pipeline_config.operators:
+                raise ConfigError(
+                    f'operator "{name}": sources references undefined operator "{src}"'
+                )
+
+    for name, op_cfg in cfg.pipeline_config.operators.items():
+        for skip_field in op_cfg.skip:
+            if not skip_field.startswith("_"):
+                raise ConfigError(
+                    f'operator "{name}": skip field "{skip_field}" must start with '
+                    "'_' (control fields are engine-internal)"
+                )
+            if skip_field not in op_cfg.metadata.common_input:
+                raise ConfigError(
+                    f'operator "{name}": skip field "{skip_field}" must also appear '
+                    "in $metadata.common_input to ensure correct DAG ordering"
+                )

pine/dag.py

@@ -0,0 +1,218 @@
+from __future__ import annotations
+
+from collections import deque
+
+from pine.config import OperatorConfig
+from pine.errors import ConfigError
+
+_ROW_SET_SENTINEL = "_row_set_"
+
+
+class _FieldTracker:
+    def __init__(self):
+        self.last_mut_writer: int = -1
+        self.additive_writers: list[int] = []
+        self.active_readers: list[int] = []
+
+
+class Node:
+    """Single operator node within the DAG."""
+
+    def __init__(self, name: str, index: int, sub_flow: str, config: OperatorConfig):
+        self.name = name
+        self.index = index
+        self.sub_flow = sub_flow
+        self.config = config
+        self.preds: list[int] = []
+        self.succs: list[int] = []
+
+
+class DAG:
+    """Directed acyclic graph encoding operator execution dependencies."""
+
+    def __init__(self, nodes: list[Node], name_to_index: dict[str, int]):
+        self.nodes = nodes
+        self.name_to_index = name_to_index
+
+    @classmethod
+    def build(
+        cls,
+        sequence: list[str],
+        operators: dict[str, OperatorConfig],
+        op_to_sub_flow: dict[str, str],
+    ) -> "DAG":
+        nodes: list[Node] = []
+        name_to_index: dict[str, int] = {}
+
+        for i, name in enumerate(sequence):
+            op_cfg = operators.get(name)
+            if op_cfg is None:
+                raise ConfigError(f'operator "{name}" not found')
+            node = Node(name, i, op_to_sub_flow.get(name, ""), op_cfg)
+            nodes.append(node)
+            name_to_index[name] = i
+
+        g = cls(nodes, name_to_index)
+
+        _add_edges(g, sequence, operators, is_common=True)
+        _add_edges(g, sequence, operators, is_common=False)
+
+        for i, name in enumerate(sequence):
+            op_cfg = operators[name]
+            for src in op_cfg.sources:
+                src_idx = name_to_index.get(src)
+                if src_idx is None:
+                    raise ConfigError(
+                        f'operator "{name}" sources references unknown operator "{src}"'
+                    )
+                _add_edge(g, src_idx, i)
+
+        _reduce(g)
+        _topological_sort(g)
+
+        return g
+
+    def topological_order(self) -> list[int]:
+        return _topological_sort(self)
+
+
+def _add_edge(g: DAG, from_idx: int, to_idx: int):
+    if from_idx == to_idx:
+        return
+    if to_idx in g.nodes[from_idx].succs:
+        return
+    g.nodes[from_idx].succs.append(to_idx)
+    g.nodes[to_idx].preds.append(from_idx)
+
+
+def _add_edges(
+    g: DAG,
+    sequence: list[str],
+    operators: dict[str, OperatorConfig],
+    is_common: bool,
+):
+    fields: dict[str, _FieldTracker] = {}
+
+    for i, name in enumerate(sequence):
+        op_cfg = operators[name]
+        meta = op_cfg.metadata
+
+        read_fields = list(meta.common_input if is_common else meta.item_input)
+        write_fields = list(meta.common_output if is_common else meta.item_output)
+        is_additive_write = not is_common and op_cfg.additive_writes_row_set
+
+        if not is_common:
+            if is_additive_write:
+                write_fields.append(_ROW_SET_SENTINEL)
+            if op_cfg.consumes_row_set:
+                read_fields.append(_ROW_SET_SENTINEL)
+
+        # RAW edges
+        for field in read_fields:
+            ft = fields.setdefault(field, _FieldTracker())
+            if ft.last_mut_writer >= 0:
+                _add_edge(g, ft.last_mut_writer, i)
+            for aw in ft.additive_writers:
+                _add_edge(g, aw, i)
+            ft.active_readers.append(i)
+
+        # WAR + WAW edges
+        for field in write_fields:
+            ft = fields.setdefault(field, _FieldTracker())
+            if is_additive_write:
+                if ft.last_mut_writer >= 0:
+                    _add_edge(g, ft.last_mut_writer, i)
+                for reader in ft.active_readers:
+                    if reader != i:
+                        _add_edge(g, reader, i)
+                ft.additive_writers.append(i)
+            else:
+                if ft.last_mut_writer >= 0:
+                    _add_edge(g, ft.last_mut_writer, i)
+                for aw in ft.additive_writers:
+                    _add_edge(g, aw, i)
+                for reader in ft.active_readers:
+                    if reader != i:
+                        _add_edge(g, reader, i)
+                ft.last_mut_writer = i
+                ft.additive_writers.clear()
+                ft.active_readers.clear()
+
+        # MutatesRowSet: mutating write to _ROW_SET_SENTINEL
+        if not is_common and op_cfg.mutates_row_set:
+            ft = fields.setdefault(_ROW_SET_SENTINEL, _FieldTracker())
+            if ft.last_mut_writer >= 0:
+                _add_edge(g, ft.last_mut_writer, i)
+            for aw in ft.additive_writers:
+                _add_edge(g, aw, i)
+            for reader in ft.active_readers:
+                if reader != i:
+                    _add_edge(g, reader, i)
+            ft.last_mut_writer = i
+            ft.additive_writers.clear()
+            ft.active_readers.clear()
+
+
+def _topological_sort(g: DAG) -> list[int]:
+    n = len(g.nodes)
+    in_degree = [len(node.preds) for node in g.nodes]
+    queue: deque[int] = deque()
+    for i in range(n):
+        if in_degree[i] == 0:
+            queue.append(i)
+
+    order: list[int] = []
+    while queue:
+        curr = queue.popleft()
+        order.append(curr)
+        for succ in g.nodes[curr].succs:
+            in_degree[succ] -= 1
+            if in_degree[succ] == 0:
+                queue.append(succ)
+
+    if len(order) != n:
+        cycle_nodes = [g.nodes[i].name for i in range(n) if in_degree[i] > 0]
+        raise ConfigError(
+            f"DAG contains a cycle involving operators: {cycle_nodes}"
+        )
+    return order
+
+
+def _reduce(g: DAG):
+    n = len(g.nodes)
+    kept: list[tuple[int, int]] = []
+
+    for u in range(n):
+        for v in g.nodes[u].succs:
+            if not _reachable_without(g, u, v):
+                kept.append((u, v))
+
+    for node in g.nodes:
+        node.preds.clear()
+        node.succs.clear()
+    for u, v in kept:
+        g.nodes[u].succs.append(v)
+        g.nodes[v].preds.append(u)
+
+
+def _reachable_without(g: DAG, src: int, dst: int) -> bool:
+    visited = set()
+    visited.add(src)
+    queue: deque[int] = deque()
+
+    for next_node in g.nodes[src].succs:
+        if next_node == dst:
+            continue
+        if next_node not in visited:
+            visited.add(next_node)
+            queue.append(next_node)
+
+    while queue:
+        cur = queue.popleft()
+        if cur == dst:
+            return True
+        for next_node in g.nodes[cur].succs:
+            if next_node not in visited:
+                visited.add(next_node)
+                queue.append(next_node)
+    return False