hexdag 0.7.0.dev12__tar.gz → 0.7.0.dev13__tar.gz

{hexdag-0.7.0.dev12 → hexdag-0.7.0.dev13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hexdag
-Version: 0.7.0.dev12
+Version: 0.7.0.dev13
 Summary: Lightweight DAG orchestration framework with enterprise pipeline capabilities
 Project-URL: Homepage, https://hexdag.ai
 Project-URL: Repository, https://github.com/omniviser/hexdag

{hexdag-0.7.0.dev12 → hexdag-0.7.0.dev13}/hexdag/compiler/system_builder.py

@@ -125,6 +125,12 @@ class SystemBuilder:
                 "ports": spec.get("ports", {}),
                 "policies": spec.get("policies", {}),
                 "services": spec.get("services", {}),
+                "state_machines": spec.get("state_machines", {}),
+                "states": spec.get("states", {}),
+                "on_transition": spec.get("on_transition", {}),
+                "observers": spec.get("observers", []),
+                "memory": spec.get("memory", {}),
+                "gc": spec.get("gc", {}),
             })
         except Exception as e:
             raise SystemBuildError(f"System validation error: {e}") from e
@@ -132,15 +138,23 @@ class SystemBuilder:
         # 5. Validate pipeline paths exist
         self._validate_pipeline_paths(system_config, resolved_base)
 
-        # 6. Validate pipe DAG is acyclic
-        self._validate_no_cycles(system_config)
+        # 6. Validate pipe DAG is acyclic (only for non-lifecycle systems)
+        if not system_config.is_lifecycle:
+            self._validate_no_cycles(system_config)
+
+        # 7. Validate lifecycle constraints (if state machines declared)
+        if system_config.is_lifecycle:
+            self._validate_lifecycle(system_config)
 
         system_name = metadata.get("name", "unnamed")
+        mode = "lifecycle" if system_config.is_lifecycle else "dag"
         logger.info(
-            "Built system '{}': {} processes, {} pipes",
+            "Built system '{}' ({}): {} processes, {} pipes, {} state machines",
             system_name,
+            mode,
             len(system_config.processes),
             len(system_config.pipes),
+            len(system_config.state_machines),
         )
 
         return system_config
@@ -201,6 +215,75 @@ class SystemBuilder:
                 "Cycle detected in pipe DAG. Processes must form a directed acyclic graph."
             )
 
+    @staticmethod
+    def _validate_lifecycle(config: SystemConfig) -> None:
+        """Validate lifecycle-mode constraints.
+
+        Ensures that ``states`` and ``on_transition`` reference valid
+        process names and valid state machine states.
+
+        Raises
+        ------
+        SystemBuildError
+            If lifecycle configuration is invalid.
+        """
+        process_names = set(config.process_names)
+
+        # Collect all valid states from state machines
+        all_states: set[str] = set()
+        for sm_spec in config.state_machines.values():
+            transitions = sm_spec.get("transitions", {})
+            for from_state, targets in transitions.items():
+                all_states.add(from_state)
+                if isinstance(targets, list):
+                    for t in targets:
+                        if isinstance(t, str):
+                            all_states.add(t)
+                        elif isinstance(t, dict) and "to" in t:
+                            all_states.add(t["to"])
+            initial = sm_spec.get("initial")
+            if initial:
+                all_states.add(initial)
+
+        # Validate spec.states references
+        for state_name, state_spec in config.states.items():
+            if state_name not in all_states:
+                raise SystemBuildError(
+                    f"spec.states references unknown state '{state_name}'. "
+                    f"Valid states: {sorted(all_states)}"
+                )
+            on_enter = state_spec.get("on_enter")
+            if on_enter and on_enter not in process_names:
+                raise SystemBuildError(
+                    f"State '{state_name}' references unknown process "
+                    f"'{on_enter}'. Declared processes: {sorted(process_names)}"
+                )
+
+        # Validate spec.on_transition references
+        for transition_key, transition_spec in config.on_transition.items():
+            parts = [p.strip() for p in transition_key.split("->")]
+            if len(parts) != 2:  # noqa: PLR2004
+                raise SystemBuildError(
+                    f"Invalid on_transition key '{transition_key}'. "
+                    "Expected format: 'FROM_STATE -> TO_STATE'"
+                )
+            from_state, to_state = parts
+            if from_state not in all_states:
+                raise SystemBuildError(
+                    f"on_transition '{transition_key}' references unknown state '{from_state}'"
+                )
+            if to_state not in all_states:
+                raise SystemBuildError(
+                    f"on_transition '{transition_key}' references unknown state '{to_state}'"
+                )
+            process_name = transition_spec.get("process")
+            if process_name and process_name not in process_names:
+                raise SystemBuildError(
+                    f"on_transition '{transition_key}' references unknown "
+                    f"process '{process_name}'. "
+                    f"Declared processes: {sorted(process_names)}"
+                )
+
     @staticmethod
     def topological_order(config: SystemConfig) -> list[str]:
         """Compute topological execution order from the pipe DAG.

{hexdag-0.7.0.dev12 → hexdag-0.7.0.dev13}/hexdag/compiler/yaml_validator.py

@@ -323,7 +323,7 @@ class YamlValidator:
         self._validate_dependencies_with_cache(nodes, result, node_ids, macro_instances)
 
         # Validate expression/mapping naming to prevent ambiguous resolution
-        self._validate_naming_collisions(nodes, result, node_ids)
+        self._validate_naming_collisions(nodes, result, node_ids, macro_instances)
 
         return result
 
@@ -659,6 +659,7 @@ class YamlValidator:
         nodes: list[dict[str, Any]],
         result: ValidationReport,
         node_ids: set[str],
+        macro_instances: set[str],
     ) -> None:
         """Validate that expression variables and input_mapping aliases don't collide.
 
@@ -725,7 +726,7 @@ class YamlValidator:
                 if not isinstance(expr, str):
                     continue
                 self._check_first_segments(
-                    expr, var_name, node_id, valid_first_segments, node_ids, result
+                    expr, var_name, node_id, valid_first_segments, node_ids, macro_instances, result
                 )
 
             for alias, source in input_mapping.items():
@@ -735,7 +736,7 @@ class YamlValidator:
                 if source.startswith("$input") or source == "$input":
                     continue
                 self._check_first_segments(
-                    source, alias, node_id, valid_first_segments, node_ids, result
+                    source, alias, node_id, valid_first_segments, node_ids, macro_instances, result
                 )
 
     @staticmethod
@@ -745,6 +746,7 @@ class YamlValidator:
         node_id: str,
         valid_first_segments: set[str],
         node_ids: set[str],
+        macro_instances: set[str],
         result: "ValidationReport",
     ) -> None:
         """Check that first path segments in text are known references."""
@@ -754,6 +756,9 @@ class YamlValidator:
                 continue
             if candidate in valid_first_segments:
                 continue
+            # Check if candidate is a macro-expanded node name
+            if any(candidate.startswith(f"{mi}_") for mi in macro_instances):
+                continue
             # Unknown first segment — likely a typo
             close_matches = difflib.get_close_matches(candidate, sorted(node_ids), n=3, cutoff=0.6)
             suggestion = ""

{hexdag-0.7.0.dev12 → hexdag-0.7.0.dev13}/hexdag/kernel/context/__init__.py

@@ -7,6 +7,7 @@ from hexdag.kernel.context.execution_context import (
     get_current_node_name,
     get_node_results,
     get_observer_manager,
+    get_pipeline_memory,
     get_pipeline_name,
     get_port,
     get_ports,
@@ -32,6 +33,7 @@ __all__ = [
     "get_current_node_name",
     "get_node_results",
     "get_observer_manager",
+    "get_pipeline_memory",
     "get_pipeline_name",
     "get_port",
     "get_ports",

{hexdag-0.7.0.dev12 → hexdag-0.7.0.dev13}/hexdag/kernel/context/execution_context.py

@@ -279,6 +279,20 @@ def get_services() -> dict[str, Any] | None:
     return _services_context.get()
 
 
+def get_pipeline_memory() -> Any | None:
+    """Convenience getter for the auto-registered PipelineMemory service.
+
+    Returns
+    -------
+    PipelineMemory | None
+        The pipeline memory instance, or None if not available.
+    """
+    services = _services_context.get()
+    if services:
+        return services.get("pipeline_memory")
+    return None
+
+
 # ============================================================================
 # Pipeline Name Context
 # ============================================================================

{hexdag-0.7.0.dev12 → hexdag-0.7.0.dev13}/hexdag/kernel/domain/dag.py

@@ -116,6 +116,8 @@ class NodeSpec:
     retry_max_delay: float | None = None  # Maximum delay cap in seconds (default: 60.0)
     when: str | None = None  # Optional expression to evaluate before execution
     on_error: str | None = None  # Name of DAG node to run if this node fails (after retries)
+    critical: bool = False  # If True and node is skipped, pipeline fails
+    required_inputs: tuple[str, ...] | list[str] = ()  # Input fields that must be non-None
     # Factory metadata for distributed execution (set by YAML pipeline builder)
     factory_class: str | None = None  # Module path, e.g. "hexdag.stdlib.nodes.LLMNode"
     factory_params: dict[str, Any] | None = None  # Original factory **kwargs
@@ -141,6 +143,9 @@ class NodeSpec:
         object.__setattr__(self, "deps", frozenset(sys.intern(d) for d in self.deps))
         object.__setattr__(self, "params", MappingProxyType(self.params))
         object.__setattr__(self, "literals", MappingProxyType(self.literals))
+        # Normalize required_inputs to tuple for immutability
+        if isinstance(self.required_inputs, list):
+            object.__setattr__(self, "required_inputs", tuple(self.required_inputs))
         # Freeze factory_params if present
         if self.factory_params is not None:
             object.__setattr__(self, "factory_params", MappingProxyType(self.factory_params))

hexdag-0.7.0.dev13/hexdag/kernel/domain/extraction_state.py

@@ -0,0 +1,107 @@
+"""Domain models for multi-round extraction state tracking.
+
+Used by :class:`ExtractionJob <hexdag.stdlib.lib.extraction_job.ExtractionJob>`
+to track extraction progress across multiple pipeline invocations.
+
+Each extraction round is a separate pipeline run. The ``ExtractionState``
+persists between runs via ``SupportsKeyValue`` storage, tracking which
+fields have been extracted and which are still needed.
+
+OS metaphor: like a process image saved to disk (SIGSTOP + core dump)
+and resumed later (SIGCONT + restore).
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+
+from pydantic import BaseModel, Field, computed_field
+
+
+class RoundRecord(BaseModel):
+    """Snapshot of a single extraction round.
+
+    Attributes
+    ----------
+    round_number : int
+        1-based round index.
+    extracted_fields : dict[str, Any]
+        Fields successfully extracted in this round.
+    source : str
+        Description of the data source (e.g., "carrier_email", "api_response").
+    timestamp : datetime
+        When this round was executed.
+    raw_data : dict[str, Any]
+        Raw data received (for audit/debugging).
+    """
+
+    round_number: int
+    extracted_fields: dict[str, Any] = Field(default_factory=dict)
+    source: str = ""
+    timestamp: datetime = Field(default_factory=datetime.now)
+    raw_data: dict[str, Any] = Field(default_factory=dict)
+
+
+class ExtractionState(BaseModel):
+    """Tracks multi-round extraction progress for a single entity.
+
+    Accumulates extracted data across rounds and computes which required
+    fields are still missing.
+
+    Attributes
+    ----------
+    job_id : str
+        Unique extraction job identifier.
+    entity_type : str
+        Type of entity being extracted (e.g., "claim", "order").
+    entity_id : str
+        Unique entity identifier.
+    status : str
+        Current status: "pending", "extracting", "complete", "failed".
+    current_round : int
+        Number of rounds completed so far.
+    max_rounds : int
+        Maximum allowed extraction rounds before auto-failure.
+    required_fields : list[str]
+        Fields that must be extracted for completion.
+    extracted_data : dict[str, Any]
+        Accumulated extracted data across all rounds.
+    round_history : list[RoundRecord]
+        Ordered list of round snapshots.
+    failure_reason : str | None
+        Reason for failure (if status is "failed").
+    """
+
+    job_id: str
+    entity_type: str = ""
+    entity_id: str = ""
+    status: str = "pending"
+    current_round: int = 0
+    max_rounds: int = 5
+    required_fields: list[str] = Field(default_factory=list)
+    extracted_data: dict[str, Any] = Field(default_factory=dict)
+    round_history: list[RoundRecord] = Field(default_factory=list)
+    failure_reason: str | None = None
+
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def missing_fields(self) -> list[str]:
+        """Fields still needed for completion."""
+        return [
+            f
+            for f in self.required_fields
+            if f not in self.extracted_data or self.extracted_data[f] is None
+        ]
+
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def is_complete(self) -> bool:
+        """Whether all required fields have been extracted."""
+        return len(self.missing_fields) == 0
+
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def is_max_rounds_reached(self) -> bool:
+        """Whether the maximum number of rounds has been reached."""
+        return self.current_round >= self.max_rounds

{hexdag-0.7.0.dev12 → hexdag-0.7.0.dev13}/hexdag/kernel/domain/pipeline_config.py

@@ -249,6 +249,17 @@ class PipelineConfig(BaseModel):
         default_factory=dict,
         description="Service configurations. Format: {service_name: {class: str, config: dict}}",
     )
+    state_machines: dict[str, dict[str, Any]] = Field(
+        default_factory=dict,
+        description=(
+            "Entity state machine definitions. "
+            "Format: {entity_type: {initial: str, transitions: dict}}"
+        ),
+    )
+    observers: list[dict[str, Any]] = Field(
+        default_factory=list,
+        description=("Observer configurations. Format: [{class: str, config: dict}]"),
+    )
 
     # Schema declarations
     input_schema: dict[str, Any] | None = Field(

{hexdag-0.7.0.dev12 → hexdag-0.7.0.dev13}/hexdag/kernel/domain/system_config.py

@@ -160,10 +160,52 @@ class SystemConfig(BaseModel):
         default_factory=dict, description="Global service configurations"
     )
 
+    # Lifecycle extensions (opt-in: presence of state_machines activates lifecycle mode)
+    state_machines: dict[str, dict[str, Any]] = Field(
+        default_factory=dict,
+        description=(
+            "Entity state machine definitions. "
+            "Format: {entity_type: {initial: str, transitions: dict, handlers: dict}}"
+        ),
+    )
+    states: dict[str, dict[str, Any]] = Field(
+        default_factory=dict,
+        description=(
+            "State-to-process mapping. "
+            "Format: {STATE_NAME: {on_enter: process_name, terminal: bool}}"
+        ),
+    )
+    on_transition: dict[str, dict[str, Any]] = Field(
+        default_factory=dict,
+        description=(
+            "Transition-specific process overrides. Format: {'FROM -> TO': {process: process_name}}"
+        ),
+    )
+    observers: list[dict[str, Any]] = Field(
+        default_factory=list,
+        description="Observer configurations. Format: [{class: str, config: dict}]",
+    )
+    memory: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Memory configuration (preload hooks).",
+    )
+    gc: dict[str, Any] = Field(
+        default_factory=dict,
+        description=(
+            "Garbage collection config for terminal entities. "
+            "Format: {check_interval: int, obligation_timeout: int, on_failure: str}"
+        ),
+    )
+
     # ------------------------------------------------------------------
     # Derived helpers
     # ------------------------------------------------------------------
 
+    @property
+    def is_lifecycle(self) -> bool:
+        """Whether this system operates in lifecycle mode."""
+        return bool(self.state_machines)
+
     @property
     def process_names(self) -> list[str]:
         """Return ordered list of process names."""