pyworkflow-engine 0.1.7__py3-none-any.whl → 0.1.10__py3-none-any.whl

pyworkflow/cli/utils/docker_manager.py

@@ -195,6 +195,131 @@ volumes:
     return template
 
 
+def generate_cassandra_docker_compose_content(
+    cassandra_host: str = "cassandra",
+    cassandra_port: int = 9042,
+    cassandra_keyspace: str = "pyworkflow",
+    cassandra_user: str | None = None,
+    cassandra_password: str | None = None,
+) -> str:
+    """
+    Generate docker-compose.yml content for Cassandra-based PyWorkflow services.
+
+    Args:
+        cassandra_host: Cassandra host (container name for docker networking)
+        cassandra_port: Cassandra CQL native transport port
+        cassandra_keyspace: Cassandra keyspace name
+        cassandra_user: Optional Cassandra user (for authentication)
+        cassandra_password: Optional Cassandra password (for authentication)
+
+    Returns:
+        docker-compose.yml content as string
+
+    Example:
+        >>> compose_content = generate_cassandra_docker_compose_content()
+    """
+    # Build environment variables for dashboard
+    dashboard_env = f"""      - DASHBOARD_STORAGE_TYPE=cassandra
+      - DASHBOARD_CASSANDRA_HOST={cassandra_host}
+      - DASHBOARD_CASSANDRA_PORT=9042
+      - DASHBOARD_CASSANDRA_KEYSPACE={cassandra_keyspace}"""
+
+    if cassandra_user and cassandra_password:
+        dashboard_env += f"""
+      - DASHBOARD_CASSANDRA_USER={cassandra_user}
+      - DASHBOARD_CASSANDRA_PASSWORD={cassandra_password}"""
+
+    # Cassandra environment for authentication
+    cassandra_env = ""
+    if cassandra_user and cassandra_password:
+        cassandra_env = f"""
+      - CASSANDRA_AUTHENTICATOR=PasswordAuthenticator
+      - CASSANDRA_AUTHORIZER=CassandraAuthorizer
+      - CASSANDRA_USER={cassandra_user}
+      - CASSANDRA_PASSWORD={cassandra_password}"""
+
+    template = f"""services:
+  cassandra:
+    image: cassandra:4.1
+    container_name: pyworkflow-cassandra
+    ports:
+      - "{cassandra_port}:9042"
+    environment:
+      - CASSANDRA_CLUSTER_NAME=pyworkflow-cluster
+      - CASSANDRA_DC=dc1
+      - CASSANDRA_RACK=rack1
+      - CASSANDRA_ENDPOINT_SNITCH=GossipingPropertyFileSnitch
+      - MAX_HEAP_SIZE=512M
+      - HEAP_NEWSIZE=128M{cassandra_env}
+    volumes:
+      - cassandra_data:/var/lib/cassandra
+    healthcheck:
+      test: ["CMD-SHELL", "cqlsh -e 'describe cluster' || exit 1"]
+      interval: 15s
+      timeout: 10s
+      retries: 10
+      start_period: 60s
+    restart: unless-stopped
+
+  redis:
+    image: redis:7-alpine
+    container_name: pyworkflow-redis
+    ports:
+      - "6379:6379"
+    volumes:
+      - redis_data:/data
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 5s
+      timeout: 3s
+      retries: 5
+    restart: unless-stopped
+
+  dashboard-backend:
+    image: yashabro/pyworkflow-dashboard-backend:latest
+    container_name: pyworkflow-dashboard-backend
+    working_dir: /app/project
+    ports:
+      - "8585:8585"
+    environment:
+      - DASHBOARD_PYWORKFLOW_CONFIG_PATH=/app/project/pyworkflow.config.yaml
+{dashboard_env}
+      - DASHBOARD_HOST=0.0.0.0
+      - DASHBOARD_PORT=8585
+      - DASHBOARD_CORS_ORIGINS=["http://localhost:5173","http://localhost:3000"]
+      - PYWORKFLOW_CELERY_BROKER=redis://redis:6379/0
+      - PYWORKFLOW_CELERY_RESULT_BACKEND=redis://redis:6379/1
+      - PYTHONPATH=/app/project
+    volumes:
+      - .:/app/project:ro
+    depends_on:
+      cassandra:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    restart: unless-stopped
+
+  dashboard-frontend:
+    image: yashabro/pyworkflow-dashboard-frontend:latest
+    container_name: pyworkflow-dashboard-frontend
+    ports:
+      - "5173:80"
+    environment:
+      - VITE_API_URL=http://localhost:8585
+    depends_on:
+      - dashboard-backend
+    restart: unless-stopped
+
+volumes:
+  cassandra_data:
+    driver: local
+  redis_data:
+    driver: local
+"""
+
+    return template
+
+
 def generate_postgres_docker_compose_content(
     postgres_host: str = "postgres",
     postgres_port: int = 5432,
@@ -301,6 +426,113 @@ volumes:
     return template
 
 
+def generate_mysql_docker_compose_content(
+    mysql_host: str = "mysql",
+    mysql_port: int = 3306,
+    mysql_user: str = "pyworkflow",
+    mysql_password: str = "pyworkflow",
+    mysql_database: str = "pyworkflow",
+) -> str:
+    """
+    Generate docker-compose.yml content for MySQL-based PyWorkflow services.
+
+    Args:
+        mysql_host: MySQL host (container name for docker networking)
+        mysql_port: MySQL port
+        mysql_user: MySQL user
+        mysql_password: MySQL password
+        mysql_database: MySQL database name
+
+    Returns:
+        docker-compose.yml content as string
+
+    Example:
+        >>> compose_content = generate_mysql_docker_compose_content()
+    """
+    template = f"""services:
+  mysql:
+    image: mysql:8.0
+    container_name: pyworkflow-mysql
+    ports:
+      - "{mysql_port}:3306"
+    environment:
+      - MYSQL_ROOT_PASSWORD={mysql_password}
+      - MYSQL_USER={mysql_user}
+      - MYSQL_PASSWORD={mysql_password}
+      - MYSQL_DATABASE={mysql_database}
+    volumes:
+      - mysql_data:/var/lib/mysql
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "{mysql_user}", "-p{mysql_password}"]
+      interval: 5s
+      timeout: 3s
+      retries: 5
+    restart: unless-stopped
+
+  redis:
+    image: redis:7-alpine
+    container_name: pyworkflow-redis
+    ports:
+      - "6379:6379"
+    volumes:
+      - redis_data:/data
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 5s
+      timeout: 3s
+      retries: 5
+    restart: unless-stopped
+
+  dashboard-backend:
+    image: yashabro/pyworkflow-dashboard-backend:latest
+    container_name: pyworkflow-dashboard-backend
+    working_dir: /app/project
+    ports:
+      - "8585:8585"
+    environment:
+      - DASHBOARD_PYWORKFLOW_CONFIG_PATH=/app/project/pyworkflow.config.yaml
+      - DASHBOARD_STORAGE_TYPE=mysql
+      - DASHBOARD_MYSQL_HOST={mysql_host}
+      - DASHBOARD_MYSQL_PORT=3306
+      - DASHBOARD_MYSQL_USER={mysql_user}
+      - DASHBOARD_MYSQL_PASSWORD={mysql_password}
+      - DASHBOARD_MYSQL_DATABASE={mysql_database}
+      - DASHBOARD_HOST=0.0.0.0
+      - DASHBOARD_PORT=8585
+      - DASHBOARD_CORS_ORIGINS=["http://localhost:5173","http://localhost:3000"]
+      - PYWORKFLOW_CELERY_BROKER=redis://redis:6379/0
+      - PYWORKFLOW_CELERY_RESULT_BACKEND=redis://redis:6379/1
+      - PYTHONPATH=/app/project
+    volumes:
+      - .:/app/project:ro
+    depends_on:
+      mysql:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    restart: unless-stopped
+
+  dashboard-frontend:
+    image: yashabro/pyworkflow-dashboard-frontend:latest
+    container_name: pyworkflow-dashboard-frontend
+    ports:
+      - "5173:80"
+    environment:
+      - VITE_API_URL=http://localhost:8585
+    depends_on:
+      - dashboard-backend
+    restart: unless-stopped
+
+volumes:
+  mysql_data:
+    driver: local
+  redis_data:
+    driver: local
+"""
+
+    return template
+
+
 def write_docker_compose(content: str, path: Path) -> None:
     """
     Write docker-compose.yml to file.

pyworkflow/config.py

@@ -5,8 +5,9 @@ Provides global configuration for runtime, storage, and default settings.
 
 Configuration is loaded in this priority order:
 1. Values set via pyworkflow.configure() (highest priority)
-2. Values from pyworkflow.config.yaml in current directory
-3. Default values
+2. Environment variables (PYWORKFLOW_*)
+3. Values from pyworkflow.config.yaml in current directory
+4. Default values
 
 Usage:
     >>> import pyworkflow
@@ -15,8 +16,26 @@ Usage:
     ...     default_durable=False,
     ...     storage=InMemoryStorageBackend(),
     ... )
+
+Environment Variables:
+    PYWORKFLOW_STORAGE_TYPE: Storage backend type (file, memory, sqlite, postgres, mysql)
+    PYWORKFLOW_STORAGE_PATH: Path for file/sqlite backends
+    PYWORKFLOW_POSTGRES_HOST: PostgreSQL host
+    PYWORKFLOW_POSTGRES_PORT: PostgreSQL port
+    PYWORKFLOW_POSTGRES_USER: PostgreSQL user
+    PYWORKFLOW_POSTGRES_PASSWORD: PostgreSQL password
+    PYWORKFLOW_POSTGRES_DATABASE: PostgreSQL database
+    PYWORKFLOW_MYSQL_HOST: MySQL host
+    PYWORKFLOW_MYSQL_PORT: MySQL port
+    PYWORKFLOW_MYSQL_USER: MySQL user
+    PYWORKFLOW_MYSQL_PASSWORD: MySQL password
+    PYWORKFLOW_MYSQL_DATABASE: MySQL database
+    PYWORKFLOW_CELERY_BROKER: Celery broker URL
+    PYWORKFLOW_CELERY_RESULT_BACKEND: Celery result backend URL
+    PYWORKFLOW_RUNTIME: Default runtime (local, celery)
 """
 
+import os
 import warnings
 from dataclasses import dataclass
 from pathlib import Path
@@ -26,6 +45,54 @@ if TYPE_CHECKING:
     from pyworkflow.storage.base import StorageBackend
 
 
+def _load_env_storage_config() -> dict[str, Any] | None:
+    """
+    Load storage configuration from environment variables.
+
+    Returns:
+        Storage configuration dict if PYWORKFLOW_STORAGE_TYPE is set, None otherwise
+    """
+    storage_type = os.getenv("PYWORKFLOW_STORAGE_TYPE") or os.getenv("PYWORKFLOW_STORAGE_BACKEND")
+    if not storage_type:
+        return None
+
+    storage_type = storage_type.lower()
+
+    if storage_type == "postgres":
+        return {
+            "type": "postgres",
+            "host": os.getenv("PYWORKFLOW_POSTGRES_HOST", "localhost"),
+            "port": int(os.getenv("PYWORKFLOW_POSTGRES_PORT", "5432")),
+            "user": os.getenv("PYWORKFLOW_POSTGRES_USER", "pyworkflow"),
+            "password": os.getenv("PYWORKFLOW_POSTGRES_PASSWORD", ""),
+            "database": os.getenv("PYWORKFLOW_POSTGRES_DATABASE", "pyworkflow"),
+        }
+    elif storage_type == "mysql":
+        return {
+            "type": "mysql",
+            "host": os.getenv("PYWORKFLOW_MYSQL_HOST", "localhost"),
+            "port": int(os.getenv("PYWORKFLOW_MYSQL_PORT", "3306")),
+            "user": os.getenv("PYWORKFLOW_MYSQL_USER", "pyworkflow"),
+            "password": os.getenv("PYWORKFLOW_MYSQL_PASSWORD", ""),
+            "database": os.getenv("PYWORKFLOW_MYSQL_DATABASE", "pyworkflow"),
+        }
+    elif storage_type == "sqlite":
+        return {
+            "type": "sqlite",
+            "base_path": os.getenv("PYWORKFLOW_STORAGE_PATH", "./pyworkflow_data/pyworkflow.db"),
+        }
+    elif storage_type == "memory":
+        return {"type": "memory"}
+    elif storage_type == "file":
+        return {
+            "type": "file",
+            "base_path": os.getenv("PYWORKFLOW_STORAGE_PATH", "./pyworkflow_data"),
+        }
+    else:
+        # Unknown type, return as-is and let config_to_storage handle it
+        return {"type": storage_type}
+
+
 def _load_yaml_config() -> dict[str, Any]:
     """
     Load configuration from pyworkflow.config.yaml in current directory.
@@ -101,23 +168,33 @@ class PyWorkflowConfig:
     event_warning_interval: int = 100  # Log warning every N events after soft limit
 
 
-def _config_from_yaml() -> PyWorkflowConfig:
-    """Create a PyWorkflowConfig from YAML file settings."""
-    yaml_config = _load_yaml_config()
+def _config_from_env_and_yaml() -> PyWorkflowConfig:
+    """
+    Create a PyWorkflowConfig from environment variables and YAML file.
 
-    if not yaml_config:
-        return PyWorkflowConfig()
+    Priority:
+    1. Environment variables (PYWORKFLOW_*)
+    2. YAML config file (pyworkflow.config.yaml)
+    3. Defaults
+    """
+    yaml_config = _load_yaml_config()
+    env_storage_config = _load_env_storage_config()
 
-    # Map YAML keys to config attributes
-    runtime = yaml_config.get("runtime", "local")
+    # Runtime: env var > yaml > default
+    runtime = os.getenv("PYWORKFLOW_RUNTIME") or yaml_config.get("runtime", "local")
     durable = runtime == "celery"  # Celery runtime defaults to durable
 
-    # Create storage from config
-    storage = _create_storage_from_config(yaml_config.get("storage", {}))
+    # Storage: env var > yaml > None
+    if env_storage_config:
+        storage = _create_storage_from_config(env_storage_config)
+    elif yaml_config.get("storage"):
+        storage = _create_storage_from_config(yaml_config.get("storage", {}))
+    else:
+        storage = None
 
-    # Get celery broker
+    # Celery broker: env var > yaml > None
     celery_config = yaml_config.get("celery", {})
-    celery_broker = celery_config.get("broker")
+    celery_broker = os.getenv("PYWORKFLOW_CELERY_BROKER") or celery_config.get("broker")
 
     return PyWorkflowConfig(
         default_runtime=runtime,
@@ -288,16 +365,16 @@ def get_config() -> PyWorkflowConfig:
     """
     Get the current configuration.
 
-    If not yet configured, loads from pyworkflow.config.yaml if present,
-    otherwise creates default configuration.
+    If not yet configured, loads from environment variables and
+    pyworkflow.config.yaml (env vars take priority).
 
     Returns:
         Current PyWorkflowConfig instance
     """
     global _config, _config_loaded_from_yaml
     if _config is None:
-        # Try to load from YAML config file first
-        _config = _config_from_yaml()
+        # Load from env vars and YAML config file
+        _config = _config_from_env_and_yaml()
         _config_loaded_from_yaml = True
     return _config
 

pyworkflow/context/__init__.py

@@ -41,6 +41,13 @@ from pyworkflow.context.base import (
 )
 from pyworkflow.context.local import LocalContext
 from pyworkflow.context.mock import MockContext
+from pyworkflow.context.step_context import (
+    StepContext,
+    get_step_context,
+    get_step_context_class,
+    has_step_context,
+    set_step_context,
+)
 
 __all__ = [
     # Base context and helpers
@@ -49,6 +56,12 @@ __all__ = [
     "has_context",
     "set_context",
     "reset_context",
+    # Step context for distributed execution
+    "StepContext",
+    "get_step_context",
+    "has_step_context",
+    "set_step_context",
+    "get_step_context_class",
     # Context implementations
     "LocalContext",
     "MockContext",

pyworkflow/context/base.py

@@ -281,6 +281,32 @@ class WorkflowContext(ABC):
         """Get the storage backend."""
         return None
 
+    @property
+    def runtime(self) -> str | None:
+        """
+        Get the runtime environment slug.
+
+        Returns the runtime identifier (e.g., "celery", "temporal") or None
+        for local/inline execution. Used to determine step dispatch behavior.
+
+        Returns:
+            Runtime slug string or None for local execution
+        """
+        return None  # Default: local/inline execution
+
+    @property
+    def storage_config(self) -> dict[str, Any] | None:
+        """Get storage configuration for distributed workers."""
+        return None
+
+    def has_step_failed(self, step_id: str) -> bool:
+        """Check if a step has a recorded failure."""
+        return False  # Default: no failures
+
+    def get_step_failure(self, step_id: str) -> dict[str, Any] | None:
+        """Get failure info for a step."""
+        return None  # Default: no failure info
+
     def should_execute_step(self, step_id: str) -> bool:
         """Check if step should be executed (not already completed)."""
         return True  # Default: always execute

pyworkflow/context/local.py

@@ -97,6 +97,13 @@ class LocalContext(WorkflowContext):
         self._child_results: dict[str, dict[str, Any]] = {}
         self._pending_children: dict[str, str] = {}  # child_id -> child_run_id
 
+        # Runtime environment (e.g., "celery", "temporal", None for local)
+        self._runtime: str | None = None
+        self._storage_config: dict[str, Any] | None = None
+
+        # Step failure tracking (for handling failures during replay)
+        self._step_failures: dict[str, dict[str, Any]] = {}
+
         # Replay state if resuming
         if event_log:
             self._is_replaying = True
@@ -134,6 +141,22 @@ class LocalContext(WorkflowContext):
                     "last_error": event.data.get("error", ""),
                 }
 
+            elif event.type == EventType.STEP_FAILED:
+                # Track step failures for distributed step dispatch
+                # If a step failed (not retrying), record it for replay detection
+                step_id = event.data.get("step_id")
+                is_retryable = event.data.get("is_retryable", True)
+                # Only track as a final failure if not retryable or if this is
+                # the last failure before a STEP_COMPLETED event
+                # For now, track all failures - the step decorator will check
+                # if there's also a STEP_COMPLETED event
+                if step_id and not is_retryable:
+                    self._step_failures[step_id] = {
+                        "error": event.data.get("error", "Unknown error"),
+                        "error_type": event.data.get("error_type", "Exception"),
+                        "is_retryable": is_retryable,
+                    }
+
             elif event.type == EventType.CANCELLATION_REQUESTED:
                 self._cancellation_requested = True
                 self._cancellation_reason = event.data.get("reason")
@@ -208,6 +231,28 @@ class LocalContext(WorkflowContext):
         """Set replay mode."""
         self._is_replaying = value
 
+    @property
+    def runtime(self) -> str | None:
+        """
+        Get the runtime environment slug.
+
+        Returns the runtime identifier (e.g., "celery", "temporal") or None
+        for local/inline execution. Used to determine step dispatch behavior.
+
+        Returns:
+            Runtime slug string or None for local execution
+        """
+        return self._runtime
+
+    @property
+    def storage_config(self) -> dict[str, Any] | None:
+        """
+        Get storage configuration for distributed workers.
+
+        This is passed to step workers so they can connect to the same storage.
+        """
+        return self._storage_config
+
     # =========================================================================
     # Step result caching (for @step decorator compatibility)
     # =========================================================================
@@ -255,6 +300,41 @@ class LocalContext(WorkflowContext):
         """Clear retry state for a step."""
         self._retry_states.pop(step_id, None)
 
+    # =========================================================================
+    # Step failure tracking (for distributed step dispatch)
+    # =========================================================================
+
+    def has_step_failed(self, step_id: str) -> bool:
+        """
+        Check if a step has a recorded failure.
+
+        Used during replay to detect steps that failed on a remote worker.
+        """
+        return step_id in self._step_failures
+
+    def get_step_failure(self, step_id: str) -> dict[str, Any] | None:
+        """
+        Get step failure info.
+
+        Returns:
+            Dict with 'error', 'error_type', 'is_retryable' or None
+        """
+        return self._step_failures.get(step_id)
+
+    def record_step_failure(
+        self,
+        step_id: str,
+        error: str,
+        error_type: str,
+        is_retryable: bool,
+    ) -> None:
+        """Record a step failure for replay detection."""
+        self._step_failures[step_id] = {
+            "error": error,
+            "error_type": error_type,
+            "is_retryable": is_retryable,
+        }
+
     # =========================================================================
     # Sleep state management (for @step decorator and EventReplayer compatibility)
     # =========================================================================