pyworkflow-engine 0.1.24__tar.gz → 0.1.26__tar.gz

{pyworkflow_engine-0.1.24 → pyworkflow_engine-0.1.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyworkflow-engine
-Version: 0.1.24
+Version: 0.1.26
 Summary: A Python implementation of durable, event-sourced workflows inspired by Vercel Workflow
 Author: PyWorkflow Contributors
 License: MIT

{pyworkflow_engine-0.1.24 → pyworkflow_engine-0.1.26}/pyproject.toml

@@ -7,7 +7,7 @@ packages = [{include = "pyworkflow"}]
 
 [project]
 name = "pyworkflow-engine"
-version = "0.1.24"
+version = "0.1.26"
 description = "A Python implementation of durable, event-sourced workflows inspired by Vercel Workflow"
 readme = "README.md"
 requires-python = ">=3.11"

{pyworkflow_engine-0.1.24 → pyworkflow_engine-0.1.26}/pyworkflow/__init__.py

@@ -29,7 +29,7 @@ Quick Start:
     >>> run_id = await start(my_workflow, "Alice")
 """
 
-__version__ = "0.1.24"
+__version__ = "0.1.26"
 
 # Configuration
 from pyworkflow.config import (

{pyworkflow_engine-0.1.24 → pyworkflow_engine-0.1.26}/pyworkflow/celery/app.py

@@ -202,11 +202,16 @@ def create_celery_app(
     """
     # Priority: parameter > environment variable > hardcoded default
     broker_url = broker_url or os.getenv("PYWORKFLOW_CELERY_BROKER") or "redis://localhost:6379/0"
-    result_backend = (
-        result_backend
-        or os.getenv("PYWORKFLOW_CELERY_RESULT_BACKEND")
-        or "redis://localhost:6379/1"
-    )
+
+    # Result backend defaults to None (disabled) unless explicitly set
+    if result_backend is None and "PYWORKFLOW_CELERY_RESULT_BACKEND" not in os.environ:
+        result_backend = None  # Disabled by default
+    else:
+        result_backend = (
+            result_backend
+            or os.getenv("PYWORKFLOW_CELERY_RESULT_BACKEND")
+            or "redis://localhost:6379/1"
+        )
 
     # Worker memory limits (KB) - prevents memory leaks from accumulating
     # Priority: parameter > env var > None (no limit by default)
@@ -218,7 +223,7 @@ def create_celery_app(
 
     # Detect broker and backend types
     is_sentinel_broker = is_sentinel_url(broker_url)
-    is_sentinel_backend = is_sentinel_url(result_backend)
+    is_sentinel_backend = is_sentinel_url(result_backend) if result_backend else False
     is_redis_broker = broker_url.startswith("redis://") or broker_url.startswith("rediss://")
 
     # Get Sentinel master name from param, env, or default
@@ -255,34 +260,44 @@ def create_celery_app(
             **(result_backend_transport_options or {}),
         }
 
-    app = Celery(
-        app_name,
-        broker=broker_url,
-        backend=result_backend,
-        include=[
-            "pyworkflow.celery.tasks",
-        ],
-    )
+    # Create Celery app - only set backend if result_backend is enabled
+    if result_backend is not None:
+        app = Celery(
+            app_name,
+            broker=broker_url,
+            backend=result_backend,
+            include=[
+                "pyworkflow.celery.tasks",
+            ],
+        )
+    else:
+        # No result backend - Celery will not store task results
+        app = Celery(
+            app_name,
+            broker=broker_url,
+            include=[
+                "pyworkflow.celery.tasks",
+            ],
+        )
 
-    # Configure Celery
-    app.conf.update(
+    # Build configuration dict
+    config_dict = {
         # Task execution settings
-        task_serializer="json",
-        result_serializer="json",
-        accept_content=["json"],
-        timezone="UTC",
-        enable_utc=True,
+        "task_serializer": "json",
+        "result_serializer": "json",
+        "accept_content": ["json"],
+        "timezone": "UTC",
+        "enable_utc": True,
         # Broker transport options - prevent task redelivery
         # See: https://github.com/celery/celery/issues/5935
-        broker_transport_options=final_broker_opts,
-        result_backend_transport_options=final_backend_opts,
+        "broker_transport_options": final_broker_opts,
         # Task routing
-        task_default_queue="pyworkflow.default",
-        task_default_exchange="pyworkflow",
-        task_default_exchange_type="topic",
-        task_default_routing_key="workflow.default",
+        "task_default_queue": "pyworkflow.default",
+        "task_default_exchange": "pyworkflow",
+        "task_default_exchange_type": "topic",
+        "task_default_routing_key": "workflow.default",
         # Task queues
-        task_queues=(
+        "task_queues": (
             Queue(
                 "pyworkflow.default",
                 Exchange("pyworkflow", type="topic"),
@@ -304,42 +319,62 @@ def create_celery_app(
                 routing_key="workflow.schedule.#",
             ),
         ),
-        # Result backend settings
-        result_expires=3600,  # 1 hour
-        result_persistent=True,
         # Task execution
-        task_acks_late=True,
-        task_reject_on_worker_lost=True,
-        worker_prefetch_multiplier=1,  # Fair task distribution
+        "task_acks_late": True,
+        "task_reject_on_worker_lost": True,
+        "worker_prefetch_multiplier": 1,  # Fair task distribution
         # Retry settings
-        task_autoretry_for=(),
-        task_retry_backoff=True,
-        task_retry_backoff_max=600,  # 10 minutes max
-        task_retry_jitter=True,
+        "task_autoretry_for": (),
+        "task_retry_backoff": True,
+        "task_retry_backoff_max": 600,  # 10 minutes max
+        "task_retry_jitter": True,
         # Monitoring
-        worker_send_task_events=True,
-        task_send_sent_event=True,
+        "worker_send_task_events": True,
+        "task_send_sent_event": True,
         # Beat scheduler (for sleep resumption)
-        beat_schedule={},
+        "beat_schedule": {},
         # Logging
-        worker_log_format="[%(asctime)s: %(levelname)s/%(processName)s] %(message)s",
-        worker_task_log_format="[%(asctime)s: %(levelname)s/%(processName)s] [%(task_name)s(%(task_id)s)] %(message)s",
+        "worker_log_format": "[%(asctime)s: %(levelname)s/%(processName)s] %(message)s",
+        "worker_task_log_format": "[%(asctime)s: %(levelname)s/%(processName)s] [%(task_name)s(%(task_id)s)] %(message)s",
         # Worker memory management - prevents memory leaks from accumulating
         # When set, workers are recycled after exceeding these limits
-        worker_max_memory_per_child=max_memory,  # KB, None = no limit
-        worker_max_tasks_per_child=max_tasks,  # None = no limit
-    )
+        "worker_max_memory_per_child": max_memory,  # KB, None = no limit
+        "worker_max_tasks_per_child": max_tasks,  # None = no limit
+    }
+
+    # Only add result backend settings if enabled
+    if result_backend is not None:
+        config_dict.update(
+            {
+                "result_backend_transport_options": final_backend_opts,
+                "result_expires": 3600,  # 1 hour
+                "result_persistent": True,
+            }
+        )
+    else:
+        # Disable task result storage when no backend configured
+        config_dict.update(
+            {
+                "task_ignore_result": True,  # Don't try to store task results
+                "task_store_errors_even_if_ignored": False,  # Don't store errors either
+            }
+        )
+
+    # Configure Celery
+    app.conf.update(config_dict)
 
     # Configure singleton locking for Redis or Sentinel brokers
     # This enables distributed locking to prevent duplicate task execution
+    # Uses broker URL since result backend may be disabled
     if is_redis_broker or is_sentinel_broker:
-        app.conf.update(
-            singleton_backend_url=broker_url,
-            singleton_backend_is_sentinel=is_sentinel_broker,
-            singleton_sentinel_master=master_name if is_sentinel_broker else None,
-            singleton_key_prefix="pyworkflow:lock:",
-            singleton_lock_expiry=3600,  # 1 hour TTL (safety net)
-        )
+        singleton_config = {
+            "singleton_backend_url": broker_url,  # Use broker, not result backend
+            "singleton_backend_is_sentinel": is_sentinel_broker,
+            "singleton_sentinel_master": master_name if is_sentinel_broker else None,
+            "singleton_key_prefix": "pyworkflow:lock:",
+            "singleton_lock_expiry": 3600,  # 1 hour TTL (safety net)
+        }
+        app.conf.update(singleton_config)
 
     # Note: Logging is configured via Celery signals (worker_init, worker_process_init)
     # to ensure proper initialization AFTER process forking.

{pyworkflow_engine-0.1.24 → pyworkflow_engine-0.1.26}/pyworkflow/storage/postgres.py

@@ -16,6 +16,7 @@ the pool is automatically recreated when a loop change is detected.
 import asyncio
 import contextlib
 import json
+import os
 from datetime import UTC, datetime
 from typing import Any
 
@@ -163,10 +164,10 @@ class PostgresStorageBackend(StorageBackend):
         user: str = "pyworkflow",
         password: str = "",
         database: str = "pyworkflow",
-        min_pool_size: int = 1,
-        max_pool_size: int = 10,
-        max_inactive_connection_lifetime: float = 1800.0,
-        command_timeout: float | None = 60.0,
+        min_pool_size: int | None = None,
+        max_pool_size: int | None = None,
+        max_inactive_connection_lifetime: float | None = None,
+        command_timeout: float | None = None,
     ):
         """
         Initialize PostgreSQL storage backend.
@@ -178,11 +179,13 @@ class PostgresStorageBackend(StorageBackend):
             user: Database user (used if dsn not provided)
             password: Database password (used if dsn not provided)
             database: Database name (used if dsn not provided)
-            min_pool_size: Minimum connections in pool
-            max_pool_size: Maximum connections in pool
+            min_pool_size: Minimum connections in pool (defaults to env var PYWORKFLOW_POSTGRES_MIN_POOL_SIZE or 1)
+            max_pool_size: Maximum connections in pool (defaults to env var PYWORKFLOW_POSTGRES_MAX_POOL_SIZE or 10)
             max_inactive_connection_lifetime: How long (seconds) an idle connection can
-                                              stay in the pool before being closed. Default 1800s (30 min).
-            command_timeout: Default timeout (seconds) for queries. None for no timeout. Default 60s.
+                                              stay in the pool before being closed.
+                                              Defaults to env var PYWORKFLOW_POSTGRES_MAX_INACTIVE_LIFETIME or 1800s (30 min).
+            command_timeout: Default timeout (seconds) for queries. None for no timeout.
+                            Defaults to env var PYWORKFLOW_POSTGRES_COMMAND_TIMEOUT or 60s.
         """
         self.dsn = dsn
         self.host = host
@@ -190,10 +193,27 @@ class PostgresStorageBackend(StorageBackend):
         self.user = user
         self.password = password
         self.database = database
-        self.min_pool_size = min_pool_size
-        self.max_pool_size = max_pool_size
-        self.max_inactive_connection_lifetime = max_inactive_connection_lifetime
-        self.command_timeout = command_timeout
+
+        # Read from env vars if not provided
+        self.min_pool_size = min_pool_size or int(
+            os.getenv("PYWORKFLOW_POSTGRES_MIN_POOL_SIZE", "1")
+        )
+        self.max_pool_size = max_pool_size or int(
+            os.getenv("PYWORKFLOW_POSTGRES_MAX_POOL_SIZE", "10")
+        )
+        self.max_inactive_connection_lifetime = max_inactive_connection_lifetime or float(
+            os.getenv("PYWORKFLOW_POSTGRES_MAX_INACTIVE_LIFETIME", "1800.0")
+        )
+        self.command_timeout = (
+            command_timeout
+            if command_timeout is not None
+            else (
+                float(os.getenv("PYWORKFLOW_POSTGRES_COMMAND_TIMEOUT", "60.0"))
+                if os.getenv("PYWORKFLOW_POSTGRES_COMMAND_TIMEOUT")
+                else 60.0
+            )
+        )
+
         self._pool: asyncpg.Pool | None = None
         self._pool_loop_id: int | None = None  # Track which loop the pool was created on
         self._initialized = False