orchestrator-core 4.4.0rc1__tar.gz → 4.4.0rc3__tar.gz

{orchestrator_core-4.4.0rc1 → orchestrator_core-4.4.0rc3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orchestrator-core
-Version: 4.4.0rc1
+Version: 4.4.0rc3
 Summary: This is the orchestrator workflow engine.
 Author-email: SURF <automation-beheer@surf.nl>
 Requires-Python: >=3.11,<3.14

{orchestrator_core-4.4.0rc1 → orchestrator_core-4.4.0rc3}/orchestrator/__init__.py

@@ -13,7 +13,7 @@
 
 """This is the orchestrator workflow engine."""
 
-__version__ = "4.4.0rc1"
+__version__ = "4.4.0rc3"
 
 from orchestrator.app import OrchestratorCore
 from orchestrator.settings import app_settings

{orchestrator_core-4.4.0rc1 → orchestrator_core-4.4.0rc3}/orchestrator/api/api_v1/endpoints/subscriptions.py

@@ -12,6 +12,7 @@
 # limitations under the License.
 
 """Module that implements subscription related API endpoints."""
+
 from http import HTTPStatus
 from typing import Any
 from uuid import UUID

{orchestrator_core-4.4.0rc1 → orchestrator_core-4.4.0rc3}/orchestrator/cli/scheduler.py

@@ -13,11 +13,13 @@
 
 
 import logging
+import time
 
 import typer
-from apscheduler.schedulers.blocking import BlockingScheduler
 
-from orchestrator.schedules.scheduler import jobstores, scheduler, scheduler_dispose_db_connections
+from orchestrator.schedules.scheduler import (
+    get_paused_scheduler,
+)
 
 log = logging.getLogger(__name__)
 
@@ -27,13 +29,11 @@ app: typer.Typer = typer.Typer()
 @app.command()
 def run() -> None:
     """Start scheduler and loop eternally to keep thread alive."""
-    blocking_scheduler = BlockingScheduler(jobstores=jobstores)
+    with get_paused_scheduler() as scheduler:
+        scheduler.resume()
 
-    try:
-        blocking_scheduler.start()
-    except (KeyboardInterrupt, SystemExit):
-        scheduler.shutdown()
-        scheduler_dispose_db_connections()
+        while True:
+            time.sleep(1)
 
 
 @app.command()
@@ -42,10 +42,8 @@ def show_schedule() -> None:
 
     in cli underscore is replaced by a dash `show-schedule`
     """
-    scheduler.start(paused=True)  # paused: avoid triggering jobs during CLI
-    jobs = scheduler.get_jobs()
-    scheduler.shutdown(wait=False)
-    scheduler_dispose_db_connections()
+    with get_paused_scheduler() as scheduler:
+        jobs = scheduler.get_jobs()
 
     for job in jobs:
         typer.echo(f"[{job.id}] Next run: {job.next_run_time} | Trigger: {job.trigger}")
@@ -54,10 +52,8 @@ def show_schedule() -> None:
 @app.command()
 def force(job_id: str) -> None:
     """Force the execution of (a) scheduler(s) based on a job_id."""
-    scheduler.start(paused=True)  # paused: avoid triggering jobs during CLI
-    job = scheduler.get_job(job_id)
-    scheduler.shutdown(wait=False)
-    scheduler_dispose_db_connections()
+    with get_paused_scheduler() as scheduler:
+        job = scheduler.get_job(job_id)
 
     if not job:
         typer.echo(f"Job '{job_id}' not found.")

{orchestrator_core-4.4.0rc1 → orchestrator_core-4.4.0rc3}/orchestrator/schedules/scheduler.py

@@ -12,9 +12,11 @@
 # limitations under the License.
 
 
+from contextlib import contextmanager
 from datetime import datetime
-from typing import Any
+from typing import Any, Generator
 
+from apscheduler.executors.pool import ThreadPoolExecutor
 from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
 from apscheduler.schedulers.background import BackgroundScheduler
 from more_itertools import partition
@@ -28,14 +30,31 @@ from orchestrator.settings import app_settings
 from orchestrator.utils.helpers import camel_to_snake, to_camel
 
 jobstores = {"default": SQLAlchemyJobStore(url=str(app_settings.DATABASE_URI))}
+executors = {
+    "default": ThreadPoolExecutor(1),
+}
+job_defaults = {
+    "coalesce": True,
+}
 
-scheduler = BackgroundScheduler(jobstores=jobstores)
+scheduler = BackgroundScheduler(jobstores=jobstores, executors=executors, job_defaults=job_defaults)
 
 
 def scheduler_dispose_db_connections() -> None:
     jobstores["default"].engine.dispose()
 
 
+@contextmanager
+def get_paused_scheduler() -> Generator[BackgroundScheduler, Any, None]:
+    scheduler.start(paused=True)
+
+    try:
+        yield scheduler
+    finally:
+        scheduler.shutdown()
+        scheduler_dispose_db_connections()
+
+
 class ScheduledTask(BaseModel):
     id: str
     name: str | None = None
@@ -131,10 +150,8 @@ def get_scheduler_tasks(
     sort_by: list[Sort] | None = None,
     error_handler: CallableErrorHandler = default_error_handler,
 ) -> tuple[list[ScheduledTask], int]:
-    scheduler.start(paused=True)
-    scheduled_tasks = scheduler.get_jobs()
-    scheduler.shutdown(wait=False)
-    scheduler_dispose_db_connections()
+    with get_paused_scheduler() as pauzed_scheduler:
+        scheduled_tasks = pauzed_scheduler.get_jobs()
 
     scheduled_tasks = filter_scheduled_tasks(scheduled_tasks, error_handler, filter_by)
     scheduled_tasks = sort_scheduled_tasks(scheduled_tasks, error_handler, sort_by)

{orchestrator_core-4.4.0rc1 → orchestrator_core-4.4.0rc3}/orchestrator/services/subscriptions.py

@@ -12,6 +12,7 @@
 # limitations under the License.
 
 """Module that provides service functions on subscriptions."""
+
 import pickle  # noqa: S403
 from collections import defaultdict
 from collections.abc import Sequence
@@ -506,6 +507,7 @@ TARGET_DEFAULT_USABLE_MAP: dict[Target, list[str]] = {
     Target.TERMINATE: ["active", "provisioning"],
     Target.SYSTEM: ["active"],
     Target.VALIDATE: ["active"],
+    Target.RECONCILE: ["active"],
 }
 
 WF_USABLE_MAP: dict[str, list[str]] = {}
@@ -532,6 +534,7 @@ def subscription_workflows(subscription: SubscriptionTable) -> dict[str, Any]:
         ...     "terminate": [],
         ...     "system": [],
         ...     "validate": [],
+        ...     "reconcile: [],
         ... }
 
     """
@@ -552,6 +555,7 @@ def subscription_workflows(subscription: SubscriptionTable) -> dict[str, Any]:
         "terminate": [],
         "system": [],
         "validate": [],
+        "reconcile": [],
     }
     for workflow in subscription.product.workflows:
         if workflow.name in WF_USABLE_WHILE_OUT_OF_SYNC or workflow.is_task:

{orchestrator_core-4.4.0rc1 → orchestrator_core-4.4.0rc3}/orchestrator/targets.py

@@ -23,3 +23,4 @@ class Target(strEnum):
     TERMINATE = "TERMINATE"
     SYSTEM = "SYSTEM"
     VALIDATE = "VALIDATE"
+    RECONCILE = "RECONCILE"

{orchestrator_core-4.4.0rc1 → orchestrator_core-4.4.0rc3}/orchestrator/workflows/utils.py

@@ -154,7 +154,7 @@ def _generate_modify_form(workflow_target: str, workflow_name: str) -> InputForm
 
 
 def wrap_modify_initial_input_form(initial_input_form: InputStepFunc | None) -> StateInputStepFunc | None:
-    """Wrap initial input for modify and terminate workflows.
+    """Wrap initial input for modify, reconcile and terminate workflows.
 
     This is needed because the frontend expects all modify workflows to start with a page that only contains the
     subscription id. It also expects the second page to have some user visible inputs and the subscription id *again*.
@@ -355,6 +355,53 @@ def validate_workflow(description: str) -> Callable[[Callable[[], StepList]], Wo
     return _validate_workflow
 
 
+def reconcile_workflow(
+    description: str,
+    additional_steps: StepList | None = None,
+    authorize_callback: Authorizer | None = None,
+    retry_auth_callback: Authorizer | None = None,
+) -> Callable[[Callable[[], StepList]], Workflow]:
+    """Similar to a modify_workflow but without required input user input to perform a sync with external systems based on the subscriptions existing configuration.
+
+    Use this for subscription reconcile workflows.
+
+    Example::
+
+        @reconcile_workflow("Reconcile l2vpn")
+        def reconcile_l2vpn() -> StepList:
+            return (
+                begin
+                >> update_l2vpn_in_external_systems
+            )
+    """
+
+    wrapped_reconcile_initial_input_form_generator = wrap_modify_initial_input_form(None)
+
+    def _reconcile_workflow(f: Callable[[], StepList]) -> Workflow:
+        steplist = (
+            init
+            >> store_process_subscription()
+            >> unsync
+            >> f()
+            >> (additional_steps or StepList())
+            >> resync
+            >> refresh_subscription_search_index
+            >> done
+        )
+
+        return make_workflow(
+            f,
+            description,
+            wrapped_reconcile_initial_input_form_generator,
+            Target.RECONCILE,
+            steplist,
+            authorize_callback=authorize_callback,
+            retry_auth_callback=retry_auth_callback,
+        )
+
+    return _reconcile_workflow
+
+
 def ensure_provisioning_status(modify_steps: Step | StepList) -> StepList:
     """Decorator to ensure subscription modifications are executed only during Provisioning status."""
     return (

{orchestrator_core-4.4.0rc1 → orchestrator_core-4.4.0rc3}/pyproject.toml

@@ -105,6 +105,7 @@ dev = [
     "pyinstrument>=5.0.2",
     "pytest==8.4.1",
     "pytest-asyncio==0.21.2",
+    "pytest-celery>=1.2.1",
     "pytest-codspeed>=3.2.0",
     "pytest-cov>=6.2.1",
     "pytest-httpx>=0.29.0",
@@ -230,3 +231,6 @@ known-first-party = [
     "oauth2_lib",
     "pydantic_forms",
 ]
+
+[tool.uv]
+index-url = "https://pypi.org/simple"