orchestrator-core 4.6.5__py3-none-any.whl → 4.7.0rc1__py3-none-any.whl

orchestrator/__init__.py

@@ -13,7 +13,7 @@
 
 """This is the orchestrator workflow engine."""
 
-__version__ = "4.6.5"
+__version__ = "4.7.0rc1"
 
 
 from structlog import get_logger

orchestrator/api/api_v1/api.py

@@ -22,6 +22,7 @@ from orchestrator.api.api_v1.endpoints import (
     product_blocks,
     products,
     resource_types,
+    schedules,
     settings,
     subscription_customer_descriptions,
     subscriptions,
@@ -88,6 +89,9 @@ api_router.include_router(
 api_router.include_router(
     ws.router, prefix="/ws", tags=["Core", "Events"]
 )  # Auth on the websocket is handled in the Websocket Manager
+api_router.include_router(
+    schedules.router, prefix="/schedules", tags=["Core", "Schedules"], dependencies=[Depends(authorize)]
+)
 
 if llm_settings.SEARCH_ENABLED:
     from orchestrator.api.api_v1.endpoints import search

orchestrator/api/api_v1/endpoints/processes.py

@@ -13,6 +13,7 @@
 
 """Module that implements process related API endpoints."""
 
+import asyncio
 import struct
 import zlib
 from http import HTTPStatus
@@ -62,10 +63,12 @@ from orchestrator.utils.enrich_process import enrich_process
 from orchestrator.websocket import (
     WS_CHANNELS,
     broadcast_invalidate_status_counts,
+    broadcast_invalidate_status_counts_async,
     broadcast_process_update_to_websocket,
     websocket_manager,
 )
 from orchestrator.workflow import ProcessStat, ProcessStatus, StepList, Workflow
+from orchestrator.workflows import get_workflow
 from pydantic_forms.types import JSON, State
 
 router = APIRouter()
@@ -175,7 +178,7 @@ def delete(process_id: UUID) -> None:
     status_code=HTTPStatus.CREATED,
     dependencies=[Depends(check_global_lock, use_cache=False)],
 )
-def new_process(
+async def new_process(
     workflow_key: str,
     request: Request,
     json_data: list[dict[str, Any]] | None = Body(...),
@@ -183,8 +186,21 @@ def new_process(
     user_model: OIDCUserModel | None = Depends(authenticate),
 ) -> dict[str, UUID]:
     broadcast_func = api_broadcast_process_data(request)
-    process_id = start_process(
-        workflow_key, user_inputs=json_data, user_model=user_model, user=user, broadcast_func=broadcast_func
+
+    workflow = get_workflow(workflow_key)
+    if not workflow:
+        raise_status(HTTPStatus.NOT_FOUND, "Workflow does not exist")
+
+    if not await workflow.authorize_callback(user_model):
+        raise_status(HTTPStatus.FORBIDDEN, f"User is not authorized to execute '{workflow_key}' workflow")
+
+    process_id = await asyncio.to_thread(
+        start_process,
+        workflow_key,
+        user_inputs=json_data,
+        user_model=user_model,
+        user=user,
+        broadcast_func=broadcast_func,
     )
 
     return {"id": process_id}
@@ -196,14 +212,14 @@ def new_process(
     status_code=HTTPStatus.NO_CONTENT,
     dependencies=[Depends(check_global_lock, use_cache=False)],
 )
-def resume_process_endpoint(
+async def resume_process_endpoint(
     process_id: UUID,
     request: Request,
     json_data: JSON = Body(...),
     user: str = Depends(user_name),
     user_model: OIDCUserModel | None = Depends(authenticate),
 ) -> None:
-    process = _get_process(process_id)
+    process = await asyncio.to_thread(_get_process, process_id)
 
     if not can_be_resumed(process.last_status):
         raise_status(HTTPStatus.CONFLICT, f"Resuming a {process.last_status.lower()} workflow is not possible")
@@ -211,16 +227,16 @@ def resume_process_endpoint(
     pstat = load_process(process)
     auth_resume, auth_retry = get_auth_callbacks(get_steps_to_evaluate_for_rbac(pstat), pstat.workflow)
     if process.last_status == ProcessStatus.SUSPENDED:
-        if auth_resume is not None and not auth_resume(user_model):
+        if auth_resume is not None and not (await auth_resume(user_model)):
             raise_status(HTTPStatus.FORBIDDEN, "User is not authorized to resume step")
     elif process.last_status in (ProcessStatus.FAILED, ProcessStatus.WAITING):
-        if auth_retry is not None and not auth_retry(user_model):
+        if auth_retry is not None and not (await auth_retry(user_model)):
             raise_status(HTTPStatus.FORBIDDEN, "User is not authorized to retry step")
 
-    broadcast_invalidate_status_counts()
+    await broadcast_invalidate_status_counts_async()
     broadcast_func = api_broadcast_process_data(request)
 
-    resume_process(process, user=user, user_inputs=json_data, broadcast_func=broadcast_func)
+    await asyncio.to_thread(resume_process, process, user=user, user_inputs=json_data, broadcast_func=broadcast_func)
 
 
 @router.post(

orchestrator/api/api_v1/endpoints/schedules.py

@@ -0,0 +1,44 @@
+# Copyright 2019-2025 SURF.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from http import HTTPStatus
+
+import structlog
+from fastapi.routing import APIRouter
+
+from orchestrator.schedules.service import add_scheduled_task_to_queue
+from orchestrator.schemas.schedules import APSchedulerJobCreate, APSchedulerJobDelete, APSchedulerJobUpdate
+
+logger = structlog.get_logger(__name__)
+
+router: APIRouter = APIRouter()
+
+
+@router.post("/", status_code=HTTPStatus.CREATED)
+def create_scheduled_task(payload: APSchedulerJobCreate) -> dict[str, str]:
+    """Create a scheduled task."""
+    add_scheduled_task_to_queue(payload)
+    return {"message": "Added to Create Queue", "status": "CREATED"}
+
+
+@router.put("/", status_code=HTTPStatus.OK)
+async def update_scheduled_task(payload: APSchedulerJobUpdate) -> dict[str, str]:
+    """Update a scheduled task."""
+    add_scheduled_task_to_queue(payload)
+    return {"message": "Added to Update Queue", "status": "UPDATED"}
+
+
+@router.delete("/", status_code=HTTPStatus.OK)
+async def delete_scheduled_task(payload: APSchedulerJobDelete) -> dict[str, str]:
+    """Delete a scheduled task."""
+    add_scheduled_task_to_queue(payload)
+    return {"message": "Added to Delete Queue", "status": "DELETED"}

orchestrator/cli/scheduler.py

@@ -10,17 +10,26 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-
 import time
+from typing import cast
 
 import typer
+from redis import Redis
 
 from orchestrator.schedules.scheduler import (
     get_all_scheduler_tasks,
     get_scheduler,
     get_scheduler_task,
 )
+from orchestrator.schedules.service import (
+    SCHEDULER_QUEUE,
+    add_scheduled_task_to_queue,
+    workflow_scheduler_queue,
+)
+from orchestrator.schemas.schedules import APSchedulerJobCreate
+from orchestrator.services.workflows import get_workflow_by_name
+from orchestrator.settings import app_settings
+from orchestrator.utils.redis_client import create_redis_client
 
 app: typer.Typer = typer.Typer()
 
@@ -28,10 +37,29 @@ app: typer.Typer = typer.Typer()
 @app.command()
 def run() -> None:
     """Start scheduler and loop eternally to keep thread alive."""
-    with get_scheduler():
-        while True:
+
+    def _get_scheduled_task_item_from_queue(redis_conn: Redis) -> tuple[str, bytes] | None:
+        """Get an item from the Redis Queue for scheduler tasks."""
+        try:
+            return redis_conn.brpop(SCHEDULER_QUEUE, timeout=1)
+        except ConnectionError as e:
+            typer.echo(f"There was a connection error with Redis. Retrying in 3 seconds... {e}")
+            time.sleep(3)
+        except Exception as e:
+            typer.echo(f"There was an unexpected error with Redis. Retrying in 1 second... {e}")
             time.sleep(1)
 
+        return None
+
+    with get_scheduler() as scheduler_connection:
+        redis_connection = create_redis_client(app_settings.CACHE_URI)
+        while True:
+            item = _get_scheduled_task_item_from_queue(redis_connection)
+            if not item:
+                continue
+
+            workflow_scheduler_queue(item, scheduler_connection)
+
 
 @app.command()
 def show_schedule() -> None:
@@ -59,3 +87,45 @@ def force(task_id: str) -> None:
     except Exception as e:
         typer.echo(f"Task execution failed: {e}")
         raise typer.Exit(code=1)
+
+
+@app.command()
+def load_initial_schedule() -> None:
+    """Load the initial schedule into the scheduler."""
+    initial_schedules = [
+        {
+            "name": "Task Resume Workflows",
+            "workflow_name": "task_resume_workflows",
+            "workflow_id": "",
+            "trigger": "interval",
+            "trigger_kwargs": {"hours": 1},
+        },
+        {
+            "name": "Task Clean Up Tasks",
+            "workflow_name": "task_clean_up_tasks",
+            "workflow_id": "",
+            "trigger": "interval",
+            "trigger_kwargs": {"hours": 6},
+        },
+        {
+            "name": "Task Validate Subscriptions",
+            "workflow_name": "task_validate_subscriptions",
+            "workflow_id": "",
+            "trigger": "cron",
+            "trigger_kwargs": {"hour": 0, "minute": 10},
+        },
+    ]
+
+    for schedule in initial_schedules:
+        # enrich with workflow id
+        workflow_name = cast(str, schedule.get("workflow_name"))
+        workflow = get_workflow_by_name(workflow_name)
+
+        if not workflow:
+            typer.echo(f"Workflow '{schedule['workflow_name']}' not found. Skipping schedule.")
+            continue
+
+        schedule["workflow_id"] = workflow.workflow_id
+
+        typer.echo(f"Initial Schedule: {schedule}")
+        add_scheduled_task_to_queue(APSchedulerJobCreate(**schedule))  # type: ignore

orchestrator/cli/search/resize_embedding.py

@@ -75,6 +75,9 @@ def alter_embedding_column_dimension(new_dimension: int) -> None:
         db.session.execute(text(f"ALTER TABLE search_queries ADD COLUMN query_embedding vector({new_dimension})"))
 
         db.session.commit()
+
+        db.session.close()
+
         logger.info(f"Altered embedding columns to dimension {new_dimension} in ai_search_index and search_queries")
 
     except SQLAlchemyError as e:

orchestrator/db/models.py

@@ -29,9 +29,11 @@ from sqlalchemy import (
     CheckConstraint,
     Column,
     Enum,
+    Float,
     ForeignKey,
     Index,
     Integer,
+    LargeBinary,
     PrimaryKeyConstraint,
     Select,
     String,
@@ -796,3 +798,27 @@ class AiSearchIndex(BaseModel):
     content_hash = mapped_column(String(64), nullable=False, index=True)
 
     __table_args__ = (PrimaryKeyConstraint("entity_id", "path", name="pk_ai_search_index"),)
+
+
+class APSchedulerJobStoreModel(BaseModel):
+    __tablename__ = "apscheduler_jobs"
+
+    id = mapped_column(String(191), primary_key=True)
+    next_run_time = mapped_column(Float, nullable=True)
+    job_state = mapped_column(LargeBinary, nullable=False)
+
+
+class WorkflowApschedulerJob(BaseModel):
+    __tablename__ = "workflows_apscheduler_jobs"
+
+    workflow_id = mapped_column(
+        UUIDType, ForeignKey("workflows.workflow_id", ondelete="CASCADE"), primary_key=True, nullable=False
+    )
+
+    # Notice the VARCHAR(512) for schedule_id to accommodate longer IDs so
+    # that if APScheduler changes its ID format in the future, we are covered.
+    schedule_id = mapped_column(
+        String(512), ForeignKey("apscheduler_jobs.id", ondelete="CASCADE"), primary_key=True, nullable=False
+    )
+
+    __table_args__ = (UniqueConstraint("workflow_id", "schedule_id", name="uq_workflow_schedule"),)

orchestrator/graphql/schemas/workflow.py

@@ -38,4 +38,4 @@ class Workflow:
         workflow_table = get_original_model(self, WorkflowTable)
         workflow = get_workflow(workflow_table.name)
 
-        return workflow.authorize_callback(oidc_user)  # type: ignore
+        return await workflow.authorize_callback(oidc_user)  # type: ignore

orchestrator/migrations/versions/schema/2025-11-18_961eddbd4c13_create_linker_table_workflow_apscheduler.py

@@ -0,0 +1,106 @@
+"""Create linker table workflow_apscheduler.
+
+Revision ID: 961eddbd4c13
+Revises: 850dccac3b02
+Create Date: 2025-11-18 10:38:57.211087
+
+"""
+
+from uuid import uuid4
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "961eddbd4c13"
+down_revision = "850dccac3b02"
+branch_labels = None
+depends_on = None
+
+
+workflows = [
+    {
+        "name": "task_validate_subscriptions",
+        "description": "Validate subscriptions",
+        "workflow_id": uuid4(),
+        "target": "SYSTEM",
+    },
+]
+
+
+def _create_workflows() -> None:
+    conn = op.get_bind()
+    for workflow in workflows:
+        conn.execute(
+            sa.text(
+                "INSERT INTO workflows VALUES (:workflow_id, :name, :target, :description, now()) ON CONFLICT DO NOTHING"
+            ),
+            workflow,
+        )
+
+
+def _downgrade_create_workflows() -> None:
+    conn = op.get_bind()
+    for workflow in workflows:
+        conn.execute(sa.text("DELETE FROM workflows WHERE name = :name"), {"name": workflow["name"]})
+
+
+def _create_apscheduler_jobs_table_if_not_exists() -> None:
+    # Check if the apscheduler_jobs table exists and create it if it does not exist.
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    if "apscheduler_jobs" not in inspector.get_table_names():
+        op.execute(
+            sa.text(
+                """
+                CREATE TABLE apscheduler_jobs
+                (
+                    id            VARCHAR(191) NOT NULL PRIMARY KEY,
+                    next_run_time DOUBLE PRECISION,
+                    job_state     bytea NOT NULL
+                );
+                """
+            )
+        )
+
+
+def _create_workflows_table_if_not_exists() -> None:
+    # Notice the VARCHAR(512) for schedule_id to accommodate longer IDs
+    # This so that if APScheduler changes its ID format in the future, we are covered.
+    op.execute(
+        sa.text(
+            """
+                CREATE TABLE workflows_apscheduler_jobs (
+                    workflow_id UUID NOT NULL,
+                    schedule_id VARCHAR(512) NOT NULL,
+                    PRIMARY KEY (workflow_id, schedule_id),
+                    CONSTRAINT fk_workflow
+                        FOREIGN KEY (workflow_id) REFERENCES public.workflows (workflow_id)
+                            ON DELETE CASCADE,
+                    CONSTRAINT fk_schedule
+                        FOREIGN KEY (schedule_id) REFERENCES public.apscheduler_jobs (id)
+                            ON DELETE CASCADE,
+                    CONSTRAINT uq_workflow_schedule UNIQUE (workflow_id, schedule_id)
+                );
+            """
+        )
+    )
+
+    op.create_index("ix_workflows_apscheduler_jobs_schedule_id", "workflows_apscheduler_jobs", ["schedule_id"])
+
+
+def upgrade() -> None:
+    _create_apscheduler_jobs_table_if_not_exists()
+    _create_workflows_table_if_not_exists()
+    _create_workflows()
+
+
+def downgrade() -> None:
+    op.execute(
+        sa.text(
+            """
+            DROP TABLE IF EXISTS workflows_apscheduler_jobs;
+            """
+        )
+    )
+    _downgrade_create_workflows()

orchestrator/schedules/__init__.py

@@ -10,16 +10,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import warnings
 
-
-from orchestrator.schedules.resume_workflows import run_resume_workflows
-from orchestrator.schedules.task_vacuum import vacuum_tasks
 from orchestrator.schedules.validate_products import validate_products
-from orchestrator.schedules.validate_subscriptions import validate_subscriptions
 
+warnings.warn(
+    "ALL_SCHEDULERS is deprecated; scheduling is now handled entirely through the scheduler API.",
+    DeprecationWarning,
+    stacklevel=2,
+)
 ALL_SCHEDULERS: list = [
-    run_resume_workflows,
-    vacuum_tasks,
-    validate_subscriptions,
     validate_products,
 ]

orchestrator/schedules/scheduler.py

@@ -11,7 +11,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 from contextlib import contextmanager
 from datetime import datetime
 from typing import Any, Generator
@@ -27,6 +26,7 @@ from orchestrator.db.filters import Filter
 from orchestrator.db.filters.filters import CallableErrorHandler
 from orchestrator.db.sorting import Sort
 from orchestrator.db.sorting.sorting import SortOrder
+from orchestrator.schedules.service import get_linker_entries_by_schedule_ids
 from orchestrator.utils.helpers import camel_to_snake, to_camel
 
 executors = {
@@ -75,6 +75,7 @@ def get_scheduler(paused: bool = False) -> Generator[BackgroundScheduler, Any, N
 
 class ScheduledTask(BaseModel):
     id: str
+    workflow_id: str | None = None
     name: str | None = None
     next_run_time: datetime | None = None
     trigger: str
@@ -161,6 +162,29 @@ def default_error_handler(message: str, **context) -> None:  # type: ignore
     raise ValueError(f"{message} {_format_context(context)}")
 
 
+def enrich_with_workflow_id(scheduled_tasks: list[ScheduledTask]) -> list[ScheduledTask]:
+    """Does a get call to the linker table to get the workflow_id for each scheduled task.
+
+    Returns all the scheduled tasks with the workflow_id added.
+    """
+    schedule_ids = [task.id for task in scheduled_tasks]
+
+    entries = {
+        str(entry.schedule_id): str(entry.workflow_id) for entry in get_linker_entries_by_schedule_ids(schedule_ids)
+    }
+
+    return [
+        ScheduledTask(
+            id=task.id,
+            workflow_id=entries.get(task.id, None),
+            name=task.name,
+            next_run_time=task.next_run_time,
+            trigger=str(task.trigger),
+        )
+        for task in scheduled_tasks
+    ]
+
+
 def get_scheduler_tasks(
     first: int = 10,
     after: int = 0,
@@ -171,6 +195,7 @@ def get_scheduler_tasks(
     scheduled_tasks = get_all_scheduler_tasks()
     scheduled_tasks = filter_scheduled_tasks(scheduled_tasks, error_handler, filter_by)
     scheduled_tasks = sort_scheduled_tasks(scheduled_tasks, error_handler, sort_by)
+    scheduled_tasks = enrich_with_workflow_id(scheduled_tasks)
 
     total = len(scheduled_tasks)
     paginated_tasks = scheduled_tasks[after : after + first + 1]
@@ -178,6 +203,7 @@ def get_scheduler_tasks(
     return [
         ScheduledTask(
             id=task.id,
+            workflow_id=task.workflow_id,
             name=task.name,
             next_run_time=task.next_run_time,
             trigger=str(task.trigger),

orchestrator/schedules/service.py

@@ -0,0 +1,253 @@
+# Copyright 2019-2025 SURF.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import json
+import logging
+from uuid import UUID, uuid4
+
+from apscheduler.schedulers.base import BaseScheduler
+from apscheduler.triggers.cron import CronTrigger
+from apscheduler.triggers.date import DateTrigger
+from apscheduler.triggers.interval import IntervalTrigger
+from sqlalchemy import delete
+
+from orchestrator import app_settings
+from orchestrator.db import db
+from orchestrator.db.models import WorkflowApschedulerJob
+from orchestrator.schemas.schedules import (
+    APSchedulerJobCreate,
+    APSchedulerJobDelete,
+    APSchedulerJobs,
+    APSchedulerJobUpdate,
+    APSJobAdapter,
+)
+from orchestrator.services.processes import start_process
+from orchestrator.services.workflows import get_workflow_by_workflow_id
+from orchestrator.utils.redis_client import create_redis_client
+
+redis_connection = create_redis_client(app_settings.CACHE_URI)
+
+SCHEDULER_QUEUE = "scheduler:queue:"
+
+
+logger = logging.getLogger(__name__)
+
+
+def serialize_payload(payload: APSchedulerJobs) -> bytes:
+    """Serialize the payload to bytes for Redis storage.
+
+    Args:
+        payload: APSchedulerJobs The scheduled task payload.
+    """
+    data = json.loads(payload.model_dump_json())
+    data["scheduled_type"] = payload.scheduled_type
+    return json.dumps(data).encode()
+
+
+def deserialize_payload(bytes_dump: bytes) -> APSchedulerJobs:
+    """Deserialize the payload from bytes for Redis retrieval.
+
+    Args:
+        bytes_dump: bytes The serialized payload.
+    """
+    json_dump = bytes_dump.decode()
+    return APSJobAdapter.validate_json(json_dump)
+
+
+def add_scheduled_task_to_queue(payload: APSchedulerJobs) -> None:
+    """Create a scheduled task service function.
+
+    We need to create a apscheduler job, and put the workflow and schedule_id in
+    the linker table workflows_apscheduler_jobs.
+
+    Args:
+        payload: APSchedulerJobCreate The scheduled task to create.
+    """
+    bytes_dump = serialize_payload(payload)
+    redis_connection.lpush(SCHEDULER_QUEUE, bytes_dump)
+    logger.info("Added scheduled task to queue.")
+
+
+def get_linker_entries_by_schedule_ids(schedule_ids: list[str]) -> list[WorkflowApschedulerJob]:
+    """Get linker table entries for multiple schedule IDs in a single query.
+
+    Args:
+        schedule_ids: list[str] — One or many schedule IDs.
+
+    Returns:
+        list[WorkflowApschedulerJob]: All linker table rows matching those IDs.
+    """
+    if not schedule_ids:
+        return []
+
+    return db.session.query(WorkflowApschedulerJob).filter(WorkflowApschedulerJob.schedule_id.in_(schedule_ids)).all()
+
+
+def _add_linker_entry(workflow_id: UUID, schedule_id: str) -> None:
+    """Add an entry to the linker table workflows_apscheduler_jobs.
+
+    Args:
+        workflow_id: UUID The workflow ID.
+        schedule_id: str The schedule ID.
+    """
+    workflows_apscheduler_job = WorkflowApschedulerJob(workflow_id=workflow_id, schedule_id=schedule_id)
+    db.session.add(workflows_apscheduler_job)
+    db.session.commit()
+
+
+def _delete_linker_entry(workflow_id: UUID, schedule_id: str) -> None:
+    """Delete an entry from the linker table workflows_apscheduler_jobs.
+
+    Args:
+        workflow_id: UUID The workflow ID.
+        schedule_id: str The schedule ID.
+    """
+    db.session.execute(
+        delete(WorkflowApschedulerJob).where(
+            WorkflowApschedulerJob.workflow_id == workflow_id, WorkflowApschedulerJob.schedule_id == schedule_id
+        )
+    )
+    db.session.commit()
+
+
+def run_start_workflow_scheduler_task(workflow_name: str) -> None:
+    """Function to start a workflow from the scheduler.
+
+    Args:
+        workflow_name: str The name of the workflow to start.
+    """
+    logger.info(f"Starting workflow: {workflow_name}")
+    start_process(workflow_name)
+
+
+def _add_scheduled_task(payload: APSchedulerJobCreate, scheduler_connection: BaseScheduler) -> None:
+    """Create a new scheduled task in the scheduler and also in the linker table.
+
+    Args:
+        payload: APSchedulerJobCreate The scheduled task to create.
+        scheduler_connection: BaseScheduler The scheduler connection.
+    """
+    logger.info(f"Adding scheduled task: {payload}")
+
+    workflow_description = None
+    # Check if a workflow exists - we cannot schedule a non-existing workflow
+    workflow = get_workflow_by_workflow_id(str(payload.workflow_id))
+    if not workflow:
+        raise ValueError(f"Workflow with id {payload.workflow_id} does not exist.")
+    workflow_description = workflow.description
+
+    # This function is always the same for scheduled tasks, it will run the workflow
+    func = run_start_workflow_scheduler_task
+
+    # Ensure payload has required data
+    if not payload.trigger or not payload.workflow_name or not payload.trigger_kwargs or not payload.workflow_id:
+        raise ValueError("Trigger must be specified for scheduled tasks.")
+
+    schedule_id = str(uuid4())
+    scheduler_connection.add_job(
+        func=func,
+        trigger=payload.trigger,
+        id=schedule_id,
+        name=payload.name or workflow_description,
+        kwargs={"workflow_name": payload.workflow_name},
+        **(payload.trigger_kwargs or {}),
+    )
+
+    _add_linker_entry(workflow_id=payload.workflow_id, schedule_id=schedule_id)
+
+
+def _build_trigger_on_update(
+    trigger_name: str | None, trigger_kwargs: dict
+) -> IntervalTrigger | CronTrigger | DateTrigger | None:
+    if not trigger_name or not trigger_kwargs:
+        logger.info("Skipping building trigger as no trigger information is provided.")
+        return None
+
+    match trigger_name:
+        case "interval":
+            return IntervalTrigger(**trigger_kwargs)
+        case "cron":
+            return CronTrigger(**trigger_kwargs)
+        case "date":
+            return DateTrigger(**trigger_kwargs)
+        case _:
+            raise ValueError(f"Invalid trigger type: {trigger_name}")
+
+
+def _update_scheduled_task(payload: APSchedulerJobUpdate, scheduler_connection: BaseScheduler) -> None:
+    """Update an existing scheduled task in the scheduler.
+
+    Only allow update of name and trigger
+    Job id must be that of an existing job
+    Do not insert in linker table - it should already exist.
+
+    Args:
+        payload: APSchedulerJobUpdate The scheduled task to update.
+        scheduler_connection: BaseScheduler The scheduler connection.
+    """
+    logger.info(f"Updating scheduled task: {payload}")
+
+    schedule_id = str(payload.schedule_id)
+    job = scheduler_connection.get_job(job_id=schedule_id)
+    if not job:
+        raise ValueError(f"Schedule Job with id {schedule_id} does not exist.")
+
+    trigger = _build_trigger_on_update(payload.trigger, payload.trigger_kwargs or {})
+    modify_kwargs = {}
+
+    if trigger:
+        job = job.reschedule(trigger=trigger)
+
+    if payload.name:
+        modify_kwargs["name"] = payload.name
+
+    job.modify(**modify_kwargs)
+
+
+def _delete_scheduled_task(payload: APSchedulerJobDelete, scheduler_connection: BaseScheduler) -> None:
+    """Delete an existing scheduled task in the scheduler and also in the linker table.
+
+    Args:
+        payload: APSchedulerJobDelete The scheduled task to delete.
+        scheduler_connection: BaseScheduler The scheduler connection.
+    """
+    logger.info(f"Deleting scheduled task: {payload}")
+
+    schedule_id = str(payload.schedule_id)
+    scheduler_connection.remove_job(job_id=schedule_id)
+    _delete_linker_entry(workflow_id=payload.workflow_id, schedule_id=schedule_id)
+
+
+def workflow_scheduler_queue(queue_item: tuple[str, bytes], scheduler_connection: BaseScheduler) -> None:
+    """Process an item from the scheduler queue.
+
+    Args:
+        queue_item: tuple[str, bytes] The item from the scheduler queue.
+        scheduler_connection: BaseScheduler The scheduler connection.
+    """
+    try:
+        _, bytes_dump = queue_item
+        payload = deserialize_payload(bytes_dump)
+        match payload:
+            case APSchedulerJobCreate():
+                _add_scheduled_task(payload, scheduler_connection)
+
+            case APSchedulerJobUpdate():
+                _update_scheduled_task(payload, scheduler_connection)
+
+            case APSchedulerJobDelete():
+                _delete_scheduled_task(payload, scheduler_connection)
+
+            case _:
+                logger.warning(f"Unexpected schedule type: {payload}")  # type: ignore
+    except Exception:
+        logger.exception("Error processing scheduler queue item")

orchestrator/schemas/schedules.py

@@ -0,0 +1,71 @@
+# Copyright 2019-2025 SURF.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Annotated, Any, Literal, Union
+from uuid import UUID
+
+from pydantic import BaseModel, Field, TypeAdapter
+
+SCHEDULER_Q_CREATE = "create"
+SCHEDULER_Q_UPDATE = "update"
+SCHEDULER_Q_DELETE = "delete"
+
+
+class APSchedulerJob(BaseModel):
+    scheduled_type: Literal["create", "update", "delete"] = Field(..., description="Discriminator for job type")
+
+
+class APSchedulerJobCreate(APSchedulerJob):
+    name: str | None = Field(None, description="Human readable name e.g. 'My Process'")
+    workflow_name: str = Field(..., description="Name of the workflow to run e.g. 'my_workflow_name'")
+    workflow_id: UUID = Field(..., description="UUID of the workflow associated with this scheduled task")
+
+    trigger: Literal["interval", "cron", "date"] = Field(..., description="APScheduler trigger type")
+    trigger_kwargs: dict[str, Any] = Field(
+        default_factory=lambda: {},
+        description="Arguments passed to the trigger on job creation",
+        examples=[{"hours": 12}, {"minutes": 30}, {"days": 1, "hours": 2}],
+    )
+
+    scheduled_type: Literal["create"] = Field("create", frozen=True)
+
+
+class APSchedulerJobUpdate(APSchedulerJob):
+    name: str | None = Field(None, description="Human readable name e.g. 'My Process'")
+    schedule_id: UUID = Field(..., description="UUID of the scheduled task")
+
+    trigger: Literal["interval", "cron", "date"] | None = Field(None, description="APScheduler trigger type")
+    trigger_kwargs: dict[str, Any] | None = Field(
+        default=None,
+        description="Arguments passed to the job function",
+        examples=[{"hours": 12}, {"minutes": 30}, {"days": 1, "hours": 2}],
+    )
+
+    scheduled_type: Literal["update"] = Field("update", frozen=True)
+
+
+class APSchedulerJobDelete(APSchedulerJob):
+    workflow_id: UUID = Field(..., description="UUID of the workflow associated with this scheduled task")
+    schedule_id: UUID | None = Field(None, description="UUID of the scheduled task")
+
+    scheduled_type: Literal["delete"] = Field("delete", frozen=True)
+
+
+APSchedulerJobs = Annotated[
+    Union[
+        APSchedulerJobCreate,
+        APSchedulerJobUpdate,
+        APSchedulerJobDelete,
+    ],
+    Field(discriminator="scheduled_type"),
+]
+APSJobAdapter = TypeAdapter(APSchedulerJobs)  # type: ignore

orchestrator/services/processes.py

@@ -421,10 +421,6 @@ def _run_process_async(process_id: UUID, f: Callable) -> UUID:
     return process_id
 
 
-def error_message_unauthorized(workflow_key: str) -> str:
-    return f"User is not authorized to execute '{workflow_key}' workflow"
-
-
 def create_process(
     workflow_key: str,
     user_inputs: list[State] | None = None,
@@ -442,9 +438,6 @@ def create_process(
     if not workflow:
         raise_status(HTTPStatus.NOT_FOUND, "Workflow does not exist")
 
-    if not workflow.authorize_callback(user_model):
-        raise_status(HTTPStatus.FORBIDDEN, error_message_unauthorized(workflow_key))
-
     initial_state = {
         "process_id": process_id,
         "reporter": user,

orchestrator/services/workflows.py

@@ -64,6 +64,10 @@ def get_workflow_by_name(workflow_name: str) -> WorkflowTable | None:
     return db.session.scalar(select(WorkflowTable).where(WorkflowTable.name == workflow_name))
 
 
+def get_workflow_by_workflow_id(workflow_id: str) -> WorkflowTable | None:
+    return db.session.scalar(select(WorkflowTable).where(WorkflowTable.workflow_id == workflow_id))
+
+
 def get_validation_product_workflows_for_subscription(
     subscription: SubscriptionTable,
 ) -> list:

orchestrator/utils/auth.py

@@ -1,4 +1,4 @@
-from collections.abc import Callable
+from collections.abc import Awaitable, Callable
 from typing import TypeAlias, TypeVar
 
 from oauth2_lib.fastapi import OIDCUserModel
@@ -7,4 +7,4 @@ from oauth2_lib.fastapi import OIDCUserModel
 
 # Can instead use "type Authorizer = ..." in later Python versions.
 T = TypeVar("T", bound=OIDCUserModel)
-Authorizer: TypeAlias = Callable[[T | None], bool]
+Authorizer: TypeAlias = Callable[[T | None], Awaitable[bool]]

orchestrator/websocket/__init__.py

@@ -105,6 +105,19 @@ async def invalidate_subscription_cache(subscription_id: UUID | UUIDstr, invalid
     await broadcast_invalidate_cache({"type": "subscriptions", "id": str(subscription_id)})
 
 
+async def broadcast_invalidate_status_counts_async() -> None:
+    """Broadcast message to invalidate the status counts of the connected websocket clients.
+
+    This breaks the pattern of `sync_` prefixes to maintain backwards compatibility of
+    broadcast_invalidate_status_counts, a sync function.
+    """
+    if not websocket_manager.enabled:
+        logger.debug("WebSocketManager is not enabled. Skip broadcasting through websocket.")
+        return
+
+    await broadcast_invalidate_cache({"type": "processStatusCounts"})
+
+
 def broadcast_invalidate_status_counts() -> None:
     """Broadcast message to invalidate the status counts of the connected websocket clients."""
     if not websocket_manager.enabled:
@@ -148,4 +161,5 @@ __all__ = [
     "broadcast_process_update_to_websocket_async",
     "WS_CHANNELS",
     "broadcast_invalidate_status_counts",
+    "broadcast_invalidate_status_counts_async",
 ]

orchestrator/workflow.py

@@ -193,7 +193,7 @@ def _handle_simple_input_form_generator(f: StateInputStepFunc) -> StateInputForm
     return form_generator
 
 
-def allow(_: OIDCUserModel | None) -> bool:
+async def allow(_: OIDCUserModel | None) -> bool:
     """Default function to return True in absence of user-defined authorize function."""
     return True
 

orchestrator/workflows/__init__.py

@@ -111,5 +111,6 @@ LazyWorkflowInstance(".tasks.cleanup_tasks_log", "task_clean_up_tasks")
 LazyWorkflowInstance(".tasks.resume_workflows", "task_resume_workflows")
 LazyWorkflowInstance(".tasks.validate_products", "task_validate_products")
 LazyWorkflowInstance(".tasks.validate_product_type", "task_validate_product_type")
+LazyWorkflowInstance(".tasks.validate_subscriptions", "task_validate_subscriptions")
 
 __doc__ = make_workflow_index_doc(ALL_WORKFLOWS)

orchestrator/{schedules → workflows/tasks}/validate_subscriptions.py

@@ -16,7 +16,6 @@ from threading import BoundedSemaphore
 
 import structlog
 
-from orchestrator.schedules.scheduler import scheduler
 from orchestrator.services.subscriptions import (
     get_subscriptions_on_product_table,
     get_subscriptions_on_product_table_in_sync,
@@ -26,6 +25,8 @@ from orchestrator.services.workflows import (
     start_validation_workflow_for_workflows,
 )
 from orchestrator.settings import app_settings
+from orchestrator.targets import Target
+from orchestrator.workflow import StepList, init, step, workflow
 
 logger = structlog.get_logger(__name__)
 
@@ -33,7 +34,7 @@ logger = structlog.get_logger(__name__)
 task_semaphore = BoundedSemaphore(value=2)
 
 
-@scheduler.scheduled_job(id="subscriptions-validator", name="Subscriptions Validator", trigger="cron", hour=0, minute=10)  # type: ignore[misc]
+@step("Validate subscriptions")
 def validate_subscriptions() -> None:
     if app_settings.VALIDATE_OUT_OF_SYNC_SUBSCRIPTIONS:
         # Automatically re-validate out-of-sync subscriptions. This is not recommended for production.
@@ -53,3 +54,8 @@ def validate_subscriptions() -> None:
             break
 
         start_validation_workflow_for_workflows(subscription=subscription, workflows=validation_product_workflows)
+
+
+@workflow("Validate subscriptions", target=Target.SYSTEM)
+def task_validate_subscriptions() -> StepList:
+    return init >> validate_subscriptions

orchestrator/workflows/translations/en-GB.json

@@ -16,6 +16,7 @@
         "task_resume_workflows": "Resume all workflows that are stuck on tasks with the status 'waiting'",
         "task_validate_products": "Validate Products and Subscriptions",
         "task_validate_product_type": "Validate all subscriptions of Product Type",
-        "reset_subscription_description": "Reset description of a subscription to default"
+        "reset_subscription_description": "Reset description of a subscription to default",
+        "task_validate_subscriptions": "Validate subscriptions"
     }
 }

{orchestrator_core-4.6.5.dist-info → orchestrator_core-4.7.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orchestrator-core
-Version: 4.6.5
+Version: 4.7.0rc1
 Summary: This is the orchestrator workflow engine.
 Author-email: SURF <automation-beheer@surf.nl>
 Requires-Python: >=3.11,<3.15
@@ -37,14 +37,14 @@ Requires-Dist: apscheduler>=3.11.0
 Requires-Dist: click==8.*
 Requires-Dist: deepmerge==2.0
 Requires-Dist: deprecated>=1.2.18
-Requires-Dist: fastapi~=0.121.1
+Requires-Dist: fastapi~=0.124.0
 Requires-Dist: fastapi-etag==0.4.0
 Requires-Dist: itsdangerous>=2.2.0
 Requires-Dist: jinja2==3.1.6
 Requires-Dist: more-itertools~=10.8.0
 Requires-Dist: nwa-stdlib~=1.11.0
 Requires-Dist: oauth2-lib>=2.5.0
-Requires-Dist: orjson==3.11.4
+Requires-Dist: orjson==3.11.5
 Requires-Dist: pgvector>=0.4.1
 Requires-Dist: prometheus-client==0.23.1
 Requires-Dist: psycopg2-binary==2.9.11
@@ -52,9 +52,9 @@ Requires-Dist: pydantic-forms>=1.4.0
 Requires-Dist: pydantic-settings~=2.12.0
 Requires-Dist: pydantic[email]~=2.12.4
 Requires-Dist: python-dateutil==2.9.0.post0
-Requires-Dist: python-rapidjson>=1.22,<1.23
+Requires-Dist: python-rapidjson>=1.23,<1.24
 Requires-Dist: pytz==2025.2
-Requires-Dist: redis==5.3.1
+Requires-Dist: redis==7.1.0
 Requires-Dist: semver==3.0.4
 Requires-Dist: sentry-sdk[fastapi]>=2.29.1
 Requires-Dist: sqlalchemy==2.0.44
@@ -64,11 +64,11 @@ Requires-Dist: structlog>=25.4.0
 Requires-Dist: tabulate==0.9.0
 Requires-Dist: typer==0.20.0
 Requires-Dist: uvicorn[standard]~=0.38.0
-Requires-Dist: pydantic-ai-slim >=1.9.0 ; extra == "agent"
-Requires-Dist: ag-ui-protocol>=0.1.8 ; extra == "agent"
+Requires-Dist: pydantic-ai-slim >=1.27.0 ; extra == "agent"
+Requires-Dist: ag-ui-protocol>=0.1.10 ; extra == "agent"
 Requires-Dist: litellm>=1.75.7 ; extra == "agent"
-Requires-Dist: celery~=5.5.1 ; extra == "celery"
-Requires-Dist: litellm>=1.75.7 ; extra == "search"
+Requires-Dist: celery~=5.6.0 ; extra == "celery"
+Requires-Dist: litellm>=1.80.0 ; extra == "search"
 Project-URL: Documentation, https://workfloworchestrator.org/orchestrator-core
 Project-URL: Homepage, https://workfloworchestrator.org/orchestrator-core
 Project-URL: Source, https://github.com/workfloworchestrator/orchestrator-core

{orchestrator_core-4.6.5.dist-info → orchestrator_core-4.7.0rc1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-orchestrator/__init__.py,sha256=hehQZgPnU4S_qlMQAFlyyXe8CPEjkPHZYOOzKd5Ff2M,1454
+orchestrator/__init__.py,sha256=HWuhzwoxPlAnuAGVtn9GBpZS5X185yqB_j-WnUzXnVE,1457
 orchestrator/agentic_app.py,sha256=ouiyyZiS4uS6Lox2DtbGGRnb2njJBMSHpSAGe-T5rX0,3028
 orchestrator/app.py,sha256=w8ubXaaogwjmwLM0TXqZaLkAhmaOTWzVlwiYbi5mHeE,13203
 orchestrator/exception_handlers.py,sha256=UsW3dw8q0QQlNLcV359bIotah8DYjMsj2Ts1LfX4ClY,1268
@@ -10,20 +10,21 @@ orchestrator/settings.py,sha256=mvs1VhBYth6Zp55HsNroML4DU1jiq5SkVM47_BLgcIo,4662
 orchestrator/targets.py,sha256=d7Fyh_mWIWPivA_E7DTNFpZID3xFW_K0JlZ5nksVX7k,830
 orchestrator/types.py,sha256=qzs7xx5AYRmKbpYRyJJP3wuDb0W0bcAzefCN0RWLAco,15459
 orchestrator/version.py,sha256=b58e08lxs47wUNXv0jXFO_ykpksmytuzEXD4La4W-NQ,1366
-orchestrator/workflow.py,sha256=QvAbJ3ySuqUnKrQ_EQ6PQqgkK1TmKZ7COioCCmeABwc,46391
+orchestrator/workflow.py,sha256=JF81oC2JqdmQwM6VcrfavRMYLv-dQa_bNeH0dkAFe-w,46397
 orchestrator/api/__init__.py,sha256=GyHNfEFCGKQwRiN6rQmvSRH2iYX7npjMZn97n8XzmLU,571
 orchestrator/api/error_handling.py,sha256=YrPCxSa-DSa9KwqIMlXI-KGBGnbGIW5ukOPiikUH9E4,1502
 orchestrator/api/helpers.py,sha256=s0QRHYw8AvEmlkmRhuEzz9xixaZKUF3YuPzUVHkcoXk,6933
 orchestrator/api/models.py,sha256=z9BDBx7uI4KBHWbD_LVrLsqNQ0_w-Mg9Qiy7PR_rZhk,5996
 orchestrator/api/api_v1/__init__.py,sha256=GyHNfEFCGKQwRiN6rQmvSRH2iYX7npjMZn97n8XzmLU,571
-orchestrator/api/api_v1/api.py,sha256=1qQRsIxKXLW3kcmSV5u3_v1TZk5RcNWb4ZOyLguhTKY,3488
+orchestrator/api/api_v1/api.py,sha256=vS3UpUG0EKuVGTa1FlSqIN8RHhDrrHIjPQP7CqWlzJo,3637
 orchestrator/api/api_v1/endpoints/__init__.py,sha256=GyHNfEFCGKQwRiN6rQmvSRH2iYX7npjMZn97n8XzmLU,571
 orchestrator/api/api_v1/endpoints/agent.py,sha256=9_s3nchTr9ua_1Sxs0kJl2mH_20o-DlUaaQPmY4vRMk,1732
 orchestrator/api/api_v1/endpoints/health.py,sha256=iaxs1XX1_250_gKNsspuULCV2GEMBjbtjsmfQTOvMAI,1284
-orchestrator/api/api_v1/endpoints/processes.py,sha256=z_bantT415cCuAw0viVrsi7HquHKMSmdjnuLkk9OZKI,16324
+orchestrator/api/api_v1/endpoints/processes.py,sha256=2pA996SqE7WUuR1sYMNN1gl4l1hTGS-jlYu6u2dRQaQ,16890
 orchestrator/api/api_v1/endpoints/product_blocks.py,sha256=kZ6ywIOsS_S2qGq7RvZ4KzjvaS1LmwbGWR37AKRvWOw,2146
 orchestrator/api/api_v1/endpoints/products.py,sha256=BfFtwu9dZXEQbtKxYj9icc73GKGvAGMR5ytyf41nQlQ,3081
 orchestrator/api/api_v1/endpoints/resource_types.py,sha256=gGyuaDyOD0TAVoeFGaGmjDGnQ8eQQArOxKrrk4MaDzA,2145
+orchestrator/api/api_v1/endpoints/schedules.py,sha256=eTG_4CQkiIi2akJUN4xDGuU_OvF6Ml6uye5MmQ_WJbc,1731
 orchestrator/api/api_v1/endpoints/search.py,sha256=R_OzfJfquoaTeGBwXTbomh16aYy0ael6_Xn3WkKfyjg,8575
 orchestrator/api/api_v1/endpoints/settings.py,sha256=5s-k169podZjgGHUbVDmSQwpY_3Cs_Bbf2PPtZIkBcw,6184
 orchestrator/api/api_v1/endpoints/subscription_customer_descriptions.py,sha256=1_6LtgQleoq3M6z_W-Qz__Bj3OFUweoPrUqHMwSH6AM,3288
@@ -40,7 +41,7 @@ orchestrator/cli/migrate_domain_models.py,sha256=WRXy_1OnziQwpsCFZXvjB30nDJtjj0i
 orchestrator/cli/migrate_tasks.py,sha256=bju8XColjSZD0v3rS4kl-24dLr8En_H4-6enBmqd494,7255
 orchestrator/cli/migrate_workflows.py,sha256=nxUpx0vgEIc_8aJrjAyrw3E9Dt8JmaamTts8oiQ4vHY,8923
 orchestrator/cli/migration_helpers.py,sha256=C5tpkP5WEBr7G9S-1k1hgSI8ili6xd9Z5ygc9notaK0,4110
-orchestrator/cli/scheduler.py,sha256=4jWpgxx0j0UFoba4Kw0nOEM6slr5XffDYBkm6hzK_C0,1766
+orchestrator/cli/scheduler.py,sha256=U-YLa_SkwGVAf7jQ8EQugtwTuycw3DtXhmCxlMpiQqQ,4396
 orchestrator/cli/domain_gen_helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 orchestrator/cli/domain_gen_helpers/fixed_input_helpers.py,sha256=uzpwsaau81hHSxNMOS9-o7kF-9_78R0f_UE0AvWooZQ,6775
 orchestrator/cli/domain_gen_helpers/helpers.py,sha256=tIPxn8ezED_xYZxH7ZAtQLwkDc6RNmLZVxWAoJ3a9lw,4203
@@ -106,7 +107,7 @@ orchestrator/cli/helpers/print_helpers.py,sha256=b3ePg6HfBLKPYBBVr5XOA__JnFEMI5H
 orchestrator/cli/search/__init__.py,sha256=K15_iW9ogR7xtX7qHDal4H09tmwVGnOBZWyPBLWhuzc,1274
 orchestrator/cli/search/display.py,sha256=PKy9sOTpq0WUdSfY2khLrIQ1OdAfsyl95ogF1Z6Dae0,3629
 orchestrator/cli/search/index_llm.py,sha256=VQlLNGXro4ZbehWZIMKPHGpGE4oF1at2bkWZssimWc4,2707
-orchestrator/cli/search/resize_embedding.py,sha256=iJdM7L6Kyq4CzRjXHWLwpGRiMnKK7xZ9133C0THebBE,4847
+orchestrator/cli/search/resize_embedding.py,sha256=5p2SmFqR-fHH1euTF8mSI3MmoXwN7yiV-JOZ7rdoxw0,4876
 orchestrator/cli/search/search_explore.py,sha256=LNAn6w13Q69fpv8CHcicHAbabrctrjGvwTjjJyC0AZY,8447
 orchestrator/cli/search/speedtest.py,sha256=J_l-8WxgN3YnqmwnbRhDyVbeqtvk3d2SfIpRBOJuhvE,4840
 orchestrator/config/__init__.py,sha256=GyHNfEFCGKQwRiN6rQmvSRH2iYX7npjMZn97n8XzmLU,571
@@ -116,7 +117,7 @@ orchestrator/db/database.py,sha256=MU_w_e95ho2dVb2JDnt_KFYholx___XDkiQXbc8wCkI,1
 orchestrator/db/helpers.py,sha256=L8kEdnSSNGnUpZhdeGx2arCodakWN8vSpKdfjoLuHdY,831
 orchestrator/db/listeners.py,sha256=UBPYcH0FE3a7aZQu_D0O_JMXpXIRYXC0gjSAvlv5GZo,1142
 orchestrator/db/loaders.py,sha256=ez6JzQ3IKVkC_oLAkVlIIiI8Do7hXbdcPKCvUSLxRog,7962
-orchestrator/db/models.py,sha256=AeBHLm0PLtoSAHoXzxjy9JsDQA5JhPvS7Ro55ynu_tk,31275
+orchestrator/db/models.py,sha256=qsXOonEDaxn_UXfuXLQfdnm1PV6a-JZVpXa46vd--Es,32201
 orchestrator/db/filters/__init__.py,sha256=RUj6P0XxEBhYj0SN5wH5-Vf_Wt_ilZR_n9DSar5m9oM,371
 orchestrator/db/filters/filters.py,sha256=55RtpQwM2rhrk4A6CCSeSXoo-BT9GnQoNTryA8CtLEg,5020
 orchestrator/db/filters/process.py,sha256=xvGhyfo_MZ1xhLvFC6yULjcT4mJk0fKc1glJIYgsWLE,4018
@@ -204,7 +205,7 @@ orchestrator/graphql/schemas/settings.py,sha256=drhm5VcLmUbiYAk6WUSJcyJqjNM96E6G
 orchestrator/graphql/schemas/strawberry_pydantic_patch.py,sha256=CjNUhTKdYmLiaem-WY_mzw4HASIeaZitxGF8pPocqVw,1602
 orchestrator/graphql/schemas/subscription.py,sha256=hTA34C27kgLguH9V53173CxMKIWiQKh3vFzyJ2yBfE0,9918
 orchestrator/graphql/schemas/version.py,sha256=HSzVg_y4Sjd5_H5rRUtu3FJKOG_8ifhvBNt_qjOtC-E,92
-orchestrator/graphql/schemas/workflow.py,sha256=ewE5mRuqMq7rnx8Au2eTUm3YTY1pivOWATNacZQ-trY,1773
+orchestrator/graphql/schemas/workflow.py,sha256=AAHQ2QecTPiAARAcB9Y4Y-ccEmUbpWuAfTYpl6WsbwU,1779
 orchestrator/graphql/utils/__init__.py,sha256=1JvenzEVW1CBa1sGVI9I8IWnnoXIkb1hneDqph9EEZY,524
 orchestrator/graphql/utils/create_resolver_error_handler.py,sha256=XzCnL482M4wz3fg5fUdGUwCAuzSZQ9Ufu1mscLyeoWU,1227
 orchestrator/graphql/utils/get_query_loaders.py,sha256=abS_HJ7K9een78gMiGq3IhwGwxQXHvZygExe0h_t9ns,815
@@ -257,13 +258,12 @@ orchestrator/migrations/versions/schema/2025-05-08_161918133bec_add_is_task_to_w
 orchestrator/migrations/versions/schema/2025-07-01_93fc5834c7e5_changed_timestamping_fields_in_process_steps.py,sha256=Oezd8b2qaI1Kyq-sZFVFmdzd4d9NjXrf6HtJGk11fy0,1914
 orchestrator/migrations/versions/schema/2025-07-04_4b58e336d1bf_deprecating_workflow_target_in_.py,sha256=xnD6w-97R4ClS7rbmXQEXc36K3fdcXKhCy7ZZNy_FX4,742
 orchestrator/migrations/versions/schema/2025-07-28_850dccac3b02_update_description_of_resume_workflows_.py,sha256=R6Qoga83DJ1IL0WYPu0u5u2ZvAmqGlDmUMv_KtJyOhQ,812
-orchestrator/schedules/__init__.py,sha256=Zy0fTOBMGIRFoh5iVFDLF9_PRAFaONYDThGK9EsysWo,981
-orchestrator/schedules/resume_workflows.py,sha256=jRnVRWDy687pQu-gtk80ecwiLSdrvtL15tG3U2zWA6I,891
-orchestrator/schedules/scheduler.py,sha256=9d6n-J2_GB6crOoVSCK29IfaktfUyzQYcTZl7gRTZ5c,6250
+orchestrator/migrations/versions/schema/2025-11-18_961eddbd4c13_create_linker_table_workflow_apscheduler.py,sha256=wJ01G2fpph9gYq0haeYRoSD21cTLOdn41axCSMyWV8o,3109
+orchestrator/schedules/__init__.py,sha256=WNjy4D4QdOKJh6ZEHJ119tXL1xJ3L7kMPuJWYnayjtE,868
+orchestrator/schedules/scheduler.py,sha256=8o7DoVs9Q1Q231FVMpv3tXtKbaydeNkYQ1h6kl7U1X4,7198
 orchestrator/schedules/scheduling.py,sha256=_mbpHMhijey8Y56ebtJ4wVkrp_kPVRm8hoByzlQF4SE,2821
-orchestrator/schedules/task_vacuum.py,sha256=mxb7fsy1GphRwvUWi_lvwNaj51YAXUdIDlkOJd90AFI,874
+orchestrator/schedules/service.py,sha256=CYWb_gB5Dw57AIiQtSVcLW4sEE69zNoWGuSe2WEIj_8,8940
 orchestrator/schedules/validate_products.py,sha256=_ucUG9HecskG2eN3tcDSiMzJK9gN3kZB1dXjrtxcApY,1324
-orchestrator/schedules/validate_subscriptions.py,sha256=bUBV45aEuqVdtqYBAXh1lX4O5vuNTeTfds4J_zq35dI,2113
 orchestrator/schemas/__init__.py,sha256=YDyZ0YBvzB4ML9oDBCBPGnBvf680zFFgUzg7X0tYBRY,2326
 orchestrator/schemas/base.py,sha256=Vc444LetsINLRhG2SxW9Bq01hOzChPOhQWCImQTr-As,930
 orchestrator/schemas/engine_settings.py,sha256=LF8al7tJssiilb5A4emPtUYo0tVDSaT1Lvo_DN_ttrY,1296
@@ -273,6 +273,7 @@ orchestrator/schemas/process.py,sha256=UACBNt-4g4v9Y528u-gZ-Wk7YxwJHhnI4cEu5CtQm
 orchestrator/schemas/product.py,sha256=MhMCh058ZuS2RJq-wSmxIPUNlhQexxXIx3DSz2OmOh4,1570
 orchestrator/schemas/product_block.py,sha256=kCqvm6qadHpegMr9aWI_fYX-T7mS-5S-ldPxnGQZg7M,1519
 orchestrator/schemas/resource_type.py,sha256=VDju4XywcDDLxdpbWU62RTvR9QF8x_GRrpTlN_NE8uI,1064
+orchestrator/schemas/schedules.py,sha256=Gb427IGR5mPTjKN8STwUhAWCJMCywJkrS8OetiiHTKY,2844
 orchestrator/schemas/search.py,sha256=d_Vs1qU9Z5zuXN4pDk6jrVwiUXRKZ93U-tHW5Zfrw-w,1546
 orchestrator/schemas/search_requests.py,sha256=j2X98eLRTr_dYeGcIc78iPyKoYaAgvl1NQjYV4L1CGY,1925
 orchestrator/schemas/subscription.py,sha256=-jXyHZIed9Xlia18ksSDyenblNN6Q2yM2FlGELyJ458,3423
@@ -331,7 +332,7 @@ orchestrator/services/__init__.py,sha256=GyHNfEFCGKQwRiN6rQmvSRH2iYX7npjMZn97n8X
 orchestrator/services/fixed_inputs.py,sha256=kyz7s2HLzyDulvcq-ZqefTw1om86COvyvTjz0_5CmgI,876
 orchestrator/services/input_state.py,sha256=6BZOpb3cHpO18K-XG-3QUIV9pIM25_ufdODrp5CmXG4,2390
 orchestrator/services/process_broadcast_thread.py,sha256=D44YbjF8mRqGuznkRUV4SoRn1J0lfy_x1H508GnSVlU,4649
-orchestrator/services/processes.py,sha256=LpJbq13UJOrNUKorwYBTV4-MJj-XLXFv6LBk7iyQgl8,30622
+orchestrator/services/processes.py,sha256=vMk30ImSE_0NXM1ffiBvXvaenAeqEYgQbbu_m-4ruGk,30350
 orchestrator/services/products.py,sha256=BP4KyE8zO-8z7Trrs5T6zKBOw53S9BfBJnHWI3p6u5Y,1943
 orchestrator/services/resource_types.py,sha256=_QBy_JOW_X3aSTqH0CuLrq4zBJL0p7Q-UDJUcuK2_qc,884
 orchestrator/services/settings.py,sha256=HEWfFulgoEDwgfxGEO__QTr5fDiwNBEj1UhAeTAdbLQ,3159
@@ -340,12 +341,12 @@ orchestrator/services/subscription_relations.py,sha256=aIdyzwyyy58OFhwjRPCPgnQTU
 orchestrator/services/subscriptions.py,sha256=XhJ5ygAAyWUIZHULhKyi1uU5DwkKZhzdxxn9vdQZYiA,27281
 orchestrator/services/tasks.py,sha256=mR3Fj1VsudltpanJKI2PvrxersyhVQ1skp8H7r3XnYI,5288
 orchestrator/services/translations.py,sha256=GyP8soUFGej8AS8uulBsk10CCK6Kwfjv9AHMFm3ElQY,1713
-orchestrator/services/workflows.py,sha256=iEkt2OBuTwkDru4V6ZSKatnw0b96ZdPV-VQqeZ9EOgU,4015
+orchestrator/services/workflows.py,sha256=H_oAMDuDVmFtCjpuDA75LA4hn5V8ay-uQe_AR98G1V8,4192
 orchestrator/services/executors/__init__.py,sha256=GyHNfEFCGKQwRiN6rQmvSRH2iYX7npjMZn97n8XzmLU,571
 orchestrator/services/executors/celery.py,sha256=j5xJo7sZAdTtc0GmmJzoYVfzuYKiqAdAe5QbtPv0bPI,4937
 orchestrator/services/executors/threadpool.py,sha256=SA0Lns17fP7qp5Y0bLZB7YzZ-sYKrmHQdYTeqs9dnV0,4931
 orchestrator/utils/__init__.py,sha256=GyHNfEFCGKQwRiN6rQmvSRH2iYX7npjMZn97n8XzmLU,571
-orchestrator/utils/auth.py,sha256=CyjVbouzP5-eQ8Fe2kpBdMaBIJ7Ej-Cx8MLq0iaIHa8,344
+orchestrator/utils/auth.py,sha256=H2wqAqq7qPka9SW5gcPbW1ES01nqAfCh_3PnxnBG-0Y,366
 orchestrator/utils/crypt.py,sha256=18eNamYWMllPkxyRtWIde3FDr3rSF74R5SAL6WsCj9Y,5584
 orchestrator/utils/datetime.py,sha256=a1WQ_yvu7MA0TiaRpC5avwbOSFdrj4eMrV4a7I2sD5Q,1477
 orchestrator/utils/deprecation_logger.py,sha256=oqju7ecJcB_r7cMnldaOAA79QUZYS_h69IkDrFV9nAg,875
@@ -365,11 +366,11 @@ orchestrator/utils/search_query.py,sha256=ji5LHtrzohGz6b1IG41cnPdpWXzLEzz4SGWgHl
 orchestrator/utils/state.py,sha256=ELH08cxvpmpnJg_ae0sMi9m_QX6SqHxNzOFaJgyW9gM,14344
 orchestrator/utils/strings.py,sha256=N0gWjmQaMjE9_99VtRvRaU8IBLTKMgBKSXcTZ9TpWAg,1077
 orchestrator/utils/validate_data_version.py,sha256=3Eioy2wE2EWKSgkyMKcEKrkCAfUIAq-eb73iRcpgppw,184
-orchestrator/websocket/__init__.py,sha256=V79jskk1z3uPIYgu0Gt6JLzuqr7NGfNeAZ-hbBqoUv4,5745
+orchestrator/websocket/__init__.py,sha256=M8_QSXOZ9R2hxEu8HoJhQA2YAEAiRXmYJn47Vvz-Y7Q,6321
 orchestrator/websocket/websocket_manager.py,sha256=hwlG9FDXcNU42jDNNsPMQLIyrvEpGX5cm_vrONsLH8s,2763
 orchestrator/websocket/managers/broadcast_websocket_manager.py,sha256=fwoSgTjkHJ2GmsLTU9dqQpAA9i8b1McPu7gLNzxtfG4,5401
 orchestrator/websocket/managers/memory_websocket_manager.py,sha256=lF5EEx1iFMCGEkTbItTDr88NENMSaSeG1QrJ7teoPkY,3324
-orchestrator/workflows/__init__.py,sha256=NzIGGI-8SNAwCk2YqH6sHhEWbgAY457ntDwjO15N8v4,4131
+orchestrator/workflows/__init__.py,sha256=FbwcAYJh8oSi0QFjXXXomdl9c8whCa_qSt_vPXcwasE,4216
 orchestrator/workflows/modify_note.py,sha256=eXt5KQvrkOXf-3YEXCn2XbBLP9N-n1pUYRW2t8Odupo,2150
 orchestrator/workflows/removed_workflow.py,sha256=V0Da5TEdfLdZZKD38ig-MTp3_IuE7VGqzHHzvPYQmLI,909
 orchestrator/workflows/steps.py,sha256=VVLRK9_7KzrBlnK7L8eSmRMNVOO7VJBh5OSjHQHM9fU,7019
@@ -379,8 +380,9 @@ orchestrator/workflows/tasks/cleanup_tasks_log.py,sha256=BfWYbPXhnLAHUJ0mlODDnjZ
 orchestrator/workflows/tasks/resume_workflows.py,sha256=T3iobSJjVgiupe0rClD34kUZ7KF4pL5yK2AVeRLZog8,4313
 orchestrator/workflows/tasks/validate_product_type.py,sha256=lo2TX_MZOfcOmYFjLyD82FrJ5AAN3HOsE6BhDVFuy9Q,3210
 orchestrator/workflows/tasks/validate_products.py,sha256=GZJBoFF-WMphS7ghMs2-gqvV2iL1F0POhk0uSNt93n0,8510
-orchestrator/workflows/translations/en-GB.json,sha256=Gc5gy_RghZOeSNcJIntAsz_7DsCg8n_vzoHBPXxCn_U,908
-orchestrator_core-4.6.5.dist-info/licenses/LICENSE,sha256=b-aA5OZQuuBATmLKo_mln8CQrDPPhg3ghLzjPjLn4Tg,11409
-orchestrator_core-4.6.5.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-orchestrator_core-4.6.5.dist-info/METADATA,sha256=VmDSDmF04lRtR_MSn6yqZc80yZRaFnf2iVO1QeJJgBU,6416
-orchestrator_core-4.6.5.dist-info/RECORD,,
+orchestrator/workflows/tasks/validate_subscriptions.py,sha256=OZtqO6aJR4KwEFvBioV2gWAjmyLiWxe9Wlps3YmFh9w,2200
+orchestrator/workflows/translations/en-GB.json,sha256=ObBlH9XILJ9uNaGcJexi3IB0e6P8CKFKRgu29luIEM8,973
+orchestrator_core-4.7.0rc1.dist-info/licenses/LICENSE,sha256=b-aA5OZQuuBATmLKo_mln8CQrDPPhg3ghLzjPjLn4Tg,11409
+orchestrator_core-4.7.0rc1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+orchestrator_core-4.7.0rc1.dist-info/METADATA,sha256=usMBg-ktGcbAy1WVADeof9gd_oBI4OQg2fPJfOhv60A,6421
+orchestrator_core-4.7.0rc1.dist-info/RECORD,,

orchestrator/schedules/resume_workflows.py

@@ -1,21 +0,0 @@
-# Copyright 2019-2020 SURF.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-from orchestrator.schedules.scheduler import scheduler
-from orchestrator.services.processes import start_process
-
-
-@scheduler.scheduled_job(id="resume-workflows", name="Resume workflows", trigger="interval", hours=1)  # type: ignore[misc]
-def run_resume_workflows() -> None:
-    start_process("task_resume_workflows")

orchestrator/schedules/task_vacuum.py

@@ -1,21 +0,0 @@
-# Copyright 2019-2020 SURF.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-from orchestrator.schedules.scheduler import scheduler
-from orchestrator.services.processes import start_process
-
-
-@scheduler.scheduled_job(id="clean-tasks", name="Clean up tasks", trigger="interval", hours=6)  # type: ignore[misc]
-def vacuum_tasks() -> None:
-    start_process("task_clean_up_tasks")