arcane-execution-handler 1.0.0__tar.gz

arcane_execution_handler-1.0.0/PKG-INFO

@@ -0,0 +1,24 @@
+Metadata-Version: 2.4
+Name: arcane-execution-handler
+Version: 1.0.0
+Summary: Generic execution-start handling for scheduled entities (ingestion, processing, ...)
+Author: Arcane
+Author-email: product@wearcane.com
+Requires-Python: >=3.11,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Dist: arcane-datastore (>=1.1.15,<2.0.0)
+Requires-Dist: arcane-pubsub (>=1.5.0,<2.0.0)
+Requires-Dist: python-dateutil (>=2.7,<3.0)
+Requires-Dist: pytz (>=2024.2,<2025.0)
+Description-Content-Type: text/markdown
+
+# Arcane execution-handler README
+
+
+## Release history
+To see changes, please see CHANGELOG.md
+

arcane_execution_handler-1.0.0/README.md

@@ -0,0 +1,5 @@
+# Arcane execution-handler README
+
+
+## Release history
+To see changes, please see CHANGELOG.md

arcane_execution_handler-1.0.0/arcane/execution_handler/__init__.py

@@ -0,0 +1,3 @@
+from .datastore_lgp import *
+from .execution_handler import *
+from .exception import *

arcane_execution_handler-1.0.0/arcane/execution_handler/datastore_lgp.py

@@ -0,0 +1,32 @@
+from datetime import datetime
+from typing import Optional
+import pytz
+
+from google.cloud.datastore import Entity
+
+from arcane.datastore import DATA_INGESTION_KIND, Client as DatastoreClient
+
+
+def set_execution_info_as_start(
+        datastore_client: DatastoreClient,
+        entity: Entity,
+        kind: str = DATA_INGESTION_KIND,
+        current_time: Optional[datetime] = None
+):
+    """Set an entity's execution_info to `running` on the given Datastore kind.
+    """
+    entity_id = entity['id']
+    if not current_time:
+        current_time = datetime.now().replace(microsecond=0).astimezone(pytz.utc)
+    execution_info = {
+        **entity.get('execution_info', {}),
+        "status": "running",
+        "timestamp": current_time,
+        "errors": []
+    }
+    updated_properties = dict(
+        execution_info=datastore_client.convert_input_to_excluded_entity(
+            execution_info)
+    )
+    datastore_client.save_entity_with_transactions(
+        entity_id, updated_properties, kind)

arcane_execution_handler-1.0.0/arcane/execution_handler/exception.py

@@ -0,0 +1,3 @@
+class SkipExecutionError(Exception):
+    """Custom exception to indicate that an execution should be skipped due to outdated schedule information or concurrent execution."""
+    pass

arcane_execution_handler-1.0.0/arcane/execution_handler/execution_handler.py

@@ -0,0 +1,120 @@
+from datetime import datetime, timedelta, timezone
+import logging
+from typing import Optional
+from dateutil.parser import isoparse
+import pytz
+
+from google.cloud.datastore import Entity
+
+from arcane.datastore import Client as DatastoreClient, DATA_INGESTION_KIND, DATA_PROCESSING_KIND
+from arcane.pubsub import Client as PubSubClient
+
+from .datastore_lgp import set_execution_info_as_start
+from .exception import SkipExecutionError
+from .types import ExecutionStartBody
+
+
+def get_delta_time_for_ingestion_type(data_ingestion_type: str) -> timedelta:
+    """Delta time to consider a running ingestion still in progress"""
+    delta = timedelta(minutes=30)
+    # Because bigquery ingestion is expensive, we prevent overretrying
+    # Because HTTP ingestion can run during 30 minutes and can be retry 3 times
+    if data_ingestion_type == 'BIG_QUERY' or data_ingestion_type == 'HTTP':
+        delta = timedelta(minutes=90)
+    # Because SHOPIFY_API ingestion can run during several minutes and can be retry 5 times (5 bulk exec per token)
+    elif data_ingestion_type == 'SHOPIFY_API':
+        delta = timedelta(minutes=60)
+    return delta
+
+
+def get_delta_time_for_processing() -> timedelta:
+    """Delta time for a running processing"""
+    return timedelta(minutes=90)
+
+
+def _get_schedule_version(entity: Entity, kind: str) -> Optional[str]:
+    if kind == DATA_PROCESSING_KIND:
+        return entity.get('schedule_version')
+    return entity['parameters']['schedule_version']
+
+
+def handle_start_execution(
+    body: ExecutionStartBody,
+    entity: Entity,
+    pubsub_client: PubSubClient,
+    datastore_client: DatastoreClient,
+    project_id: str,
+    pf_monitoring_topic: str,
+    delta_time: timedelta,
+    monitoring_step: str,
+    task_retry_count: int = 0,
+    *,
+    kind: str = DATA_INGESTION_KIND,
+    check_version: bool = True
+):
+
+    """
+    Handle the start of an execution for an entity.
+    This function validates the schedule version, checks if the entity is already running,
+    updates the execution status, and publishes a monitoring event for product flow services.
+    Args:
+        body (dict): The request body containing monitoring_id, entity_id, and schedule_version.
+        entity (Entity): The entity object containing parameters and execution information.
+        pubsub_client (PubSubClient): Client for publishing messages to Pub/Sub topics.
+        datastore_client (DatastoreClient): Client for interacting with the datastore.
+        project_id (str): The GCP project ID.
+        pf_monitoring_topic (str): The Pub/Sub topic name for product flow monitoring.
+        delta_time (timedelta): How long a running execution is still considered in progress
+            (anti over-retry window). Callers pass get_delta_time_for_ingestion_type(type) for
+            ingestions or get_delta_time_for_processing() for processings.
+        monitoring_step (str): The step name to include in the monitoring event.
+        task_retry_count (int): The value of the X-CloudTasks-TaskRetryCount header. If > 0, a running entity is treated as a Cloud Tasks retry and execution is allowed.
+        kind (str): Datastore kind of the entity (DATA_INGESTION_KIND by default).
+        check_version (bool): When False, skip the schedule_version comparison (used for the event-driven / OFF processing path that has no schedule_version). The `running` status is still set. Defaults to True.
+    Raises:
+        SkipExecutionError: If the schedule version is outdated (only when check_version) or if a previous execution
+            is still running (less than the allowed delta time).
+    """
+
+    monitoring_id = body["monitoring_id"]
+    entity_id = int(body["entity_id"])
+
+    schedule_version = body.get('schedule_version')
+    manual_execution = body.get('manual_execution', False)
+    if manual_execution:
+        logging.info(f'Manual execution for entity with id {entity_id}. Running without checking schedule version and execution status.')
+    else:
+        if entity['enabled'] is False:
+            logging.info(f'Entity with id {entity_id} is disabled. Skipping execution.')
+            raise SkipExecutionError('Entity is disabled')
+
+        if check_version:
+            current_schedule_version = _get_schedule_version(entity, kind)
+            if schedule_version != current_schedule_version:
+                logging.info(f'Entity with id {entity_id} has outdated schedule info. Previous version {schedule_version} and {current_schedule_version}')
+                raise SkipExecutionError(f'Schedule info is outdated. Previous version {schedule_version} and current version {current_schedule_version}')
+
+
+        if entity.get('execution_info') is not None and 'status' in entity['execution_info']:
+            if entity['execution_info']['status'] == 'running':
+                timestamp = entity['execution_info'].get('timestamp')
+                if timestamp and not isinstance(timestamp, datetime):
+                    timestamp = isoparse(timestamp).astimezone(pytz.utc)
+                if timestamp and (datetime.now(timezone.utc) - timestamp).total_seconds() < delta_time.total_seconds():
+                    if task_retry_count > 0:
+                        logging.info(f'Entity with id {entity_id} is already running but this is a Cloud Tasks retry (retry_count={task_retry_count}). Allowing retry.')
+                    else:
+                        logging.info(f'Entity with id {entity_id} is already running since {timestamp}. Skipping execution.')
+                        raise SkipExecutionError('Previous execution is still running')
+
+        set_execution_info_as_start(datastore_client, entity, kind)
+    if entity['service'] != 'datalab':
+        pubsub_client.pubsub_publish_pf_monitoring(
+            monitoring_id=monitoring_id,
+            entity_id=str(entity_id),
+            step=monitoring_step,
+            status='start',
+            project_id=project_id,
+            topic=pf_monitoring_topic
+        )
+

arcane_execution_handler-1.0.0/arcane/execution_handler/types.py

@@ -0,0 +1,10 @@
+from typing import Any, NotRequired, Optional, TypedDict
+
+
+class ExecutionStartBody(TypedDict):
+    """Expected body of an execution-start event (ingestion or processing)."""
+    monitoring_id: str
+    entity_id: Any
+    schedule_version: NotRequired[Optional[str]]
+    manual_execution: NotRequired[bool]
+    execution_time: NotRequired[str]

arcane_execution_handler-1.0.0/pyproject.toml

@@ -0,0 +1,25 @@
+[tool.poetry]
+name = "arcane-execution-handler"
+version = "1.0.0"
+description = "Generic execution-start handling for scheduled entities (ingestion, processing, ...)"
+readme = "README.md"
+authors = ["Arcane <product@wearcane.com>"]
+packages = [
+    { include = "arcane" }
+]
+
+[tool.poetry.dependencies]
+python = "^3.11"
+arcane-datastore = "^1.1.15"
+arcane-pubsub = "^1.5.0"
+python-dateutil = "^2.7"
+pytz = "^2024.2"
+
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^7.0"
+pytest-mock = "^3.10"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"