arcane-ingestion 0.1.0__tar.gz

arcane_ingestion-0.1.0/PKG-INFO

@@ -0,0 +1,24 @@
+Metadata-Version: 2.4
+Name: arcane-ingestion
+Version: 0.1.0
+Summary: Utils function for ingestion entity
+Author: Arcane
+Author-email: product@wearcane.com
+Requires-Python: >=3.11,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Dist: arcane-datastore (>=1.1.15,<2.0.0)
+Requires-Dist: arcane-pubsub (>=1.5.0,<2.0.0)
+Requires-Dist: python-dateutil (>=2.7,<3.0)
+Requires-Dist: pytz (>=2024.2,<2025.0)
+Description-Content-Type: text/markdown
+
+# Arcane ingestion README
+
+
+## Release history
+To see changes, please see CHANGELOG.md
+

arcane_ingestion-0.1.0/README.md

@@ -0,0 +1,5 @@
+# Arcane ingestion README
+
+
+## Release history
+To see changes, please see CHANGELOG.md

arcane_ingestion-0.1.0/arcane/ingestion/__init__.py

@@ -0,0 +1,3 @@
+from .datastore_lgp import *
+from .execution_handler import *
+from .exception import *

arcane_ingestion-0.1.0/arcane/ingestion/datastore_lgp.py

@@ -0,0 +1,29 @@
+from datetime import datetime
+from typing import Optional
+import pytz
+
+from google.cloud.datastore import Entity
+
+from arcane.datastore import DATA_INGESTION_KIND, Client as DatastoreClient
+
+
+def set_ingestion_execution_info_as_start(
+        datastore_client: DatastoreClient,
+        ingestion: Entity,
+        current_time: Optional[datetime] = None
+):
+    ingestion_id = ingestion['id']
+    if not current_time:
+        current_time = datetime.now().replace(microsecond=0).astimezone(pytz.utc)
+    execution_info = {
+        **ingestion.get('execution_info', {}),
+        "status": "running",
+        "timestamp": current_time,
+        "errors": []
+    }
+    updated_properties = dict(
+        execution_info=datastore_client.convert_input_to_excluded_entity(
+            execution_info)
+    )
+    datastore_client.save_entity_with_transactions(
+        ingestion_id, updated_properties, DATA_INGESTION_KIND)

arcane_ingestion-0.1.0/arcane/ingestion/exception.py

@@ -0,0 +1,3 @@
+class SkipExecutionError(Exception):
+    """Custom exception to indicate that an execution should be skipped due to outdated schedule information or concurrent execution."""
+    pass

arcane_ingestion-0.1.0/arcane/ingestion/execution_handler.py

@@ -0,0 +1,90 @@
+from datetime import datetime, timedelta, timezone
+import logging
+from dateutil.parser import isoparse
+import pytz
+
+from google.cloud.datastore import Entity
+
+from arcane.datastore import Client as DatastoreClient
+from arcane.pubsub import Client as PubSubClient
+
+from .datastore_lgp import set_ingestion_execution_info_as_start
+from .exception import SkipExecutionError
+
+
+def _get_delta_time_for_ingestion_type(data_ingestion_type: str) -> timedelta:
+    """Determine the delta time to consider for skipping execution based on the ingestion type. This helps to prevent over-retrying for expensive or long-running ingestions."""
+    delta = timedelta(minutes=30)
+    # Because bigquery ingestion is expensive, we prevent overretrying
+    # Because HTTP ingestion can run during 30 minutes and can be retry 3 times
+    if data_ingestion_type == 'BIG_QUERY' or data_ingestion_type == 'HTTP':
+        delta = timedelta(minutes=90)
+    # Because SHOPIFY_API ingestion can run during several minutes and can be retry 5 times (5 bulk exec per token)
+    elif data_ingestion_type == 'SHOPIFY_API':
+        delta = timedelta(minutes=60)
+    return delta
+
+
+
+def handle_start_execution(
+    body: dict,
+    entity: Entity,
+    pubsub_client: PubSubClient,
+    datastore_client: DatastoreClient,
+    project_id: str,
+    pf_monitoring_topic: str,
+    ingestion_type: str,
+    monitoring_step: str
+):
+
+    """
+    Handle the start of an execution for an entity.
+    This function validates the schedule version, checks if the entity is already running,
+    updates the execution status, and publishes a monitoring event for product flow services.
+    Args:
+        body (dict): The request body containing monitoring_id, entity_id, and schedule_version.
+        entity (Entity): The entity object containing parameters and execution information.
+        pubsub_client (PubSubClient): Client for publishing messages to Pub/Sub topics.
+        datastore_client (DatastoreClient): Client for interacting with the datastore.
+        project_id (str): The GCP project ID.
+        pf_monitoring_topic (str): The Pub/Sub topic name for product flow monitoring.
+        ingestion_type (str): The type of ingestion (e.g., 'BIG_QUERY', 'HTTP', 'SHOPIFY_API') to determine the waiting queue time.
+        monitoring_step (str): The step name to include in the monitoring event.
+    Raises:
+        SkipExecutionError: If the schedule version is outdated or if a previous execution
+            is still running (less than the allowed delta time for the ingestion type).
+    """
+
+    monitoring_id = body["monitoring_id"]
+    entity_id = int(body["entity_id"])
+
+    schedule_version = body['schedule_version']
+    manual_execution = body.get('manual_execution', False)
+    if manual_execution:
+        logging.info(f'Manual execution for entity with id {entity_id}. Running without checking schedule version and execution status.')
+    else:
+        if schedule_version != entity['parameters']['schedule_version']:
+            logging.info(f'Entity with id {entity_id} has its schedule info. Previous version {schedule_version} and {entity["parameters"]["schedule_version"]}')
+            raise SkipExecutionError(f'Schedule info is outdated. Previous version {schedule_version} and current version {entity["parameters"]["schedule_version"]}')
+
+
+        if entity.get('execution_info') is not None and 'status' in entity['execution_info']:
+            if entity['execution_info']['status'] == 'running':
+                timestamp = entity['execution_info'].get('timestamp')
+                if timestamp and not isinstance(timestamp, datetime):
+                    timestamp = isoparse(timestamp).astimezone(pytz.utc)
+                if timestamp and (datetime.now(timezone.utc) - timestamp).total_seconds() < _get_delta_time_for_ingestion_type(ingestion_type).total_seconds():
+                    logging.info(f'Entity with id {entity_id} is already running since {timestamp}. Skipping execution.')
+                    raise SkipExecutionError('Previous execution is still running')
+
+        set_ingestion_execution_info_as_start(datastore_client, entity)
+    if entity['service'] != 'datalab':
+        pubsub_client.pubsub_publish_pf_monitoring(
+            monitoring_id=monitoring_id,
+            entity_id=str(entity_id),
+            step=monitoring_step,
+            status='start',
+            project_id=project_id,
+            topic=pf_monitoring_topic
+        )
+

arcane_ingestion-0.1.0/pyproject.toml

@@ -0,0 +1,25 @@
+[tool.poetry]
+name = "arcane-ingestion"
+version = "0.1.0"
+description = "Utils function for ingestion entity"
+readme = "README.md"
+authors = ["Arcane <product@wearcane.com>"]
+packages = [
+    { include = "arcane" }
+]
+
+[tool.poetry.dependencies]
+python = "^3.11"
+arcane-datastore = "^1.1.15"
+arcane-pubsub = "^1.5.0"
+python-dateutil = "^2.7"
+pytz = "^2024.2"
+
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^7.0"
+pytest-mock = "^3.10"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"