ingestify 0.3.4__py3-none-any.whl → 0.4.1__py3-none-any.whl

ingestify/__init__.py

@@ -8,4 +8,4 @@ if not __INGESTIFY_SETUP__:
     from .infra import retrieve_http
     from .source_base import Source, DatasetResource
 
-__version__ = "0.3.4"
+__version__ = "0.4.1"

ingestify/application/dataset_store.py

@@ -270,6 +270,7 @@ class DatasetStore:
             metadata=metadata,
             created_at=now,
             updated_at=now,
+            last_modified_at=None,  # Not known at this moment
         )
         revision = self.add_revision(dataset, files, revision_source, description)
 

ingestify/application/ingestion_engine.py

@@ -21,8 +21,13 @@ class IngestionEngine:
     def add_ingestion_plan(self, ingestion_plan: IngestionPlan):
         self.loader.add_ingestion_plan(ingestion_plan)
 
-    def load(self, dry_run: bool = False, provider: Optional[str] = None):
-        self.loader.collect_and_run(dry_run=dry_run, provider=provider)
+    def load(
+        self,
+        dry_run: bool = False,
+        provider: Optional[str] = None,
+        source: Optional[str] = None,
+    ):
+        self.loader.collect_and_run(dry_run=dry_run, provider=provider, source=source)
 
     def list_datasets(self, as_count: bool = False):
         """Consider moving this to DataStore"""

ingestify/application/loader.py

@@ -29,7 +29,12 @@ class Loader:
     def add_ingestion_plan(self, ingestion_plan: IngestionPlan):
         self.ingestion_plans.append(ingestion_plan)
 
-    def collect_and_run(self, dry_run: bool = False, provider: Optional[str] = None):
+    def collect_and_run(
+        self,
+        dry_run: bool = False,
+        provider: Optional[str] = None,
+        source: Optional[str] = None,
+    ):
         # First collect all selectors, before discovering datasets
         selectors = {}
         for ingestion_plan in self.ingestion_plans:
@@ -42,6 +47,13 @@ class Loader:
                     )
                     continue
 
+            if source is not None:
+                if ingestion_plan.source.name != source:
+                    logger.info(
+                        f"Skipping {ingestion_plan} because source doesn't match '{source}'"
+                    )
+                    continue
+
             static_selectors = [
                 selector
                 for selector in ingestion_plan.selectors
@@ -60,6 +72,7 @@ class Loader:
 
                     # TODO: consider making this lazy and fetch once per Source instead of
                     #       once per IngestionPlan
+                    # TODO: Log exception when `discover_selectors` fails
                     all_selectors = ingestion_plan.source.discover_selectors(
                         ingestion_plan.dataset_type
                     )

ingestify/cmdline.py

@@ -58,7 +58,14 @@ def cli():
     help="bucket",
     type=str,
 )
-@click.option("--debug", "debug", required=False, help="Debugging enabled", type=bool)
+@click.option(
+    "--debug",
+    "debug",
+    required=False,
+    help="Debugging enabled",
+    is_flag=True,
+    type=bool,
+)
 @click.option(
     "--dry-run",
     "dry_run",
@@ -74,11 +81,19 @@ def cli():
     help="Provider - only run tasks for a single provider",
     type=str,
 )
+@click.option(
+    "--source",
+    "source",
+    required=False,
+    help="Source - only run tasks for a single source",
+    type=str,
+)
 def run(
     config_file: str,
     bucket: Optional[str],
     dry_run: Optional[bool],
     provider: Optional[str],
+    source: Optional[str],
     debug: Optional[bool],
 ):
     try:
@@ -90,7 +105,10 @@ def run(
             logger.exception(f"Failed due a configuration error: {e}")
             sys.exit(1)
 
-    engine.load(dry_run=dry_run, provider=provider)
+    if debug:
+        logging.getLogger("root").setLevel(logging.DEBUG)
+
+    engine.load(dry_run=dry_run, provider=provider, source=source)
 
     logger.info("Done")
 

ingestify/domain/models/dataset/collection_metadata.py

@@ -6,7 +6,8 @@ from typing import Optional
 @dataclass
 class DatasetCollectionMetadata:
     # This can be useful to figure out if a backfill is required
-    first_modified: Optional[datetime]
+    # TODO - Note: not stored at Dataset level and requires joined query to retrieve
+    # first_modified: Optional[datetime]
 
     # Use the last modified to only retrieve datasets that are changed
     last_modified: Optional[datetime]

ingestify/domain/models/dataset/dataset.py

@@ -22,7 +22,10 @@ class Dataset(BaseModel):
     metadata: dict
     created_at: datetime
     updated_at: datetime
+
     revisions: List[Revision] = Field(default_factory=list)
+    # The last_modified_at is equal to the max modified_at of all files in all revisions
+    last_modified_at: Optional[datetime]
 
     @field_validator("identifier", mode="before")
     @classmethod
@@ -42,6 +45,13 @@ class Dataset(BaseModel):
         self.revisions.append(revision)
         self.updated_at = utcnow()
 
+        if self.last_modified_at:
+            self.last_modified_at = max(
+                self.last_modified_at, revision.last_modified_at
+            )
+        else:
+            self.last_modified_at = revision.last_modified_at
+
     def update_metadata(self, name: str, metadata: dict, state: DatasetState) -> bool:
         changed = False
         if self.name != name:

ingestify/domain/models/dataset/revision.py

@@ -36,6 +36,10 @@ class Revision(BaseModel):
     is_squashed: bool = False
     state: RevisionState = RevisionState.PENDING_VALIDATION
 
+    @property
+    def last_modified_at(self):
+        return max(file.modified_at for file in self.modified_files)
+
     @property
     def modified_files_map(self) -> Dict[str, File]:
         return {file.file_id: file for file in self.modified_files}

ingestify/domain/models/ingestion/ingestion_job.py

@@ -214,9 +214,6 @@ class IngestionJob:
         self, store: DatasetStore, task_executor: TaskExecutor
     ) -> Iterator[IngestionJobSummary]:
         is_first_chunk = True
-        ingestion_job_exception = (
-            None  # Indicate if there was an exception during the IngestionJob itself
-        )
         ingestion_job_summary = IngestionJobSummary.new(ingestion_job=self)
         # Process all items in batches. Yield a IngestionJobSummary per batch
 
@@ -224,6 +221,7 @@ class IngestionJob:
         with ingestion_job_summary.record_timing("get_dataset_collection"):
             dataset_collection_metadata = store.get_dataset_collection(
                 dataset_type=self.ingestion_plan.dataset_type,
+                provider=self.ingestion_plan.source.provider,
                 data_spec_versions=self.selector.data_spec_versions,
                 selector=self.selector,
                 metadata_only=True,
@@ -233,8 +231,8 @@ class IngestionJob:
         # There are two different, but similar flows here:
         # 1. The discover_datasets returns a list, and the entire list can be processed at once
         # 2. The discover_datasets returns an iterator of batches, in this case we need to process each batch
-        with ingestion_job_summary.record_timing("find_datasets"):
-            try:
+        try:
+            with ingestion_job_summary.record_timing("find_datasets"):
                 dataset_resources = self.ingestion_plan.source.find_datasets(
                     dataset_type=self.ingestion_plan.dataset_type,
                     data_spec_versions=self.selector.data_spec_versions,
@@ -244,12 +242,12 @@ class IngestionJob:
 
                 # We need to include the to_batches as that will start the generator
                 batches = to_batches(dataset_resources)
-            except Exception as e:
-                logger.exception("Failed to find datasets")
+        except Exception as e:
+            logger.exception("Failed to find datasets")
 
-                ingestion_job_summary.set_exception(e)
-                yield ingestion_job_summary
-                return
+            ingestion_job_summary.set_exception(e)
+            yield ingestion_job_summary
+            return
 
         finish_task_timer = ingestion_job_summary.start_timing("tasks")
 

ingestify/domain/models/ingestion/ingestion_job_summary.py

@@ -9,7 +9,7 @@ from ingestify.domain import Selector, DataSpecVersionCollection
 from ingestify.domain.models.base import BaseModel
 from ingestify.domain.models.task.task_summary import TaskSummary, TaskState
 from ingestify.domain.models.timing import Timing
-from ingestify.utils import utcnow
+from ingestify.utils import utcnow, HasTiming
 
 if TYPE_CHECKING:
     from ingestify.domain.models.ingestion.ingestion_job import IngestionJob
@@ -25,7 +25,7 @@ def format_duration(duration: timedelta):
     return f"{duration.total_seconds():.2f}sec"
 
 
-class IngestionJobSummary(BaseModel):
+class IngestionJobSummary(BaseModel, HasTiming):
     ingestion_job_summary_id: str
     ingestion_job_id: str
 
@@ -39,7 +39,6 @@ class IngestionJobSummary(BaseModel):
     started_at: datetime = Field(default_factory=utcnow)
     ended_at: Optional[datetime] = None
     state: IngestionJobState = IngestionJobState.RUNNING
-    timings: List[Timing] = Field(default_factory=list)
     task_summaries: List[TaskSummary] = Field(default_factory=list)
 
     skipped_datasets: int = 0
@@ -60,22 +59,6 @@ class IngestionJobSummary(BaseModel):
         )
         return cls(**args)
 
-    @contextmanager
-    def record_timing(self, name: str):
-        start = utcnow()
-        try:
-            yield
-        finally:
-            self.timings.append(Timing(name=name, started_at=start, ended_at=utcnow()))
-
-    def start_timing(self, name):
-        start = utcnow()
-
-        def finish():
-            self.timings.append(Timing(name=name, started_at=start, ended_at=utcnow()))
-
-        return finish
-
     def add_task_summaries(self, task_summaries: List[TaskSummary]):
         self.task_summaries.extend(task_summaries)
 
@@ -101,6 +84,11 @@ class IngestionJobSummary(BaseModel):
         )
         self.ended_at = utcnow()
 
+        # Only keep failed tasks. Rest isn't interesting
+        self.task_summaries = [
+            task for task in self.task_summaries if task.state == TaskState.FAILED
+        ]
+
     def set_finished(self):
         self.state = IngestionJobState.FINISHED
         self._set_ended()
@@ -114,25 +102,27 @@ class IngestionJobSummary(BaseModel):
         return self.ended_at - self.started_at
 
     def output_report(self):
-        print(f"\nIngestionJobSummary {self.state} in {format_duration(self.duration)}")
-        print("--------------------")
-        print(f"  - IngestionPlan:")
-        print(f"        Source: {self.source_name}")
-        print(f"        Provider: {self.provider}")
-        print(f"        DatasetType: {self.dataset_type}")
-        print(f"  - Selector: {self.selector}")
-        print(f"  - Timings: ")
+        print(
+            f"\nIngestionJobSummary {self.state.value} in {format_duration(self.duration)}"
+        )
+        print("********************************")
+        print(f"*  - IngestionPlan:")
+        print(f"*        Source: {self.source_name}")
+        print(f"*        Provider: {self.provider}")
+        print(f"*        DatasetType: {self.dataset_type}")
+        print(f"*  - Selector: {self.selector}")
+        print(f"*  - Timings: ")
         for timing in self.timings:
-            print(f"    - {timing.name}: {format_duration(timing.duration)}")
+            print(f"*    - {timing.name}: {format_duration(timing.duration)}")
         print(
-            f"  - Tasks: {len(self.task_summaries)} - {(len(self.task_summaries) / self.duration.total_seconds()):.1f} tasks/sec"
+            f"*  - Tasks: {len(self.task_summaries)} - {(len(self.task_summaries) / self.duration.total_seconds()):.1f} tasks/sec"
         )
 
-        print(f"    - Failed tasks: {self.failed_tasks}")
-        print(f"    - Successful tasks: {self.successful_tasks}")
-        print(f"    - Successful ignored tasks: {self.ignored_successful_tasks}")
-        print(f"    - Skipped datasets: {self.skipped_datasets}")
-        print("--------------------")
+        print(f"*    - Failed tasks: {self.failed_tasks}")
+        print(f"*    - Successful tasks: {self.successful_tasks}")
+        print(f"*    - Successful ignored tasks: {self.ignored_successful_tasks}")
+        print(f"*    - Skipped datasets: {self.skipped_datasets}")
+        print("********************************")
 
     def __enter__(self):
         return self

ingestify/domain/models/task/task_summary.py

@@ -10,8 +10,7 @@ from ingestify.domain.models.base import BaseModel
 from ingestify.domain.models.dataset.identifier import Identifier
 from ingestify.domain.models.timing import Timing
 from ingestify.exceptions import IngestifyError
-from ingestify.utils import utcnow
-
+from ingestify.utils import utcnow, HasTiming
 
 logger = logging.getLogger(__name__)
 
@@ -28,7 +27,7 @@ class Operation(str, Enum):
     UPDATE = "UPDATE"
 
 
-class TaskSummary(BaseModel):
+class TaskSummary(BaseModel, HasTiming):
     task_id: str
     started_at: datetime
     operation: Operation
@@ -38,7 +37,6 @@ class TaskSummary(BaseModel):
     bytes_retrieved: int = 0
     last_modified: Optional[datetime] = None
     state: TaskState = TaskState.RUNNING
-    timings: List[Timing] = Field(default_factory=list)
 
     @field_validator("dataset_identifier", mode="before")
     @classmethod
@@ -48,27 +46,8 @@ class TaskSummary(BaseModel):
         return value
 
     def record_load_file(self, fn, metadata: dict):
-        start = utcnow()
-        try:
-            result = None
+        with self.record_timing(f"Load of {metadata.get('file_id', 'file')}", metadata):
             return fn()
-        except Exception as e:
-            result = {
-                "type": type(e).__name__,
-                "message": str(e),
-                "traceback": traceback.format_exc(),
-            }
-            raise e
-        finally:
-            metadata = dict(result=result, **metadata)
-            self.timings.append(
-                Timing(
-                    name=f"Load of {metadata.get('file_id', 'file')}",
-                    started_at=start,
-                    ended_at=utcnow(),
-                    metadata=metadata,
-                )
-            )
 
     @classmethod
     @contextmanager

ingestify/infra/serialization/__init__.py

@@ -1,12 +1,5 @@
-import json
-from datetime import datetime
-from typing import Type, Any, TypeVar
-
-from dataclass_factory import Schema, Factory, NameStyle
-from dataclass_factory.schema_helpers import type_checker
-
-from ingestify.domain import DatasetCreated, Identifier
-from ingestify.domain.models.dataset.events import MetadataUpdated, RevisionAdded
+from ingestify.domain import DatasetCreated
+from ingestify.domain.models.dataset.events import RevisionAdded
 from ingestify.domain.models.event import DomainEvent
 
 
@@ -18,10 +11,6 @@ event_types = {
 
 def deserialize(event_dict: dict) -> DomainEvent:
     event_cls = event_types[event_dict["event_type"]]
-    event_dict["dataset"]["identifier"] = Identifier(
-        **event_dict["dataset"]["identifier"]
-    )
-
     return event_cls.model_validate(event_dict)
 
 

ingestify/infra/store/dataset/sqlalchemy/repository.py

@@ -1,32 +1,31 @@
 import itertools
-import json
 import uuid
-from collections import defaultdict
 from typing import Optional, Union, List
 
 from sqlalchemy import (
     create_engine,
     func,
     text,
-    tuple_,
     Table,
-    insert,
-    Transaction,
     Connection,
+    union_all,
+    literal,
+    select,
+    and_,
+    Column,
+    or_,
 )
 from sqlalchemy.engine import make_url
 from sqlalchemy.exc import NoSuchModuleError
-from sqlalchemy.orm import Session, joinedload
+from sqlalchemy.orm import Session
 
 from ingestify.domain import File, Revision
 from ingestify.domain.models import (
     Dataset,
     DatasetCollection,
     DatasetRepository,
-    Identifier,
     Selector,
 )
-from ingestify.domain.models.base import BaseModel
 from ingestify.domain.models.dataset.collection_metadata import (
     DatasetCollectionMetadata,
 )
@@ -127,6 +126,10 @@ class SqlAlchemySessionProvider:
         return self.session
 
 
+def in_(column: Column, values):
+    return or_(*[column == value for value in values])
+
+
 class SqlAlchemyDatasetRepository(DatasetRepository):
     def __init__(self, session_provider: SqlAlchemySessionProvider):
         self.session_provider = session_provider
@@ -169,11 +172,6 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
         dataset_id: Optional[Union[str, List[str]]] = None,
         selector: Optional[Union[Selector, List[Selector]]] = None,
     ):
-        query = query.filter(dataset_table.c.bucket == bucket)
-        if dataset_type:
-            query = query.filter(dataset_table.c.dataset_type == dataset_type)
-        if provider:
-            query = query.filter(dataset_table.c.provider == provider)
         if dataset_id is not None:
             if isinstance(dataset_id, list):
                 if len(dataset_id) == 0:
@@ -181,7 +179,7 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
                     # return an empty DatasetCollection
                     return DatasetCollection()
 
-                query = query.filter(dataset_table.c.dataset_id.in_(dataset_id))
+                query = query.filter(in_(dataset_table.c.dataset_id, dataset_id))
             else:
                 query = query.filter(dataset_table.c.dataset_id == dataset_id)
 
@@ -201,13 +199,25 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
             if not selectors:
                 raise ValueError("Selectors must contain at least one item")
 
-            keys = list(selectors[0].filtered_attributes.keys())
+            attribute_keys = selectors[
+                0
+            ].filtered_attributes.keys()  # Assume all selectors have the same keys
+            attribute_sets = {
+                tuple(selector.filtered_attributes.items()) for selector in selectors
+            }
+
+            # Define a virtual table using a CTE for all attributes
+            attribute_cte = union_all(
+                *[
+                    select(*(literal(value).label(key) for key, value in attr_set))
+                    for attr_set in attribute_sets
+                ]
+            ).cte("attributes")
 
-            columns = []
+            keys = list(selectors[0].filtered_attributes.keys())
             first_selector = selectors[0].filtered_attributes
 
-            # Create a query like this:
-            #  SELECT * FROM dataset WHERE (column1, column2, column3) IN ((1, 2, 3), (4, 5, 6), (7, 8, 9))
+            join_conditions = []
             for k in keys:
                 if dialect == "postgresql":
                     column = dataset_table.c.identifier[k]
@@ -215,25 +225,28 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
                     # Take the value from the first selector to determine the type.
                     # TODO: check all selectors to determine the type
                     v = first_selector[k]
-                    if isint(v):
+                    if isinstance(v, int):
                         column = column.as_integer()
-                    elif isfloat(v):
-                        column = column.as_float()
                     else:
                         column = column.as_string()
                 else:
                     column = func.json_extract(dataset_table.c.identifier, f"$.{k}")
-                columns.append(column)
 
-            values = []
-            for selector in selectors:
-                filtered_attributes = selector.filtered_attributes
-                values.append(tuple([filtered_attributes[k] for k in keys]))
+                join_conditions.append(attribute_cte.c[k] == column)
 
-            query = query.filter(tuple_(*columns).in_(values))
+            query = query.select_from(
+                dataset_table.join(attribute_cte, and_(*join_conditions))
+            )
 
         if where:
             query = query.filter(text(where))
+
+        query = query.filter(dataset_table.c.bucket == bucket)
+        if dataset_type:
+            query = query.filter(dataset_table.c.dataset_type == dataset_type)
+        if provider:
+            query = query.filter(dataset_table.c.provider == provider)
+
         return query
 
     def load_datasets(self, dataset_ids: list[str]) -> list[Dataset]:
@@ -242,13 +255,13 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
 
         dataset_rows = list(
             self.session.query(dataset_table).filter(
-                dataset_table.c.dataset_id.in_(dataset_ids)
+                in_(dataset_table.c.dataset_id, dataset_ids)
             )
         )
         revisions_per_dataset = {}
         rows = (
             self.session.query(revision_table)
-            .filter(revision_table.c.dataset_id.in_(dataset_ids))
+            .filter(in_(revision_table.c.dataset_id, dataset_ids))
             .order_by(revision_table.c.dataset_id)
         )
 
@@ -260,7 +273,7 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
         files_per_revision = {}
         rows = (
             self.session.query(file_table)
-            .filter(file_table.c.dataset_id.in_(dataset_ids))
+            .filter(in_(file_table.c.dataset_id, dataset_ids))
             .order_by(file_table.c.dataset_id, file_table.c.revision_id)
         )
 
@@ -320,10 +333,9 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
 
         metadata_result_row = apply_query_filter(
             self.session.query(
-                func.min(file_table.c.modified_at).label("first_modified_at"),
-                func.max(file_table.c.modified_at).label("last_modified_at"),
+                func.max(dataset_table.c.last_modified_at).label("last_modified_at"),
                 func.count().label("row_count"),
-            ).join(dataset_table, dataset_table.c.dataset_id == file_table.c.dataset_id)
+            )
         ).first()
         dataset_collection_metadata = DatasetCollectionMetadata(*metadata_result_row)
 

ingestify/infra/store/dataset/sqlalchemy/tables.py

@@ -14,8 +14,11 @@ from sqlalchemy import (
     String,
     Table,
     TypeDecorator,
+    Index,
 )
 
+from sqlalchemy.dialects.postgresql import JSONB
+
 from ingestify.domain import Identifier, DataSpecVersionCollection, Selector
 from ingestify.domain.models.dataset.dataset import DatasetState
 from ingestify.domain.models.ingestion.ingestion_job_summary import IngestionJobState
@@ -25,18 +28,18 @@ from ingestify.domain.models.timing import Timing
 from ingestify.domain.models.dataset.revision import RevisionState
 
 
-def JSONType(serializer=None, deserializer=None):
+def JSONType(serializer=None, deserializer=None, base_type=JSON):
     class _JsonType(TypeDecorator):
         cache_ok = True
-        impl = JSON
+        impl = base_type
 
         def process_bind_param(self, value, dialect):
-            if serializer is not None:
+            if serializer and value is not None:
                 return serializer(value)
             return value
 
         def process_result_value(self, value, dialect):
-            if deserializer is not None:
+            if deserializer and value is not None:
                 return deserializer(value)
             return value
 
@@ -152,14 +155,28 @@ dataset_table = Table(
     metadata,
     Column("bucket", String(255), default=None),
     Column("dataset_id", String(255), primary_key=True),
-    Column("provider", String(255)),
-    Column("dataset_type", String(255)),
+    Column("provider", String(255), index=True),
+    Column("dataset_type", String(255), index=True),
     Column("state", DatasetStateString),
     Column("name", String(255)),
-    Column("identifier", JSONType(deserializer=lambda item: Identifier(**item))),
+    Column(
+        "identifier",
+        # Use JSONB when available
+        JSON().with_variant(JSONB(), "postgresql"),
+    ),
     Column("metadata", JSON),
     Column("created_at", TZDateTime(6)),
     Column("updated_at", TZDateTime(6)),
+    Column("last_modified_at", TZDateTime(6)),
+    # Required for performance querying when there are a lot of Datasets
+    # with the same provider and dataset_type
+    Index(
+        "idx_bucket_type_provider_last_modified",
+        "bucket",
+        "provider",
+        "dataset_type",
+        "last_modified_at",
+    ),
 )
 
 revision_table = Table(

ingestify/utils.py

@@ -1,34 +1,23 @@
-import abc
-import asyncio
-import inspect
 import logging
 import os
 import time
 import re
+import traceback
+from contextlib import contextmanager
 from multiprocessing import get_context, cpu_count, get_all_start_methods
 
 from datetime import datetime, timezone
 from string import Template
-from typing import (
-    Dict,
-    Generic,
-    Type,
-    TypeVar,
-    Tuple,
-    Optional,
-    Any,
-    Callable,
-    Awaitable,
-    List,
-    Iterable,
-)
+from typing import Dict, Tuple, Optional, Any, List
 
 import cloudpickle
+from pydantic import Field
 from typing_extensions import Self
 
 
 from itertools import islice
 
+from ingestify.domain.models.timing import Timing
 
 logger = logging.getLogger(__name__)
 
@@ -221,3 +210,46 @@ def try_number(s: str):
             return float(s)
         except ValueError:
             return s
+
+
+class HasTiming:
+    """Mixin to give Pydantic models ability to time actions."""
+
+    timings: List[Timing] = Field(default_factory=list)
+
+    @contextmanager
+    def record_timing(
+        self, description: str, metadata: Optional[dict] = None
+    ) -> Timing:
+        if not metadata:
+            metadata = {}
+
+        start = utcnow()
+        try:
+            result = None
+            yield
+        except Exception as e:
+            result = {
+                "type": type(e).__name__,
+                "message": str(e),
+                "traceback": traceback.format_exc(),
+            }
+            raise e
+        finally:
+            metadata = dict(result=result, **metadata)
+            self.timings.append(
+                Timing(
+                    name=description,
+                    started_at=start,
+                    ended_at=utcnow(),
+                    metadata=metadata,
+                )
+            )
+
+    def start_timing(self, name):
+        start = utcnow()
+
+        def finish():
+            self.timings.append(Timing(name=name, started_at=start, ended_at=utcnow()))
+
+        return finish

{ingestify-0.3.4.dist-info → ingestify-0.4.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.3.4
+Version: 0.4.1
 Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl

{ingestify-0.3.4.dist-info → ingestify-0.4.1.dist-info}/RECORD

@@ -1,14 +1,14 @@
-ingestify/__init__.py,sha256=lyBZ_P8y4qlkE1e11F4T41fSTp8WbReifRxX9UGizxA,301
-ingestify/cmdline.py,sha256=bIuyPgGEw4wIglNzpG9zp7TsJozsP8NSVsCe4eAyWUg,7189
+ingestify/__init__.py,sha256=xCS7JQ_JaB6zVzrq6WUeAZyNxVKJEOc7AKh-3vY_Ji8,301
+ingestify/cmdline.py,sha256=oagUe-Jup1SU3s6jVl25f0cSG0wlNYhxFY-gGBwWmr0,7482
 ingestify/exceptions.py,sha256=izRzaLQmMy-4P8ZqGqVZyf4k6LFYOYqwYLuRaUH8BJw,187
 ingestify/main.py,sha256=Xr0VbGgstPO7doDX18xqk4lBb4W2sbGWtQuXZaARsHA,8763
 ingestify/server.py,sha256=OVrf_XtpAQIn88MzqQzShXgsA9_jbnqYvD8YPBjn3cs,2413
 ingestify/source_base.py,sha256=GXAFCoT11Zov9M2v-fqQr9gFCXbtVfEIEH32V7r2oE8,382
-ingestify/utils.py,sha256=HETGhAoUlutLG0cQR63nac2JbFei9gnktDHeBQoYWfU,5692
+ingestify/utils.py,sha256=6BqgEZjecLW_anqYP5WrFpi93bmdhF-EdrebEkm59Ds,6806
 ingestify/application/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ingestify/application/dataset_store.py,sha256=c10EIxzOfO4ksKwPOI9jcOn33j54QWu_qXOMLwe-Y-A,11617
-ingestify/application/ingestion_engine.py,sha256=PtMjKMpvfqB802G5zfKLzyamdH7qFOXl3x6_97y8w60,2288
-ingestify/application/loader.py,sha256=v8ZcpMDEml9k_uFPFqT4WaCjXED_OIpAr7g0Pz5Hp6Y,7153
+ingestify/application/dataset_store.py,sha256=JkAb1W0HaUgOwbohKntM4ttyrFXQ7df1uZSu2rbZllY,11680
+ingestify/application/ingestion_engine.py,sha256=4SAmPZDm3e2QA5jZvMrb6xz1eDDshKoSZDWH3TCe4Bo,2372
+ingestify/application/loader.py,sha256=2LpYSHvedsoe5wFsIkQv0xLcKcqtebwVOSPWapAypao,7566
 ingestify/application/secrets_manager.py,sha256=5qCbPfUvRGP1Xbq6xPkMfpgYl8uPzF_0NbiKeRY5zxU,1757
 ingestify/domain/__init__.py,sha256=M7_fVTJjQUx53P4UQUPhowRKPKsIIjx4JYZL1yjHKsM,46
 ingestify/domain/models/__init__.py,sha256=cjQmdSDFA-saXjdF1mLPNWILFHIFgdj20J_fC5FmFsI,770
@@ -20,8 +20,8 @@ ingestify/domain/models/source.py,sha256=sB3aqr2LfjIbtw7ODJpHnPj3RUeo7gYmTU7MXvf
 ingestify/domain/models/timing.py,sha256=TvvH6Szo61CD8wCP7Awyc45CXga5lKqvoW2U-0TRHlA,388
 ingestify/domain/models/dataset/__init__.py,sha256=i1kswluvWjw0xn4OUByRt7yeRvNHu1mauevv-Vmayx4,630
 ingestify/domain/models/dataset/collection.py,sha256=E2utQ6oyaFFrfQFMiwP9J_I7Wm21z0sRvE4Zc3QEs20,1310
-ingestify/domain/models/dataset/collection_metadata.py,sha256=gI5cb9M0QRsheIr2jA71wOyWfI5lGx5ES2Qw7rbDIoA,371
-ingestify/domain/models/dataset/dataset.py,sha256=6iQgBApRK08GhxArnJjjE9SuJMMOsKx_gI6JDHy5nZc,2970
+ingestify/domain/models/dataset/collection_metadata.py,sha256=u2H3XZ-6NMfuAcVD_mb7jEc1IkS1MgQahDIe5CWHjtc,458
+ingestify/domain/models/dataset/dataset.py,sha256=STew8_zCBro_x_u03JrjMvq8yqUDaNndlOlolf9osdM,3332
 ingestify/domain/models/dataset/dataset_repository.py,sha256=kUjiqW58kOUOli1gZCLR5xw4dBX0bqI1UJsf16hgNsQ,812
 ingestify/domain/models/dataset/dataset_state.py,sha256=O95mea5N34HDXw7XsYzxHna4FVk_T-ZNUDezkvt7VzY,220
 ingestify/domain/models/dataset/events.py,sha256=58VacQejQt-WPh9BywP4st5McauM3gXBQo0kaDnSekY,481
@@ -29,7 +29,7 @@ ingestify/domain/models/dataset/file.py,sha256=1Thdv6A1YmC1UfutaRf2q3FGHQYO0SWEp
 ingestify/domain/models/dataset/file_collection.py,sha256=yaQmqFlmbajLCkU5QnjgqCvKzvVEZJrXVvinx5UGHcM,1193
 ingestify/domain/models/dataset/file_repository.py,sha256=9EQprch9isAH2pbK7e7tfOKl6ulip4Ij1kBCTbO_rTc,1721
 ingestify/domain/models/dataset/identifier.py,sha256=EJYsxt0OS_43Y989DZQq8U9NjwmtvnHGYGMe6-hOBlI,575
-ingestify/domain/models/dataset/revision.py,sha256=HPOZpVmQSwdcsr90RNVlOQ7c1_W7grzi5E1NOiEK92g,1331
+ingestify/domain/models/dataset/revision.py,sha256=jBjMqYXDbvt_VAIwL_db09jcH4W8JPRKsXJb4JCniuM,1447
 ingestify/domain/models/dataset/selector.py,sha256=kEGpU8pIyjZ0zwE9n2uo_NY5xrNanWiTTgapyMAUEsw,1039
 ingestify/domain/models/event/__init__.py,sha256=OdPTpE9bj5QqdGmrYqRTLPX1f-LR9GWJYlGMPPEsuL8,138
 ingestify/domain/models/event/_old_event.py,sha256=RktgCAj9SMdtqkAc_bOwoghEb2Z6m4r5_xWXin9wqx4,472
@@ -39,15 +39,15 @@ ingestify/domain/models/event/event_bus.py,sha256=iseourbCwdUg-ODM5bM_u6cageJmce
 ingestify/domain/models/event/publisher.py,sha256=TOAawYYiPQCLR2Gm17LumMEzeapMDYcAYeklLFmwqAY,620
 ingestify/domain/models/event/subscriber.py,sha256=tP1ZFSvpJWKUITnATYekRxJzepz85UY7egBTMiP-dwg,1039
 ingestify/domain/models/ingestion/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ingestify/domain/models/ingestion/ingestion_job.py,sha256=H3vnEUS3izuNJfmD7ZGbznemX9r2JZ1po7D7D9ArzwM,13230
-ingestify/domain/models/ingestion/ingestion_job_summary.py,sha256=cgm8kLoX3eK9SkBYe5HhwA7kg5FAyN4kfTCJrVHaRlc,4702
+ingestify/domain/models/ingestion/ingestion_job.py,sha256=H9jfbbWFZw73nxMOW0480LgSHV-o4sA5IcvpUZmFpS4,13140
+ingestify/domain/models/ingestion/ingestion_job_summary.py,sha256=Xvmtu0BwE9C7FxBl6D8tN49I6--E_RngcMfWeme4DPA,4499
 ingestify/domain/models/ingestion/ingestion_plan.py,sha256=KAvITBMQt3zmMFokESQJyp3rMuz1Hxr6msfZK1_faZM,648
 ingestify/domain/models/resources/__init__.py,sha256=ZuY9DPRfwk-aLB3Lj6DYP_NqMkcQfcYjZp4VejTtcbU,46
 ingestify/domain/models/resources/dataset_resource.py,sha256=NRnN029ct3P_Eg2d9Unb1t7A12Ksv_emBGhoe9DpPwM,3118
 ingestify/domain/models/task/__init__.py,sha256=BdlyIPvE07Xax_IzLgO9DUw0wsz9OZutxnxdDNyRlys,79
 ingestify/domain/models/task/set.py,sha256=04txDYgS5rotXofD9TqChKdW0VZIYshrkfPIpXtlhW4,430
 ingestify/domain/models/task/task.py,sha256=OwLZQi9GGe0O8m1dKvJdN2Rham5oilI49KyKc5uV20A,161
-ingestify/domain/models/task/task_summary.py,sha256=Ncf6ij_aLkElZOsBgep-kd82FyzHjr5xjhAbAXNRJUs,3757
+ingestify/domain/models/task/task_summary.py,sha256=T9BSGhOZjKCPfym34VUdBXuMy0o6E832GAI4WMtjGao,3181
 ingestify/domain/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/domain/services/identifier_key_transformer.py,sha256=y4GS9u9Ej1MO2jUhAxWbifp0mrE_MqTHvVVcoQzSKb4,4034
 ingestify/domain/services/transformers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -55,7 +55,7 @@ ingestify/domain/services/transformers/kloppy_to_pandas.py,sha256=NcN6nTBGVn9gz-
 ingestify/infra/__init__.py,sha256=V0hpLzPVTcOHRVh0gguF6FT30YIgEOUd5v87xUHkfZ4,88
 ingestify/infra/fetch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/infra/fetch/http.py,sha256=ldaXy6alBbI9z63H97lXfYZNT0ZCBkTac1W6-acNjjY,4127
-ingestify/infra/serialization/__init__.py,sha256=-i8XLJDI2hwlX65JITcIzuOaGLJaNekgG9OfA6L7Enc,1035
+ingestify/infra/serialization/__init__.py,sha256=UqXWJmKTp7Mi58ZyDASGguPFlqdVWVUbm_sg9GWx9eI,702
 ingestify/infra/sink/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/infra/sink/postgresql.py,sha256=SxuM3LntfYcpCriUpqJhMvgAf0s9cohXf6WkxSEDYDY,1816
 ingestify/infra/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -64,8 +64,8 @@ ingestify/infra/source/wyscout.py,sha256=DxCzdkzYpVRHTfV9GpF8pe3FzwIk-WHYUlea6nO
 ingestify/infra/store/__init__.py,sha256=3dA6NWfB6FS5SFdQiSlJ0ZghBfnUAUuGIP5Vr4rkCqk,43
 ingestify/infra/store/dataset/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/infra/store/dataset/sqlalchemy/__init__.py,sha256=Z5JHWGO_hwT6rO-ecMOOAmOKjFFJi449KZvJTQgt6vQ,52
-ingestify/infra/store/dataset/sqlalchemy/repository.py,sha256=86BqLhj5pB45iNSfYWbuMNwo-9KnGbbSYtdD8WJw_qo,16003
-ingestify/infra/store/dataset/sqlalchemy/tables.py,sha256=b73jqpW-_QubtZpFJv7BTKdTsKbufESP0O1uJCmFfBE,10106
+ingestify/infra/store/dataset/sqlalchemy/repository.py,sha256=3xDTqEEy_MxZoIX9qezpXasOFW7NMmduJEaR0PwTZXk,16110
+ingestify/infra/store/dataset/sqlalchemy/tables.py,sha256=OLB1FMElb3gSAnOsKX-oiLl_YVXaVEa6Q29QoHp2okU,10602
 ingestify/infra/store/file/__init__.py,sha256=DuEekZa2pmDuRCFiulbgoGotN0wGv3OrRXSvokY0PhY,104
 ingestify/infra/store/file/dummy_file_repository.py,sha256=azUq9c43Mz9-GWk9j0E97BaqyUKu-ZMrcuaIednLq5E,723
 ingestify/infra/store/file/local_file_repository.py,sha256=1hhLqds5LlppJq2QBB0oN0Q98j6aXreCtYQYz3Q1P8g,819
@@ -80,8 +80,8 @@ ingestify/static/templates/wyscout/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRk
 ingestify/static/templates/wyscout/config.yaml.jinja2,sha256=0zQXuvJVwd0oL2OJsPMZ8sOvRbdfRbieSGLQ44ezmYc,379
 ingestify/static/templates/wyscout/query.py,sha256=wjAOMoKvhX-BzCRqEm1SJp6YAcF8Fsq7ddrOaOpAeOk,364
 ingestify/static/templates/wyscout/database/README.md,sha256=7IuzjKo7Pqkx5wkmOETRZDljVOslqfA3ALuHMONq5dg,32
-ingestify-0.3.4.dist-info/METADATA,sha256=v5rEF3343auBHwK8K5Zu0C8tTYfm0WjGtyZs0SmY3xg,18854
-ingestify-0.3.4.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-ingestify-0.3.4.dist-info/entry_points.txt,sha256=czYYXeX2ul4zdeB6bKlz3HaUF7zyVVcj9E_sRNDisI0,53
-ingestify-0.3.4.dist-info/top_level.txt,sha256=Lwnjgns4KequS7KiicXhh6mLUvcdfjzLyPI4qf_s4A0,10
-ingestify-0.3.4.dist-info/RECORD,,
+ingestify-0.4.1.dist-info/METADATA,sha256=Tz062FbilTuQmmW2FPyr2sj0GIK1vjtZs189R5bkxEM,18854
+ingestify-0.4.1.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+ingestify-0.4.1.dist-info/entry_points.txt,sha256=czYYXeX2ul4zdeB6bKlz3HaUF7zyVVcj9E_sRNDisI0,53
+ingestify-0.4.1.dist-info/top_level.txt,sha256=Lwnjgns4KequS7KiicXhh6mLUvcdfjzLyPI4qf_s4A0,10
+ingestify-0.4.1.dist-info/RECORD,,