ingestify 0.3.2__tar.gz → 0.3.4__tar.gz

{ingestify-0.3.2 → ingestify-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.3.2
+Version: 0.3.4
 Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl
@@ -215,23 +215,23 @@ dataset_collection = store.get_dataset_collection(
 store.map(
     lambda dataset: (
         store
-        
-        # As it's related to https://github.com/PySport/kloppy the store can load files using kloppy
-        .load_with_kloppy(dataset)
-        
-        # Convert it into a polars dataframe using all columns in the original data and some more additional ones
-        .to_df(
-            "*", 
-            match_id=dataset.identifier.match_id,
-            competition_id=dataset.identifier.competition_id,
-            season_id=dataset.identifier.season_id, 
-            
+
+            # As it's related to https://github.com/PySport/kloppy the store can load files using kloppy
+            .load_with_kloppy(dataset)
+
+            # Convert it into a polars dataframe using all columns in the original data and some more additional ones
+            .to_df(
+            "*",
+            match_id=dataset.dataset_resource_id.match_id,
+            competition_id=dataset.dataset_resource_id.competition_id,
+            season_id=dataset.dataset_resource_id.season_id,
+
             engine="polars"
         )
-        
-        # Write to parquet format
-        .write_parquet(
-            f"/tmp/files/blaat/{dataset.identifier.match_id}.parquet"
+
+            # Write to parquet format
+            .write_parquet(
+            f"/tmp/files/blaat/{dataset.dataset_resource_id.match_id}.parquet"
         )
     ),
     dataset_collection,

{ingestify-0.3.2 → ingestify-0.3.4}/README.md

@@ -205,23 +205,23 @@ dataset_collection = store.get_dataset_collection(
 store.map(
     lambda dataset: (
         store
-        
-        # As it's related to https://github.com/PySport/kloppy the store can load files using kloppy
-        .load_with_kloppy(dataset)
-        
-        # Convert it into a polars dataframe using all columns in the original data and some more additional ones
-        .to_df(
-            "*", 
-            match_id=dataset.identifier.match_id,
-            competition_id=dataset.identifier.competition_id,
-            season_id=dataset.identifier.season_id, 
-            
+
+            # As it's related to https://github.com/PySport/kloppy the store can load files using kloppy
+            .load_with_kloppy(dataset)
+
+            # Convert it into a polars dataframe using all columns in the original data and some more additional ones
+            .to_df(
+            "*",
+            match_id=dataset.dataset_resource_id.match_id,
+            competition_id=dataset.dataset_resource_id.competition_id,
+            season_id=dataset.dataset_resource_id.season_id,
+
             engine="polars"
         )
-        
-        # Write to parquet format
-        .write_parquet(
-            f"/tmp/files/blaat/{dataset.identifier.match_id}.parquet"
+
+            # Write to parquet format
+            .write_parquet(
+            f"/tmp/files/blaat/{dataset.dataset_resource_id.match_id}.parquet"
         )
     ),
     dataset_collection,

{ingestify-0.3.2 → ingestify-0.3.4}/ingestify/__init__.py

@@ -8,4 +8,4 @@ if not __INGESTIFY_SETUP__:
     from .infra import retrieve_http
     from .source_base import Source, DatasetResource
 
-__version__ = "0.3.2"
+__version__ = "0.3.4"

{ingestify-0.3.2 → ingestify-0.3.4}/ingestify/application/dataset_store.py

@@ -58,8 +58,7 @@ class DatasetStore:
             self.event_bus.dispatch(event)
 
     def save_ingestion_job_summary(self, ingestion_job_summary):
-        self.dataset_repository.session.add(ingestion_job_summary)
-        self.dataset_repository.session.commit()
+        self.dataset_repository.save_ingestion_job_summary(ingestion_job_summary)
 
     def get_dataset_collection(
         self,
@@ -298,8 +297,8 @@ class DatasetStore:
                 )
 
             loaded_file = LoadedFile(
-                _stream=get_stream if lazy else get_stream(file),
-                **asdict(file),
+                stream_=get_stream if lazy else get_stream(file),
+                **file.model_dump(),
             )
             files[file.file_id] = loaded_file
         return FileCollection(files, auto_rewind=auto_rewind)

ingestify-0.3.4/ingestify/domain/models/base.py

@@ -0,0 +1,5 @@
+from pydantic import BaseModel as PydanticBaseModel, ConfigDict
+
+
+class BaseModel(PydanticBaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True, from_attributes=True)

{ingestify-0.3.2 → ingestify-0.3.4}/ingestify/domain/models/dataset/dataset.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 from enum import Enum
 from typing import List, Optional
-from pydantic import Field
+from pydantic import Field, field_validator
 
 from ingestify.utils import utcnow
 from .dataset_state import DatasetState
@@ -24,6 +24,13 @@ class Dataset(BaseModel):
     updated_at: datetime
     revisions: List[Revision] = Field(default_factory=list)
 
+    @field_validator("identifier", mode="before")
+    @classmethod
+    def parse_identifier(cls, value):
+        if not isinstance(value, Identifier):
+            return Identifier(value)
+        return value
+
     @property
     def is_complete(self):
         return self.state.is_complete

{ingestify-0.3.2 → ingestify-0.3.4}/ingestify/domain/models/dataset/file.py

@@ -116,18 +116,18 @@ class LoadedFile(BaseModel):
     data_serialization_format: Optional[str]  # Example: 'json'
     storage_compression_method: Optional[str]  # Example: 'gzip'
     storage_path: Path
-    _stream: Union[BinaryIO, BytesIO, Callable[[], Awaitable[Union[BinaryIO, BytesIO]]]]
+    stream_: Union[BinaryIO, BytesIO, Callable[[], Awaitable[Union[BinaryIO, BytesIO]]]]
     revision_id: Optional[int] = None  # This can be used when a Revision is squashed
 
     def load_stream(self):
-        if callable(self._stream):
-            self._stream = self._stream(self)
+        if callable(self.stream_):
+            self.stream_ = self.stream_(self)
 
     @property
     def stream(self):
-        if callable(self._stream):
+        if callable(self.stream_):
             raise Exception("You should load the stream first using `load_stream`")
-        return self._stream
+        return self.stream_
 
 
 __all__ = ["File", "DraftFile", "LoadedFile"]

{ingestify-0.3.2 → ingestify-0.3.4}/ingestify/domain/models/dataset/revision.py

@@ -1,6 +1,6 @@
 from datetime import datetime
 from enum import Enum
-from typing import Dict, List
+from typing import Dict, List, Optional
 
 from typing_extensions import TypedDict
 
@@ -32,7 +32,7 @@ class Revision(BaseModel):
     created_at: datetime
     description: str
     modified_files: List[File]
-    source: RevisionSource
+    source: Optional[RevisionSource]
     is_squashed: bool = False
     state: RevisionState = RevisionState.PENDING_VALIDATION
 

{ingestify-0.3.2 → ingestify-0.3.4}/ingestify/domain/models/ingestion/ingestion_job.py

@@ -2,6 +2,7 @@ import itertools
 import json
 import logging
 import uuid
+from enum import Enum
 from typing import Optional, Iterator
 
 from ingestify import retrieve_http
@@ -17,6 +18,7 @@ from ingestify.domain.models.resources.dataset_resource import (
     DatasetResource,
 )
 from ingestify.domain.models.task.task_summary import TaskSummary
+from ingestify.exceptions import SaveError
 from ingestify.utils import TaskExecutor, chunker
 
 logger = logging.getLogger(__name__)
@@ -120,21 +122,27 @@ class UpdateDatasetTask(Task):
         with TaskSummary.update(
             self.task_id, dataset_identifier=dataset_identifier
         ) as task_summary:
-            revision = self.store.update_dataset(
-                dataset=self.dataset,
-                name=self.dataset_resource.name,
-                state=self.dataset_resource.state,
-                metadata=self.dataset_resource.metadata,
-                files={
-                    file_id: task_summary.record_load_file(
-                        lambda: load_file(file_resource, dataset=self.dataset),
-                        metadata={"file_id": file_id},
-                    )
-                    for file_id, file_resource in self.dataset_resource.files.items()
-                },
-                revision_source=revision_source,
-            )
-            task_summary.set_stats_from_revision(revision)
+
+            files = {
+                file_id: task_summary.record_load_file(
+                    lambda: load_file(file_resource, dataset=self.dataset),
+                    metadata={"file_id": file_id},
+                )
+                for file_id, file_resource in self.dataset_resource.files.items()
+            }
+
+            try:
+                revision = self.store.update_dataset(
+                    dataset=self.dataset,
+                    name=self.dataset_resource.name,
+                    state=self.dataset_resource.state,
+                    metadata=self.dataset_resource.metadata,
+                    files=files,
+                    revision_source=revision_source,
+                )
+                task_summary.set_stats_from_revision(revision)
+            except Exception as e:
+                raise SaveError("Could not update dataset") from e
 
         return task_summary
 
@@ -159,24 +167,28 @@ class CreateDatasetTask(Task):
         )
 
         with TaskSummary.create(self.task_id, dataset_identifier) as task_summary:
-            revision = self.store.create_dataset(
-                dataset_type=self.dataset_resource.dataset_type,
-                provider=self.dataset_resource.provider,
-                dataset_identifier=dataset_identifier,
-                name=self.dataset_resource.name,
-                state=self.dataset_resource.state,
-                metadata=self.dataset_resource.metadata,
-                files={
-                    file_id: task_summary.record_load_file(
-                        lambda: load_file(file_resource, dataset=None),
-                        metadata={"file_id": file_id},
-                    )
-                    for file_id, file_resource in self.dataset_resource.files.items()
-                },
-                revision_source=revision_source,
-            )
+            files = {
+                file_id: task_summary.record_load_file(
+                    lambda: load_file(file_resource, dataset=None),
+                    metadata={"file_id": file_id},
+                )
+                for file_id, file_resource in self.dataset_resource.files.items()
+            }
+            try:
+                revision = self.store.create_dataset(
+                    dataset_type=self.dataset_resource.dataset_type,
+                    provider=self.dataset_resource.provider,
+                    dataset_identifier=dataset_identifier,
+                    name=self.dataset_resource.name,
+                    state=self.dataset_resource.state,
+                    metadata=self.dataset_resource.metadata,
+                    files=files,
+                    revision_source=revision_source,
+                )
 
-            task_summary.set_stats_from_revision(revision)
+                task_summary.set_stats_from_revision(revision)
+            except Exception as e:
+                raise SaveError("Could not create dataset") from e
 
         return task_summary
 
@@ -202,6 +214,9 @@ class IngestionJob:
         self, store: DatasetStore, task_executor: TaskExecutor
     ) -> Iterator[IngestionJobSummary]:
         is_first_chunk = True
+        ingestion_job_exception = (
+            None  # Indicate if there was an exception during the IngestionJob itself
+        )
         ingestion_job_summary = IngestionJobSummary.new(ingestion_job=self)
         # Process all items in batches. Yield a IngestionJobSummary per batch
 
@@ -219,26 +234,37 @@ class IngestionJob:
         # 1. The discover_datasets returns a list, and the entire list can be processed at once
         # 2. The discover_datasets returns an iterator of batches, in this case we need to process each batch
         with ingestion_job_summary.record_timing("find_datasets"):
-            # Timing might be incorrect as it is an iterator
-            dataset_resources = self.ingestion_plan.source.find_datasets(
-                dataset_type=self.ingestion_plan.dataset_type,
-                data_spec_versions=self.selector.data_spec_versions,
-                dataset_collection_metadata=dataset_collection_metadata,
-                **self.selector.custom_attributes,
-            )
+            try:
+                dataset_resources = self.ingestion_plan.source.find_datasets(
+                    dataset_type=self.ingestion_plan.dataset_type,
+                    data_spec_versions=self.selector.data_spec_versions,
+                    dataset_collection_metadata=dataset_collection_metadata,
+                    **self.selector.custom_attributes,
+                )
 
-        finish_task_timer = ingestion_job_summary.start_timing("tasks")
+                # We need to include the to_batches as that will start the generator
+                batches = to_batches(dataset_resources)
+            except Exception as e:
+                logger.exception("Failed to find datasets")
 
-        batches = to_batches(dataset_resources)
+                ingestion_job_summary.set_exception(e)
+                yield ingestion_job_summary
+                return
+
+        finish_task_timer = ingestion_job_summary.start_timing("tasks")
 
         while True:
             try:
                 batch = next(batches)
             except StopIteration:
                 break
-            except Exception:
-                # TODO: handle exception on IngestionJob level
-                raise
+            except Exception as e:
+                logger.exception("Failed to fetch next batch")
+
+                finish_task_timer()
+                ingestion_job_summary.set_exception(e)
+                yield ingestion_job_summary
+                return
 
             dataset_identifiers = [
                 Identifier.create_from_selector(

{ingestify-0.3.2 → ingestify-0.3.4}/ingestify/domain/models/ingestion/ingestion_job_summary.py

@@ -1,12 +1,13 @@
 import uuid
 from contextlib import contextmanager
 from datetime import datetime, timedelta
+from enum import Enum
 from typing import Optional, List, TYPE_CHECKING
 from pydantic import Field
 
 from ingestify.domain import Selector, DataSpecVersionCollection
 from ingestify.domain.models.base import BaseModel
-from ingestify.domain.models.task.task_summary import TaskSummary, TaskStatus
+from ingestify.domain.models.task.task_summary import TaskSummary, TaskState
 from ingestify.domain.models.timing import Timing
 from ingestify.utils import utcnow
 
@@ -14,6 +15,12 @@ if TYPE_CHECKING:
     from ingestify.domain.models.ingestion.ingestion_job import IngestionJob
 
 
+class IngestionJobState(str, Enum):
+    RUNNING = "RUNNING"
+    FINISHED = "FINISHED"
+    FAILED = "FAILED"
+
+
 def format_duration(duration: timedelta):
     return f"{duration.total_seconds():.2f}sec"
 
@@ -30,7 +37,8 @@ class IngestionJobSummary(BaseModel):
     selector: Selector
 
     started_at: datetime = Field(default_factory=utcnow)
-    finished_at: Optional[datetime] = None
+    ended_at: Optional[datetime] = None
+    state: IngestionJobState = IngestionJobState.RUNNING
     timings: List[Timing] = Field(default_factory=list)
     task_summaries: List[TaskSummary] = Field(default_factory=list)
 
@@ -55,8 +63,10 @@ class IngestionJobSummary(BaseModel):
     @contextmanager
     def record_timing(self, name: str):
         start = utcnow()
-        yield
-        self.timings.append(Timing(name=name, started_at=start, ended_at=utcnow()))
+        try:
+            yield
+        finally:
+            self.timings.append(Timing(name=name, started_at=start, ended_at=utcnow()))
 
     def start_timing(self, name):
         start = utcnow()
@@ -75,28 +85,36 @@ class IngestionJobSummary(BaseModel):
     def task_count(self):
         return len(self.task_summaries)
 
-    def set_finished(self):
+    def _set_ended(self):
         self.failed_tasks = len(
-            [task for task in self.task_summaries if task.status == TaskStatus.FAILED]
+            [task for task in self.task_summaries if task.state == TaskState.FAILED]
         )
         self.successful_tasks = len(
-            [task for task in self.task_summaries if task.status == TaskStatus.FINISHED]
+            [task for task in self.task_summaries if task.state == TaskState.FINISHED]
         )
         self.ignored_successful_tasks = len(
             [
                 task
                 for task in self.task_summaries
-                if task.status == TaskStatus.FINISHED_IGNORED
+                if task.state == TaskState.FINISHED_IGNORED
             ]
         )
-        self.finished_at = utcnow()
+        self.ended_at = utcnow()
+
+    def set_finished(self):
+        self.state = IngestionJobState.FINISHED
+        self._set_ended()
+
+    def set_exception(self, e: Exception):
+        self.state = IngestionJobState.FAILED
+        self._set_ended()
 
     @property
     def duration(self) -> timedelta:
-        return self.finished_at - self.started_at
+        return self.ended_at - self.started_at
 
     def output_report(self):
-        print(f"\nIngestionJobSummary finished in {format_duration(self.duration)}")
+        print(f"\nIngestionJobSummary {self.state} in {format_duration(self.duration)}")
         print("--------------------")
         print(f"  - IngestionPlan:")
         print(f"        Source: {self.source_name}")

{ingestify-0.3.2 → ingestify-0.3.4}/ingestify/domain/models/task/task_summary.py

@@ -16,7 +16,7 @@ from ingestify.utils import utcnow
 logger = logging.getLogger(__name__)
 
 
-class TaskStatus(str, Enum):
+class TaskState(str, Enum):
     RUNNING = "RUNNING"
     FINISHED = "FINISHED"
     FINISHED_IGNORED = "FINISHED_IGNORED"  # Finished, but didn't produce any new data
@@ -37,7 +37,7 @@ class TaskSummary(BaseModel):
     persisted_file_count: int = 0
     bytes_retrieved: int = 0
     last_modified: Optional[datetime] = None
-    status: TaskStatus = TaskStatus.RUNNING
+    state: TaskState = TaskState.RUNNING
     timings: List[Timing] = Field(default_factory=list)
 
     @field_validator("dataset_identifier", mode="before")
@@ -83,10 +83,10 @@ class TaskSummary(BaseModel):
         try:
             yield task_summary
 
-            task_summary.set_status(TaskStatus.FINISHED)
+            task_summary.set_state(TaskState.FINISHED)
         except Exception as e:
             logger.exception(f"Failed to execute task.")
-            task_summary.set_status(TaskStatus.FAILED)
+            task_summary.set_state(TaskState.FAILED)
 
             # When the error comes from our own code, make sure it will be raised to the highest level
             # raise
@@ -111,8 +111,8 @@ class TaskSummary(BaseModel):
                 file.modified_at for file in revision.modified_files
             )
         else:
-            self.status = TaskStatus.FINISHED_IGNORED
+            self.state = TaskState.FINISHED_IGNORED
 
-    def set_status(self, status: TaskStatus):
-        if self.status == TaskStatus.RUNNING:
-            self.status = status
+    def set_state(self, state: TaskState):
+        if self.state == TaskState.RUNNING:
+            self.state = state

{ingestify-0.3.2 → ingestify-0.3.4}/ingestify/exceptions.py

@@ -8,3 +8,7 @@ class ConfigurationError(IngestifyError):
 
 class DuplicateFile(IngestifyError):
     pass
+
+
+class SaveError(IngestifyError):
+    pass

ingestify-0.3.4/ingestify/infra/serialization/__init__.py

@@ -0,0 +1,33 @@
+import json
+from datetime import datetime
+from typing import Type, Any, TypeVar
+
+from dataclass_factory import Schema, Factory, NameStyle
+from dataclass_factory.schema_helpers import type_checker
+
+from ingestify.domain import DatasetCreated, Identifier
+from ingestify.domain.models.dataset.events import MetadataUpdated, RevisionAdded
+from ingestify.domain.models.event import DomainEvent
+
+
+event_types = {
+    DatasetCreated.event_type: DatasetCreated,
+    RevisionAdded.event_type: RevisionAdded,
+}
+
+
+def deserialize(event_dict: dict) -> DomainEvent:
+    event_cls = event_types[event_dict["event_type"]]
+    event_dict["dataset"]["identifier"] = Identifier(
+        **event_dict["dataset"]["identifier"]
+    )
+
+    return event_cls.model_validate(event_dict)
+
+
+def serialize(event: DomainEvent) -> dict:
+    event_dict = event.model_dump(mode="json")
+
+    # Make sure event_type is always part of the event_dict. Pydantic might skip it when the type is ClassVar
+    event_dict["event_type"] = event.event_type
+    return event_dict