ingestify 0.3.3__py3-none-any.whl → 0.3.4__py3-none-any.whl

ingestify/__init__.py

@@ -8,4 +8,4 @@ if not __INGESTIFY_SETUP__:
     from .infra import retrieve_http
     from .source_base import Source, DatasetResource
 
-__version__ = "0.3.3"
+__version__ = "0.3.4"

ingestify/application/dataset_store.py

@@ -58,8 +58,7 @@ class DatasetStore:
             self.event_bus.dispatch(event)
 
     def save_ingestion_job_summary(self, ingestion_job_summary):
-        self.dataset_repository.session.add(ingestion_job_summary)
-        self.dataset_repository.session.commit()
+        self.dataset_repository.save_ingestion_job_summary(ingestion_job_summary)
 
     def get_dataset_collection(
         self,
@@ -298,8 +297,8 @@ class DatasetStore:
                 )
 
             loaded_file = LoadedFile(
-                _stream=get_stream if lazy else get_stream(file),
-                **asdict(file),
+                stream_=get_stream if lazy else get_stream(file),
+                **file.model_dump(),
             )
             files[file.file_id] = loaded_file
         return FileCollection(files, auto_rewind=auto_rewind)

ingestify/domain/models/base.py

@@ -1,22 +1,5 @@
-from functools import partial
-from typing import ClassVar, Any, Optional
-
-import pydantic
 from pydantic import BaseModel as PydanticBaseModel, ConfigDict
 
 
-# class BaseModel(PydanticBaseModel):
-#     model_config = ConfigDict(arbitrary_types_allowed=True)
-#
-#     _sa_instance_state: Optional[dict] = None
-from sqlalchemy.orm import MappedAsDataclass
-
-
-class BaseModel(
-    MappedAsDataclass,
-    # DeclarativeBase,
-    dataclass_callable=partial(
-        pydantic.dataclasses.dataclass, config=ConfigDict(arbitrary_types_allowed=True)
-    ),
-):
-    pass
+class BaseModel(PydanticBaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True, from_attributes=True)

ingestify/domain/models/dataset/dataset.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 from enum import Enum
 from typing import List, Optional
-from pydantic import Field
+from pydantic import Field, field_validator
 
 from ingestify.utils import utcnow
 from .dataset_state import DatasetState
@@ -24,6 +24,13 @@ class Dataset(BaseModel):
     updated_at: datetime
     revisions: List[Revision] = Field(default_factory=list)
 
+    @field_validator("identifier", mode="before")
+    @classmethod
+    def parse_identifier(cls, value):
+        if not isinstance(value, Identifier):
+            return Identifier(value)
+        return value
+
     @property
     def is_complete(self):
         return self.state.is_complete

ingestify/domain/models/dataset/file.py

@@ -116,18 +116,18 @@ class LoadedFile(BaseModel):
     data_serialization_format: Optional[str]  # Example: 'json'
     storage_compression_method: Optional[str]  # Example: 'gzip'
     storage_path: Path
-    _stream: Union[BinaryIO, BytesIO, Callable[[], Awaitable[Union[BinaryIO, BytesIO]]]]
+    stream_: Union[BinaryIO, BytesIO, Callable[[], Awaitable[Union[BinaryIO, BytesIO]]]]
     revision_id: Optional[int] = None  # This can be used when a Revision is squashed
 
     def load_stream(self):
-        if callable(self._stream):
-            self._stream = self._stream(self)
+        if callable(self.stream_):
+            self.stream_ = self.stream_(self)
 
     @property
     def stream(self):
-        if callable(self._stream):
+        if callable(self.stream_):
             raise Exception("You should load the stream first using `load_stream`")
-        return self._stream
+        return self.stream_
 
 
 __all__ = ["File", "DraftFile", "LoadedFile"]

ingestify/domain/models/dataset/revision.py

@@ -1,6 +1,6 @@
 from datetime import datetime
 from enum import Enum
-from typing import Dict, List
+from typing import Dict, List, Optional
 
 from typing_extensions import TypedDict
 
@@ -32,7 +32,7 @@ class Revision(BaseModel):
     created_at: datetime
     description: str
     modified_files: List[File]
-    source: RevisionSource
+    source: Optional[RevisionSource]
     is_squashed: bool = False
     state: RevisionState = RevisionState.PENDING_VALIDATION
 

ingestify/domain/models/ingestion/ingestion_job.py

@@ -2,6 +2,7 @@ import itertools
 import json
 import logging
 import uuid
+from enum import Enum
 from typing import Optional, Iterator
 
 from ingestify import retrieve_http
@@ -17,6 +18,7 @@ from ingestify.domain.models.resources.dataset_resource import (
     DatasetResource,
 )
 from ingestify.domain.models.task.task_summary import TaskSummary
+from ingestify.exceptions import SaveError
 from ingestify.utils import TaskExecutor, chunker
 
 logger = logging.getLogger(__name__)
@@ -120,21 +122,27 @@ class UpdateDatasetTask(Task):
         with TaskSummary.update(
             self.task_id, dataset_identifier=dataset_identifier
         ) as task_summary:
-            revision = self.store.update_dataset(
-                dataset=self.dataset,
-                name=self.dataset_resource.name,
-                state=self.dataset_resource.state,
-                metadata=self.dataset_resource.metadata,
-                files={
-                    file_id: task_summary.record_load_file(
-                        lambda: load_file(file_resource, dataset=self.dataset),
-                        metadata={"file_id": file_id},
-                    )
-                    for file_id, file_resource in self.dataset_resource.files.items()
-                },
-                revision_source=revision_source,
-            )
-            task_summary.set_stats_from_revision(revision)
+
+            files = {
+                file_id: task_summary.record_load_file(
+                    lambda: load_file(file_resource, dataset=self.dataset),
+                    metadata={"file_id": file_id},
+                )
+                for file_id, file_resource in self.dataset_resource.files.items()
+            }
+
+            try:
+                revision = self.store.update_dataset(
+                    dataset=self.dataset,
+                    name=self.dataset_resource.name,
+                    state=self.dataset_resource.state,
+                    metadata=self.dataset_resource.metadata,
+                    files=files,
+                    revision_source=revision_source,
+                )
+                task_summary.set_stats_from_revision(revision)
+            except Exception as e:
+                raise SaveError("Could not update dataset") from e
 
         return task_summary
 
@@ -159,24 +167,28 @@ class CreateDatasetTask(Task):
         )
 
         with TaskSummary.create(self.task_id, dataset_identifier) as task_summary:
-            revision = self.store.create_dataset(
-                dataset_type=self.dataset_resource.dataset_type,
-                provider=self.dataset_resource.provider,
-                dataset_identifier=dataset_identifier,
-                name=self.dataset_resource.name,
-                state=self.dataset_resource.state,
-                metadata=self.dataset_resource.metadata,
-                files={
-                    file_id: task_summary.record_load_file(
-                        lambda: load_file(file_resource, dataset=None),
-                        metadata={"file_id": file_id},
-                    )
-                    for file_id, file_resource in self.dataset_resource.files.items()
-                },
-                revision_source=revision_source,
-            )
+            files = {
+                file_id: task_summary.record_load_file(
+                    lambda: load_file(file_resource, dataset=None),
+                    metadata={"file_id": file_id},
+                )
+                for file_id, file_resource in self.dataset_resource.files.items()
+            }
+            try:
+                revision = self.store.create_dataset(
+                    dataset_type=self.dataset_resource.dataset_type,
+                    provider=self.dataset_resource.provider,
+                    dataset_identifier=dataset_identifier,
+                    name=self.dataset_resource.name,
+                    state=self.dataset_resource.state,
+                    metadata=self.dataset_resource.metadata,
+                    files=files,
+                    revision_source=revision_source,
+                )
 
-            task_summary.set_stats_from_revision(revision)
+                task_summary.set_stats_from_revision(revision)
+            except Exception as e:
+                raise SaveError("Could not create dataset") from e
 
         return task_summary
 
@@ -202,6 +214,9 @@ class IngestionJob:
         self, store: DatasetStore, task_executor: TaskExecutor
     ) -> Iterator[IngestionJobSummary]:
         is_first_chunk = True
+        ingestion_job_exception = (
+            None  # Indicate if there was an exception during the IngestionJob itself
+        )
         ingestion_job_summary = IngestionJobSummary.new(ingestion_job=self)
         # Process all items in batches. Yield a IngestionJobSummary per batch
 
@@ -219,26 +234,37 @@ class IngestionJob:
         # 1. The discover_datasets returns a list, and the entire list can be processed at once
         # 2. The discover_datasets returns an iterator of batches, in this case we need to process each batch
         with ingestion_job_summary.record_timing("find_datasets"):
-            # Timing might be incorrect as it is an iterator
-            dataset_resources = self.ingestion_plan.source.find_datasets(
-                dataset_type=self.ingestion_plan.dataset_type,
-                data_spec_versions=self.selector.data_spec_versions,
-                dataset_collection_metadata=dataset_collection_metadata,
-                **self.selector.custom_attributes,
-            )
+            try:
+                dataset_resources = self.ingestion_plan.source.find_datasets(
+                    dataset_type=self.ingestion_plan.dataset_type,
+                    data_spec_versions=self.selector.data_spec_versions,
+                    dataset_collection_metadata=dataset_collection_metadata,
+                    **self.selector.custom_attributes,
+                )
 
-        finish_task_timer = ingestion_job_summary.start_timing("tasks")
+                # We need to include the to_batches as that will start the generator
+                batches = to_batches(dataset_resources)
+            except Exception as e:
+                logger.exception("Failed to find datasets")
 
-        batches = to_batches(dataset_resources)
+                ingestion_job_summary.set_exception(e)
+                yield ingestion_job_summary
+                return
+
+        finish_task_timer = ingestion_job_summary.start_timing("tasks")
 
         while True:
             try:
                 batch = next(batches)
             except StopIteration:
                 break
-            except Exception:
-                # TODO: handle exception on IngestionJob level
-                raise
+            except Exception as e:
+                logger.exception("Failed to fetch next batch")
+
+                finish_task_timer()
+                ingestion_job_summary.set_exception(e)
+                yield ingestion_job_summary
+                return
 
             dataset_identifiers = [
                 Identifier.create_from_selector(

ingestify/domain/models/ingestion/ingestion_job_summary.py

@@ -1,12 +1,13 @@
 import uuid
 from contextlib import contextmanager
 from datetime import datetime, timedelta
+from enum import Enum
 from typing import Optional, List, TYPE_CHECKING
 from pydantic import Field
 
 from ingestify.domain import Selector, DataSpecVersionCollection
 from ingestify.domain.models.base import BaseModel
-from ingestify.domain.models.task.task_summary import TaskSummary, TaskStatus
+from ingestify.domain.models.task.task_summary import TaskSummary, TaskState
 from ingestify.domain.models.timing import Timing
 from ingestify.utils import utcnow
 
@@ -14,6 +15,12 @@ if TYPE_CHECKING:
     from ingestify.domain.models.ingestion.ingestion_job import IngestionJob
 
 
+class IngestionJobState(str, Enum):
+    RUNNING = "RUNNING"
+    FINISHED = "FINISHED"
+    FAILED = "FAILED"
+
+
 def format_duration(duration: timedelta):
     return f"{duration.total_seconds():.2f}sec"
 
@@ -30,7 +37,8 @@ class IngestionJobSummary(BaseModel):
     selector: Selector
 
     started_at: datetime = Field(default_factory=utcnow)
-    finished_at: Optional[datetime] = None
+    ended_at: Optional[datetime] = None
+    state: IngestionJobState = IngestionJobState.RUNNING
     timings: List[Timing] = Field(default_factory=list)
     task_summaries: List[TaskSummary] = Field(default_factory=list)
 
@@ -55,8 +63,10 @@ class IngestionJobSummary(BaseModel):
     @contextmanager
     def record_timing(self, name: str):
         start = utcnow()
-        yield
-        self.timings.append(Timing(name=name, started_at=start, ended_at=utcnow()))
+        try:
+            yield
+        finally:
+            self.timings.append(Timing(name=name, started_at=start, ended_at=utcnow()))
 
     def start_timing(self, name):
         start = utcnow()
@@ -75,28 +85,36 @@ class IngestionJobSummary(BaseModel):
     def task_count(self):
         return len(self.task_summaries)
 
-    def set_finished(self):
+    def _set_ended(self):
         self.failed_tasks = len(
-            [task for task in self.task_summaries if task.status == TaskStatus.FAILED]
+            [task for task in self.task_summaries if task.state == TaskState.FAILED]
         )
         self.successful_tasks = len(
-            [task for task in self.task_summaries if task.status == TaskStatus.FINISHED]
+            [task for task in self.task_summaries if task.state == TaskState.FINISHED]
         )
         self.ignored_successful_tasks = len(
             [
                 task
                 for task in self.task_summaries
-                if task.status == TaskStatus.FINISHED_IGNORED
+                if task.state == TaskState.FINISHED_IGNORED
             ]
         )
-        self.finished_at = utcnow()
+        self.ended_at = utcnow()
+
+    def set_finished(self):
+        self.state = IngestionJobState.FINISHED
+        self._set_ended()
+
+    def set_exception(self, e: Exception):
+        self.state = IngestionJobState.FAILED
+        self._set_ended()
 
     @property
     def duration(self) -> timedelta:
-        return self.finished_at - self.started_at
+        return self.ended_at - self.started_at
 
     def output_report(self):
-        print(f"\nIngestionJobSummary finished in {format_duration(self.duration)}")
+        print(f"\nIngestionJobSummary {self.state} in {format_duration(self.duration)}")
         print("--------------------")
         print(f"  - IngestionPlan:")
         print(f"        Source: {self.source_name}")

ingestify/domain/models/task/task_summary.py

@@ -16,7 +16,7 @@ from ingestify.utils import utcnow
 logger = logging.getLogger(__name__)
 
 
-class TaskStatus(str, Enum):
+class TaskState(str, Enum):
     RUNNING = "RUNNING"
     FINISHED = "FINISHED"
     FINISHED_IGNORED = "FINISHED_IGNORED"  # Finished, but didn't produce any new data
@@ -37,7 +37,7 @@ class TaskSummary(BaseModel):
     persisted_file_count: int = 0
     bytes_retrieved: int = 0
     last_modified: Optional[datetime] = None
-    status: TaskStatus = TaskStatus.RUNNING
+    state: TaskState = TaskState.RUNNING
     timings: List[Timing] = Field(default_factory=list)
 
     @field_validator("dataset_identifier", mode="before")
@@ -83,10 +83,10 @@ class TaskSummary(BaseModel):
         try:
             yield task_summary
 
-            task_summary.set_status(TaskStatus.FINISHED)
+            task_summary.set_state(TaskState.FINISHED)
         except Exception as e:
             logger.exception(f"Failed to execute task.")
-            task_summary.set_status(TaskStatus.FAILED)
+            task_summary.set_state(TaskState.FAILED)
 
             # When the error comes from our own code, make sure it will be raised to the highest level
             # raise
@@ -111,8 +111,8 @@ class TaskSummary(BaseModel):
                 file.modified_at for file in revision.modified_files
             )
         else:
-            self.status = TaskStatus.FINISHED_IGNORED
+            self.state = TaskState.FINISHED_IGNORED
 
-    def set_status(self, status: TaskStatus):
-        if self.status == TaskStatus.RUNNING:
-            self.status = status
+    def set_state(self, state: TaskState):
+        if self.state == TaskState.RUNNING:
+            self.state = state

ingestify/exceptions.py

@@ -8,3 +8,7 @@ class ConfigurationError(IngestifyError):
 
 class DuplicateFile(IngestifyError):
     pass
+
+
+class SaveError(IngestifyError):
+    pass

ingestify/infra/serialization/__init__.py

@@ -7,44 +7,27 @@ from dataclass_factory.schema_helpers import type_checker
 
 from ingestify.domain import DatasetCreated, Identifier
 from ingestify.domain.models.dataset.events import MetadataUpdated, RevisionAdded
+from ingestify.domain.models.event import DomainEvent
 
-isotime_schema = Schema(
-    parser=lambda x: datetime.fromisoformat(x.replace("Z", "+00:00")),  # type: ignore
-    serializer=lambda x: datetime.isoformat(x).replace("+00:00", "Z"),
-)
-
-identifier_schema = Schema(
-    # json.loads(x) for backwards compatibility
-    parser=lambda x: Identifier(x if isinstance(x, dict) else json.loads(x)),
-    serializer=lambda x: dict(x),
-)
-
-factory = Factory(
-    schemas={
-        datetime: isotime_schema,
-        Identifier: identifier_schema,
-        DatasetCreated: Schema(
-            pre_parse=type_checker(DatasetCreated.event_type, "event_type")
-        ),
-        MetadataUpdated: Schema(
-            pre_parse=type_checker(MetadataUpdated.event_type, "event_type")
-        ),
-        RevisionAdded: Schema(
-            pre_parse=type_checker(RevisionAdded.event_type, "event_type")
-        ),
-        # ClipSelectionContent: Schema(pre_parse=type_checker(ClipSelectionContent.content_type, field="contentType")),
-        # TeamInfoImageContent: Schema(pre_parse=type_checker(TeamInfoImageContent.content_type, field="contentType")),
-        # StaticVideoContent: Schema(pre_parse=type_checker(StaticVideoContent.content_type, field="contentType"))
-    },
-    default_schema=Schema(),
-)
-
-T = TypeVar("T")
-
-
-def serialize(data: T, class_: Type[T] = None) -> Any:
-    return factory.dump(data, class_)
-
-
-def unserialize(data: Any, class_: Type[T]) -> T:
-    return factory.load(data, class_)
+
+event_types = {
+    DatasetCreated.event_type: DatasetCreated,
+    RevisionAdded.event_type: RevisionAdded,
+}
+
+
+def deserialize(event_dict: dict) -> DomainEvent:
+    event_cls = event_types[event_dict["event_type"]]
+    event_dict["dataset"]["identifier"] = Identifier(
+        **event_dict["dataset"]["identifier"]
+    )
+
+    return event_cls.model_validate(event_dict)
+
+
+def serialize(event: DomainEvent) -> dict:
+    event_dict = event.model_dump(mode="json")
+
+    # Make sure event_type is always part of the event_dict. Pydantic might skip it when the type is ClassVar
+    event_dict["event_type"] = event.event_type
+    return event_dict

ingestify/infra/store/dataset/sqlalchemy/repository.py

@@ -1,13 +1,24 @@
+import itertools
 import json
 import uuid
+from collections import defaultdict
 from typing import Optional, Union, List
 
-from sqlalchemy import create_engine, func, text, tuple_
+from sqlalchemy import (
+    create_engine,
+    func,
+    text,
+    tuple_,
+    Table,
+    insert,
+    Transaction,
+    Connection,
+)
 from sqlalchemy.engine import make_url
 from sqlalchemy.exc import NoSuchModuleError
 from sqlalchemy.orm import Session, joinedload
 
-from ingestify.domain import File
+from ingestify.domain import File, Revision
 from ingestify.domain.models import (
     Dataset,
     DatasetCollection,
@@ -15,11 +26,22 @@ from ingestify.domain.models import (
     Identifier,
     Selector,
 )
+from ingestify.domain.models.base import BaseModel
 from ingestify.domain.models.dataset.collection_metadata import (
     DatasetCollectionMetadata,
 )
-
-from .mapping import dataset_table, metadata
+from ingestify.domain.models.ingestion.ingestion_job_summary import IngestionJobSummary
+from ingestify.domain.models.task.task_summary import TaskSummary
+from ingestify.exceptions import IngestifyError
+
+from .tables import (
+    metadata,
+    dataset_table,
+    file_table,
+    revision_table,
+    ingestion_job_summary_table,
+    task_summary_table,
+)
 
 
 def parse_value(v):
@@ -113,6 +135,31 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
     def session(self):
         return self.session_provider.get()
 
+    def _upsert(self, connection: Connection, table: Table, entities: list[dict]):
+        dialect = self.session.bind.dialect.name
+        if dialect == "mysql":
+            from sqlalchemy.dialects.mysql import insert
+        elif dialect == "postgresql":
+            from sqlalchemy.dialects.postgresql import insert
+        elif dialect == "sqlite":
+            from sqlalchemy.dialects.sqlite import insert
+        else:
+            raise IngestifyError(f"Don't know how to do an upsert in {dialect}")
+
+        stmt = insert(table).values(entities)
+
+        primary_key_columns = [column for column in table.columns if column.primary_key]
+
+        set_ = {
+            name: getattr(stmt.excluded, name)
+            for name, column in table.columns.items()
+            if column not in primary_key_columns
+        }
+
+        stmt = stmt.on_conflict_do_update(index_elements=primary_key_columns, set_=set_)
+
+        connection.execute(stmt)
+
     def _filter_query(
         self,
         query,
@@ -122,11 +169,11 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
         dataset_id: Optional[Union[str, List[str]]] = None,
         selector: Optional[Union[Selector, List[Selector]]] = None,
     ):
-        query = query.filter(Dataset.bucket == bucket)
+        query = query.filter(dataset_table.c.bucket == bucket)
         if dataset_type:
-            query = query.filter(Dataset.dataset_type == dataset_type)
+            query = query.filter(dataset_table.c.dataset_type == dataset_type)
         if provider:
-            query = query.filter(Dataset.provider == provider)
+            query = query.filter(dataset_table.c.provider == provider)
         if dataset_id is not None:
             if isinstance(dataset_id, list):
                 if len(dataset_id) == 0:
@@ -134,9 +181,9 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
                     # return an empty DatasetCollection
                     return DatasetCollection()
 
-                query = query.filter(Dataset.dataset_id.in_(dataset_id))
+                query = query.filter(dataset_table.c.dataset_id.in_(dataset_id))
             else:
-                query = query.filter(Dataset.dataset_id == dataset_id)
+                query = query.filter(dataset_table.c.dataset_id == dataset_id)
 
         dialect = self.session.bind.dialect.name
 
@@ -175,7 +222,7 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
                     else:
                         column = column.as_string()
                 else:
-                    column = func.json_extract(Dataset.identifier, f"$.{k}")
+                    column = func.json_extract(dataset_table.c.identifier, f"$.{k}")
                 columns.append(column)
 
             values = []
@@ -189,6 +236,60 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
             query = query.filter(text(where))
         return query
 
+    def load_datasets(self, dataset_ids: list[str]) -> list[Dataset]:
+        if not dataset_ids:
+            return []
+
+        dataset_rows = list(
+            self.session.query(dataset_table).filter(
+                dataset_table.c.dataset_id.in_(dataset_ids)
+            )
+        )
+        revisions_per_dataset = {}
+        rows = (
+            self.session.query(revision_table)
+            .filter(revision_table.c.dataset_id.in_(dataset_ids))
+            .order_by(revision_table.c.dataset_id)
+        )
+
+        for dataset_id, revisions in itertools.groupby(
+            rows, key=lambda row: row.dataset_id
+        ):
+            revisions_per_dataset[dataset_id] = list(revisions)
+
+        files_per_revision = {}
+        rows = (
+            self.session.query(file_table)
+            .filter(file_table.c.dataset_id.in_(dataset_ids))
+            .order_by(file_table.c.dataset_id, file_table.c.revision_id)
+        )
+
+        for (dataset_id, revision_id), files in itertools.groupby(
+            rows, key=lambda row: (row.dataset_id, row.revision_id)
+        ):
+            files_per_revision[(dataset_id, revision_id)] = list(files)
+
+        datasets = []
+        for dataset_row in dataset_rows:
+            dataset_id = dataset_row.dataset_id
+            revisions = []
+            for revision_row in revisions_per_dataset.get(dataset_id, []):
+                files = [
+                    File.model_validate(file_row)
+                    for file_row in files_per_revision.get(
+                        (dataset_id, revision_row.revision_id), []
+                    )
+                ]
+                revision = Revision.model_validate(
+                    {**revision_row._mapping, "modified_files": files}
+                )
+                revisions.append(revision)
+
+            datasets.append(
+                Dataset.model_validate({**dataset_row._mapping, "revisions": revisions})
+            )
+        return datasets
+
     def get_dataset_collection(
         self,
         bucket: str,
@@ -209,17 +310,20 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
             )
 
         if not metadata_only:
-            dataset_query = apply_query_filter(self.session.query(Dataset))
-            datasets = list(dataset_query)
+            dataset_query = apply_query_filter(
+                self.session.query(dataset_table.c.dataset_id)
+            )
+            dataset_ids = [row.dataset_id for row in dataset_query]
+            datasets = self.load_datasets(dataset_ids)
         else:
             datasets = []
 
         metadata_result_row = apply_query_filter(
             self.session.query(
-                func.min(File.modified_at).label("first_modified_at"),
-                func.max(File.modified_at).label("last_modified_at"),
+                func.min(file_table.c.modified_at).label("first_modified_at"),
+                func.max(file_table.c.modified_at).label("last_modified_at"),
                 func.count().label("row_count"),
-            ).join(Dataset, Dataset.dataset_id == File.dataset_id)
+            ).join(dataset_table, dataset_table.c.dataset_id == file_table.c.dataset_id)
         ).first()
         dataset_collection_metadata = DatasetCollectionMetadata(*metadata_result_row)
 
@@ -228,12 +332,153 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
     def save(self, bucket: str, dataset: Dataset):
         # Just make sure
         dataset.bucket = bucket
-        self.session.add(dataset)
-        self.session.commit()
+
+        self._save([dataset])
+
+    def connect(self):
+        return self.session_provider.engine.connect()
+
+    def _save(self, datasets: list[Dataset]):
+        """Only do upserts. Never delete. Rows get only deleted when an entire Dataset is removed."""
+        datasets_entities = []
+        revision_entities = []
+        file_entities = []
+
+        for dataset in datasets:
+            datasets_entities.append(dataset.model_dump(exclude={"revisions"}))
+            for revision in dataset.revisions:
+                revision_entities.append(
+                    {
+                        **revision.model_dump(
+                            exclude={"is_squashed", "modified_files"}
+                        ),
+                        "dataset_id": dataset.dataset_id,
+                    }
+                )
+                for file in revision.modified_files:
+                    file_entities.append(
+                        {
+                            **file.model_dump(),
+                            "dataset_id": dataset.dataset_id,
+                            "revision_id": revision.revision_id,
+                        }
+                    )
+
+        with self.connect() as connection:
+            try:
+                self._upsert(connection, dataset_table, datasets_entities)
+                self._upsert(connection, revision_table, revision_entities)
+                self._upsert(connection, file_table, file_entities)
+            except Exception:
+                connection.rollback()
+                raise
+            else:
+                connection.commit()
 
     def destroy(self, dataset: Dataset):
-        self.session.delete(dataset)
-        self.session.commit()
+        with self.connect() as connection:
+            try:
+                # Delete modified files related to the dataset
+                file_table.delete().where(
+                    file_table.c.dataset_id == dataset.dataset_id
+                ).execute()
+
+                # Delete revisions related to the dataset
+                revision_table.delete().where(
+                    revision_table.c.dataset_id == dataset.dataset_id
+                ).execute()
+
+                # Delete the dataset itself
+                dataset_table.delete().where(
+                    dataset_table.c.dataset_id == dataset.dataset_id
+                ).execute()
+
+                connection.commit()
+            except Exception:
+                connection.rollback()
+                raise
 
     def next_identity(self):
         return str(uuid.uuid4())
+
+    # TODO: consider moving the IngestionJobSummary methods to a different Repository
+    def save_ingestion_job_summary(self, ingestion_job_summary: IngestionJobSummary):
+        ingestion_job_summary_entities = [
+            ingestion_job_summary.model_dump(exclude={"task_summaries"})
+        ]
+        task_summary_entities = []
+        for task_summary in ingestion_job_summary.task_summaries:
+            task_summary_entities.append(
+                {
+                    **task_summary.model_dump(),
+                    "ingestion_job_summary_id": ingestion_job_summary.ingestion_job_summary_id,
+                }
+            )
+
+        with self.session_provider.engine.connect() as connection:
+            try:
+                self._upsert(
+                    connection,
+                    ingestion_job_summary_table,
+                    ingestion_job_summary_entities,
+                )
+                if task_summary_entities:
+                    self._upsert(connection, task_summary_table, task_summary_entities)
+            except Exception:
+                connection.rollback()
+                raise
+            else:
+                connection.commit()
+
+    def load_ingestion_job_summaries(self) -> list[IngestionJobSummary]:
+        ingestion_job_summary_ids = [
+            row.ingestion_job_summary_id
+            for row in self.session.query(
+                ingestion_job_summary_table.c.ingestion_job_summary_id
+            )
+        ]
+
+        ingestion_job_summary_rows = list(
+            self.session.query(ingestion_job_summary_table).filter(
+                ingestion_job_summary_table.c.ingestion_job_summary_id.in_(
+                    ingestion_job_summary_ids
+                )
+            )
+        )
+
+        task_summary_entities_per_job_summary = {}
+        rows = (
+            self.session.query(task_summary_table)
+            .filter(
+                task_summary_table.c.ingestion_job_summary_id.in_(
+                    ingestion_job_summary_ids
+                )
+            )
+            .order_by(task_summary_table.c.ingestion_job_summary_id)
+        )
+
+        for ingestion_job_summary_id, task_summaries_rows in itertools.groupby(
+            rows, key=lambda row: row.ingestion_job_summary_id
+        ):
+            task_summary_entities_per_job_summary[ingestion_job_summary_id] = list(
+                task_summaries_rows
+            )
+
+        ingestion_job_summaries = []
+        for ingestion_job_summary_row in ingestion_job_summary_rows:
+            task_summaries = [
+                TaskSummary.model_validate(row)
+                for row in task_summary_entities_per_job_summary.get(
+                    ingestion_job_summary_row.ingestion_job_summary_id, []
+                )
+            ]
+
+            ingestion_job_summaries.append(
+                IngestionJobSummary.model_validate(
+                    {
+                        **ingestion_job_summary_row._mapping,
+                        "task_summaries": task_summaries,
+                    }
+                )
+            )
+        return ingestion_job_summaries

ingestify/infra/store/dataset/sqlalchemy/{mapping.py → tables.py}

@@ -1,5 +1,4 @@
 import datetime
-from dataclasses import is_dataclass, asdict
 from pathlib import Path
 from typing import Optional
 
@@ -15,17 +14,13 @@ from sqlalchemy import (
     String,
     Table,
     TypeDecorator,
-    Boolean,
 )
-from sqlalchemy.orm import registry, relationship
 
-from ingestify.domain import Selector, Identifier, DataSpecVersionCollection
-from ingestify.domain.models import Dataset, File, Revision
+from ingestify.domain import Identifier, DataSpecVersionCollection, Selector
 from ingestify.domain.models.dataset.dataset import DatasetState
-from ingestify.domain.models.ingestion.ingestion_job_summary import (
-    IngestionJobSummary,
-)
-from ingestify.domain.models.task.task_summary import TaskSummary, Operation, TaskStatus
+from ingestify.domain.models.ingestion.ingestion_job_summary import IngestionJobState
+
+from ingestify.domain.models.task.task_summary import Operation, TaskState
 from ingestify.domain.models.timing import Timing
 from ingestify.domain.models.dataset.revision import RevisionState
 
@@ -106,7 +101,7 @@ class RevisionStateString(TypeDecorator):
 
     def process_result_value(self, value, dialect):
         if not value:
-            return value
+            return RevisionState.PENDING_VALIDATION
 
         return RevisionState[value]
 
@@ -124,20 +119,31 @@ class OperationString(TypeDecorator):
         return Operation[value]
 
 
-class TaskStatusString(TypeDecorator):
+class TaskStateString(TypeDecorator):
     impl = String(255)
 
-    def process_bind_param(self, value: TaskStatus, dialect):
+    def process_bind_param(self, value: TaskState, dialect):
         return value.value
 
     def process_result_value(self, value, dialect):
         if not value:
             return value
 
-        return TaskStatus[value]
+        return TaskState[value]
+
 
+class IngestionJobStateString(TypeDecorator):
+    impl = String(255)
+
+    def process_bind_param(self, value: IngestionJobState, dialect):
+        return value.value
+
+    def process_result_value(self, value, dialect):
+        if not value:
+            return value
+
+        return IngestionJobState[value]
 
-mapper_registry = registry()
 
 metadata = MetaData()
 
@@ -193,40 +199,7 @@ file_table = Table(
     ),
 )
 
-
-mapper_registry.map_imperatively(
-    Dataset,
-    dataset_table,
-    properties={
-        "revisions": relationship(
-            Revision,
-            backref="dataset",
-            order_by=revision_table.c.revision_id,
-            lazy="selectin",
-            cascade="all, delete-orphan",
-        ),
-    },
-)
-
-mapper_registry.map_imperatively(
-    Revision,
-    revision_table,
-    properties={
-        "modified_files": relationship(
-            File,
-            order_by=file_table.c.file_id,
-            primaryjoin="and_(Revision.revision_id==File.revision_id, Revision.dataset_id==File.dataset_id)",
-            lazy="selectin",
-            cascade="all, delete-orphan",
-        )
-    },
-)
-
-
-mapper_registry.map_imperatively(File, file_table)
-
-
-ingestion_job_summary = Table(
+ingestion_job_summary_table = Table(
     "ingestion_job_summary",
     metadata,
     Column("ingestion_job_summary_id", String(255), primary_key=True),
@@ -238,18 +211,25 @@ ingestion_job_summary = Table(
     Column(
         "data_spec_versions",
         JSONType(
-            serializer=lambda data_spec_versions: data_spec_versions.to_dict(),
+            serializer=lambda data_spec_versions: {
+                key: list(value) for key, value in data_spec_versions.items()
+            },
             deserializer=lambda data_spec_versions: DataSpecVersionCollection.from_dict(
                 data_spec_versions
             ),
         ),
     ),
     Column(
-        "selector", JSONType(serializer=lambda selector: selector.filtered_attributes)
+        "selector",
+        JSONType(
+            serializer=lambda selector: selector.filtered_attributes,
+            deserializer=lambda selector: Selector(**selector),
+        ),
     ),
     Column("started_at", TZDateTime(6)),
-    Column("finished_at", TZDateTime(6)),
+    Column("ended_at", TZDateTime(6)),
     # Some task counters
+    Column("state", IngestionJobStateString),
     Column("successful_tasks", Integer),
     Column("ignored_successful_tasks", Integer),
     Column("skipped_datasets", Integer),
@@ -258,7 +238,10 @@ ingestion_job_summary = Table(
         "timings",
         JSONType(
             serializer=lambda timings: [
-                timing.model_dump(mode="json") for timing in timings
+                # Timing is probably already a dictionary. Load it into Timing first, so it can be dumped
+                # in json mode
+                Timing.model_validate(timing).model_dump(mode="json")
+                for timing in timings
             ],
             deserializer=lambda timings: [
                 Timing.model_validate(timing) for timing in timings
@@ -299,12 +282,13 @@ task_summary_table = Table(
     Column("persisted_file_count", Integer),
     Column("bytes_retrieved", Integer),
     Column("last_modified", TZDateTime(6)),
-    Column("status", TaskStatusString),
+    Column("state", TaskStateString),
     Column(
         "timings",
         JSONType(
             serializer=lambda timings: [
-                timing.model_dump(mode="json") for timing in timings
+                Timing.model_validate(timing).model_dump(mode="json")
+                for timing in timings
             ],
             deserializer=lambda timings: [
                 Timing.model_validate(timing) for timing in timings
@@ -316,21 +300,54 @@ task_summary_table = Table(
     # Column("state", RevisionStateString, default=RevisionState.PENDING_VALIDATION),
     # Column("source", JSONType()),
 )
-
-
-mapper_registry.map_imperatively(
-    IngestionJobSummary,
-    ingestion_job_summary,
-    properties={
-        "task_summaries": relationship(
-            TaskSummary,
-            backref="ingestion_job_summary",
-            # order_by=task_summary_table.c.revision_id,
-            lazy="selectin",
-            cascade="all, delete-orphan",
-        ),
-    },
-)
-
-
-mapper_registry.map_imperatively(TaskSummary, task_summary_table)
+#
+#
+# mapper_registry = registry()
+#
+# mapper_registry.map_imperatively(
+#     Dataset,
+#     dataset_table,
+#     properties={
+#         "revisions": relationship(
+#             Revision,
+#             backref="dataset",
+#             order_by=revision_table.c.revision_id,
+#             lazy="selectin",
+#             cascade="all, delete-orphan",
+#         ),
+#     },
+# )
+#
+# mapper_registry.map_imperatively(
+#     Revision,
+#     revision_table,
+#     properties={
+#         "modified_files": relationship(
+#             File,
+#             order_by=file_table.c.file_id,
+#             primaryjoin="and_(Revision.revision_id==File.revision_id, Revision.dataset_id==File.dataset_id)",
+#             lazy="selectin",
+#             cascade="all, delete-orphan",
+#         )
+#     },
+# )
+#
+#
+# mapper_registry.map_imperatively(File, file_table)
+#
+# mapper_registry.map_imperatively(
+#     IngestionJobSummary,
+#     ingestion_job_summary,
+#     properties={
+#         "task_summaries": relationship(
+#             TaskSummary,
+#             backref="ingestion_job_summary",
+#             # order_by=task_summary_table.c.revision_id,
+#             lazy="selectin",
+#             cascade="all, delete-orphan",
+#         ),
+#     },
+# )
+#
+#
+# mapper_registry.map_imperatively(TaskSummary, task_summary_table)

{ingestify-0.3.3.dist-info → ingestify-0.3.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.3.3
+Version: 0.3.4
 Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl
@@ -227,23 +227,23 @@ dataset_collection = store.get_dataset_collection(
 store.map(
     lambda dataset: (
         store
-        
-        # As it's related to https://github.com/PySport/kloppy the store can load files using kloppy
-        .load_with_kloppy(dataset)
-        
-        # Convert it into a polars dataframe using all columns in the original data and some more additional ones
-        .to_df(
-            "*", 
-            match_id=dataset.identifier.match_id,
-            competition_id=dataset.identifier.competition_id,
-            season_id=dataset.identifier.season_id, 
-            
+
+            # As it's related to https://github.com/PySport/kloppy the store can load files using kloppy
+            .load_with_kloppy(dataset)
+
+            # Convert it into a polars dataframe using all columns in the original data and some more additional ones
+            .to_df(
+            "*",
+            match_id=dataset.dataset_resource_id.match_id,
+            competition_id=dataset.dataset_resource_id.competition_id,
+            season_id=dataset.dataset_resource_id.season_id,
+
             engine="polars"
         )
-        
-        # Write to parquet format
-        .write_parquet(
-            f"/tmp/files/blaat/{dataset.identifier.match_id}.parquet"
+
+            # Write to parquet format
+            .write_parquet(
+            f"/tmp/files/blaat/{dataset.dataset_resource_id.match_id}.parquet"
         )
     ),
     dataset_collection,

{ingestify-0.3.3.dist-info → ingestify-0.3.4.dist-info}/RECORD

@@ -1,18 +1,18 @@
-ingestify/__init__.py,sha256=skDa1VfOP7IslAz1tXtfTAwPzohhFlzwGkD_1wV8m50,301
+ingestify/__init__.py,sha256=lyBZ_P8y4qlkE1e11F4T41fSTp8WbReifRxX9UGizxA,301
 ingestify/cmdline.py,sha256=bIuyPgGEw4wIglNzpG9zp7TsJozsP8NSVsCe4eAyWUg,7189
-ingestify/exceptions.py,sha256=wMMuajl4AkQRfW60TLN7btJmQaH8-lUczXyW_2g9kOU,143
+ingestify/exceptions.py,sha256=izRzaLQmMy-4P8ZqGqVZyf4k6LFYOYqwYLuRaUH8BJw,187
 ingestify/main.py,sha256=Xr0VbGgstPO7doDX18xqk4lBb4W2sbGWtQuXZaARsHA,8763
 ingestify/server.py,sha256=OVrf_XtpAQIn88MzqQzShXgsA9_jbnqYvD8YPBjn3cs,2413
 ingestify/source_base.py,sha256=GXAFCoT11Zov9M2v-fqQr9gFCXbtVfEIEH32V7r2oE8,382
 ingestify/utils.py,sha256=HETGhAoUlutLG0cQR63nac2JbFei9gnktDHeBQoYWfU,5692
 ingestify/application/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ingestify/application/dataset_store.py,sha256=5CZ2v_fjqhNyC8JdjE9O4huTwy3WtHhawyc8Gw4DeJ4,11646
+ingestify/application/dataset_store.py,sha256=c10EIxzOfO4ksKwPOI9jcOn33j54QWu_qXOMLwe-Y-A,11617
 ingestify/application/ingestion_engine.py,sha256=PtMjKMpvfqB802G5zfKLzyamdH7qFOXl3x6_97y8w60,2288
 ingestify/application/loader.py,sha256=v8ZcpMDEml9k_uFPFqT4WaCjXED_OIpAr7g0Pz5Hp6Y,7153
 ingestify/application/secrets_manager.py,sha256=5qCbPfUvRGP1Xbq6xPkMfpgYl8uPzF_0NbiKeRY5zxU,1757
 ingestify/domain/__init__.py,sha256=M7_fVTJjQUx53P4UQUPhowRKPKsIIjx4JYZL1yjHKsM,46
 ingestify/domain/models/__init__.py,sha256=cjQmdSDFA-saXjdF1mLPNWILFHIFgdj20J_fC5FmFsI,770
-ingestify/domain/models/base.py,sha256=6jzzIqSkH1mPsXZ2OTXMj09S_IlvMOrOBHBJyWAKEjE,555
+ingestify/domain/models/base.py,sha256=4gKbREajxJHlS-VwKoosNtHVupZ4eDLKMqnJ4ib0aS8,184
 ingestify/domain/models/data_spec_version_collection.py,sha256=CAXlO4W2AOOWAPdPAuymqBHnJpiYtkr2z7fYFJ3HSCk,1372
 ingestify/domain/models/fetch_policy.py,sha256=d7K1TzliNJXxqaqzqEOQWLhvgIvmmqhUQEliXvSUcTs,1405
 ingestify/domain/models/sink.py,sha256=OBVfFMpB7puJmHg4q2KYx4qgoAnlmX8xKWYnPi8a9pc,178
@@ -21,15 +21,15 @@ ingestify/domain/models/timing.py,sha256=TvvH6Szo61CD8wCP7Awyc45CXga5lKqvoW2U-0T
 ingestify/domain/models/dataset/__init__.py,sha256=i1kswluvWjw0xn4OUByRt7yeRvNHu1mauevv-Vmayx4,630
 ingestify/domain/models/dataset/collection.py,sha256=E2utQ6oyaFFrfQFMiwP9J_I7Wm21z0sRvE4Zc3QEs20,1310
 ingestify/domain/models/dataset/collection_metadata.py,sha256=gI5cb9M0QRsheIr2jA71wOyWfI5lGx5ES2Qw7rbDIoA,371
-ingestify/domain/models/dataset/dataset.py,sha256=ReL50BXNaJVU29OB5_9CQEI7BekWsgi1t3AR7e5jENc,2743
+ingestify/domain/models/dataset/dataset.py,sha256=6iQgBApRK08GhxArnJjjE9SuJMMOsKx_gI6JDHy5nZc,2970
 ingestify/domain/models/dataset/dataset_repository.py,sha256=kUjiqW58kOUOli1gZCLR5xw4dBX0bqI1UJsf16hgNsQ,812
 ingestify/domain/models/dataset/dataset_state.py,sha256=O95mea5N34HDXw7XsYzxHna4FVk_T-ZNUDezkvt7VzY,220
 ingestify/domain/models/dataset/events.py,sha256=58VacQejQt-WPh9BywP4st5McauM3gXBQo0kaDnSekY,481
-ingestify/domain/models/dataset/file.py,sha256=1oj03zKdkO_9F85LuDcihbB0Kr3suf12KZNGHpVo3w0,4144
+ingestify/domain/models/dataset/file.py,sha256=1Thdv6A1YmC1UfutaRf2q3FGHQYO0SWEptCxur6Ahfs,4144
 ingestify/domain/models/dataset/file_collection.py,sha256=yaQmqFlmbajLCkU5QnjgqCvKzvVEZJrXVvinx5UGHcM,1193
 ingestify/domain/models/dataset/file_repository.py,sha256=9EQprch9isAH2pbK7e7tfOKl6ulip4Ij1kBCTbO_rTc,1721
 ingestify/domain/models/dataset/identifier.py,sha256=EJYsxt0OS_43Y989DZQq8U9NjwmtvnHGYGMe6-hOBlI,575
-ingestify/domain/models/dataset/revision.py,sha256=O_1HG2S2EmYdWqI2K282S_D-d6IhRh_f4Q3wV8MEhkk,1311
+ingestify/domain/models/dataset/revision.py,sha256=HPOZpVmQSwdcsr90RNVlOQ7c1_W7grzi5E1NOiEK92g,1331
 ingestify/domain/models/dataset/selector.py,sha256=kEGpU8pIyjZ0zwE9n2uo_NY5xrNanWiTTgapyMAUEsw,1039
 ingestify/domain/models/event/__init__.py,sha256=OdPTpE9bj5QqdGmrYqRTLPX1f-LR9GWJYlGMPPEsuL8,138
 ingestify/domain/models/event/_old_event.py,sha256=RktgCAj9SMdtqkAc_bOwoghEb2Z6m4r5_xWXin9wqx4,472
@@ -39,15 +39,15 @@ ingestify/domain/models/event/event_bus.py,sha256=iseourbCwdUg-ODM5bM_u6cageJmce
 ingestify/domain/models/event/publisher.py,sha256=TOAawYYiPQCLR2Gm17LumMEzeapMDYcAYeklLFmwqAY,620
 ingestify/domain/models/event/subscriber.py,sha256=tP1ZFSvpJWKUITnATYekRxJzepz85UY7egBTMiP-dwg,1039
 ingestify/domain/models/ingestion/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ingestify/domain/models/ingestion/ingestion_job.py,sha256=U6B62c7NGeHBAjmKhgOa4uHeul34xyR66WtWaPSRNTU,12276
-ingestify/domain/models/ingestion/ingestion_job_summary.py,sha256=e8wULXsKAGNGrOV4dyiRcFlAfYCWcLa2iqJKNMwirlk,4270
+ingestify/domain/models/ingestion/ingestion_job.py,sha256=H3vnEUS3izuNJfmD7ZGbznemX9r2JZ1po7D7D9ArzwM,13230
+ingestify/domain/models/ingestion/ingestion_job_summary.py,sha256=cgm8kLoX3eK9SkBYe5HhwA7kg5FAyN4kfTCJrVHaRlc,4702
 ingestify/domain/models/ingestion/ingestion_plan.py,sha256=KAvITBMQt3zmMFokESQJyp3rMuz1Hxr6msfZK1_faZM,648
 ingestify/domain/models/resources/__init__.py,sha256=ZuY9DPRfwk-aLB3Lj6DYP_NqMkcQfcYjZp4VejTtcbU,46
 ingestify/domain/models/resources/dataset_resource.py,sha256=NRnN029ct3P_Eg2d9Unb1t7A12Ksv_emBGhoe9DpPwM,3118
 ingestify/domain/models/task/__init__.py,sha256=BdlyIPvE07Xax_IzLgO9DUw0wsz9OZutxnxdDNyRlys,79
 ingestify/domain/models/task/set.py,sha256=04txDYgS5rotXofD9TqChKdW0VZIYshrkfPIpXtlhW4,430
 ingestify/domain/models/task/task.py,sha256=OwLZQi9GGe0O8m1dKvJdN2Rham5oilI49KyKc5uV20A,161
-ingestify/domain/models/task/task_summary.py,sha256=ovzqKPstngRVzVA_JboQMluq5uQjKVJDsWNNcfcadhU,3774
+ingestify/domain/models/task/task_summary.py,sha256=Ncf6ij_aLkElZOsBgep-kd82FyzHjr5xjhAbAXNRJUs,3757
 ingestify/domain/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/domain/services/identifier_key_transformer.py,sha256=y4GS9u9Ej1MO2jUhAxWbifp0mrE_MqTHvVVcoQzSKb4,4034
 ingestify/domain/services/transformers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -55,7 +55,7 @@ ingestify/domain/services/transformers/kloppy_to_pandas.py,sha256=NcN6nTBGVn9gz-
 ingestify/infra/__init__.py,sha256=V0hpLzPVTcOHRVh0gguF6FT30YIgEOUd5v87xUHkfZ4,88
 ingestify/infra/fetch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/infra/fetch/http.py,sha256=ldaXy6alBbI9z63H97lXfYZNT0ZCBkTac1W6-acNjjY,4127
-ingestify/infra/serialization/__init__.py,sha256=LwfmRoO4qykZkJZXxVPSKpwoVIkg9qzXa7Egut9JjL4,1772
+ingestify/infra/serialization/__init__.py,sha256=-i8XLJDI2hwlX65JITcIzuOaGLJaNekgG9OfA6L7Enc,1035
 ingestify/infra/sink/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/infra/sink/postgresql.py,sha256=SxuM3LntfYcpCriUpqJhMvgAf0s9cohXf6WkxSEDYDY,1816
 ingestify/infra/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -64,8 +64,8 @@ ingestify/infra/source/wyscout.py,sha256=DxCzdkzYpVRHTfV9GpF8pe3FzwIk-WHYUlea6nO
 ingestify/infra/store/__init__.py,sha256=3dA6NWfB6FS5SFdQiSlJ0ZghBfnUAUuGIP5Vr4rkCqk,43
 ingestify/infra/store/dataset/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/infra/store/dataset/sqlalchemy/__init__.py,sha256=Z5JHWGO_hwT6rO-ecMOOAmOKjFFJi449KZvJTQgt6vQ,52
-ingestify/infra/store/dataset/sqlalchemy/mapping.py,sha256=pyqxy7LAyRK2Mt6knaKYozXN07oNPYztU9x4DGDQD0U,9451
-ingestify/infra/store/dataset/sqlalchemy/repository.py,sha256=-eSR_F9tS9Hd3JNEpoJoDAb5RY38rFaKLMI3eBedjx8,7068
+ingestify/infra/store/dataset/sqlalchemy/repository.py,sha256=86BqLhj5pB45iNSfYWbuMNwo-9KnGbbSYtdD8WJw_qo,16003
+ingestify/infra/store/dataset/sqlalchemy/tables.py,sha256=b73jqpW-_QubtZpFJv7BTKdTsKbufESP0O1uJCmFfBE,10106
 ingestify/infra/store/file/__init__.py,sha256=DuEekZa2pmDuRCFiulbgoGotN0wGv3OrRXSvokY0PhY,104
 ingestify/infra/store/file/dummy_file_repository.py,sha256=azUq9c43Mz9-GWk9j0E97BaqyUKu-ZMrcuaIednLq5E,723
 ingestify/infra/store/file/local_file_repository.py,sha256=1hhLqds5LlppJq2QBB0oN0Q98j6aXreCtYQYz3Q1P8g,819
@@ -80,8 +80,8 @@ ingestify/static/templates/wyscout/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRk
 ingestify/static/templates/wyscout/config.yaml.jinja2,sha256=0zQXuvJVwd0oL2OJsPMZ8sOvRbdfRbieSGLQ44ezmYc,379
 ingestify/static/templates/wyscout/query.py,sha256=wjAOMoKvhX-BzCRqEm1SJp6YAcF8Fsq7ddrOaOpAeOk,364
 ingestify/static/templates/wyscout/database/README.md,sha256=7IuzjKo7Pqkx5wkmOETRZDljVOslqfA3ALuHMONq5dg,32
-ingestify-0.3.3.dist-info/METADATA,sha256=ln_MGBlqm4wgPBldNv_VofJ4snw981jv667X4JOylmY,18832
-ingestify-0.3.3.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-ingestify-0.3.3.dist-info/entry_points.txt,sha256=czYYXeX2ul4zdeB6bKlz3HaUF7zyVVcj9E_sRNDisI0,53
-ingestify-0.3.3.dist-info/top_level.txt,sha256=Lwnjgns4KequS7KiicXhh6mLUvcdfjzLyPI4qf_s4A0,10
-ingestify-0.3.3.dist-info/RECORD,,
+ingestify-0.3.4.dist-info/METADATA,sha256=v5rEF3343auBHwK8K5Zu0C8tTYfm0WjGtyZs0SmY3xg,18854
+ingestify-0.3.4.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+ingestify-0.3.4.dist-info/entry_points.txt,sha256=czYYXeX2ul4zdeB6bKlz3HaUF7zyVVcj9E_sRNDisI0,53
+ingestify-0.3.4.dist-info/top_level.txt,sha256=Lwnjgns4KequS7KiicXhh6mLUvcdfjzLyPI4qf_s4A0,10
+ingestify-0.3.4.dist-info/RECORD,,