ingestify 0.1.3__tar.gz → 0.3.0__tar.gz

{ingestify-0.1.3 → ingestify-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.1.3
+Version: 0.3.0
 Summary: Standardizing soccer tracking- and event data
 Author: Koen Vossen
 Author-email: info@koenvossen.nl

{ingestify-0.1.3 → ingestify-0.3.0}/ingestify/__init__.py

@@ -8,4 +8,4 @@ if not __INGESTIFY_SETUP__:
     from .infra import retrieve_http
     from .source_base import Source, DatasetResource
 
-__version__ = "0.1.3"
+__version__ = "0.3.0"

{ingestify-0.1.3 → ingestify-0.3.0}/ingestify/application/dataset_store.py

@@ -5,13 +5,14 @@ import mimetypes
 import os
 import shutil
 from dataclasses import asdict
-from io import BytesIO, StringIO
+from io import BytesIO
 
-from typing import Dict, List, Optional, Union, Callable, BinaryIO
+from typing import Dict, List, Optional, Union, Callable, BinaryIO, Awaitable
 
 from ingestify.domain.models.dataset.dataset import DatasetState
 from ingestify.domain.models.dataset.events import RevisionAdded, MetadataUpdated
 from ingestify.domain.models.dataset.file_collection import FileCollection
+from ingestify.domain.models.dataset.revision import RevisionSource
 from ingestify.domain.models.event import EventBus
 from ingestify.domain.models import (
     Dataset,
@@ -27,7 +28,7 @@ from ingestify.domain.models import (
     Revision,
     DatasetCreated,
 )
-from ingestify.utils import utcnow, map_in_pool
+from ingestify.utils import utcnow
 
 
 logger = logging.getLogger(__name__)
@@ -56,11 +57,16 @@ class DatasetStore:
         if self.event_bus:
             self.event_bus.dispatch(event)
 
+    def save_ingestion_job_summary(self, ingestion_job_summary):
+        self.dataset_repository.session.add(ingestion_job_summary)
+        self.dataset_repository.session.commit()
+
     def get_dataset_collection(
         self,
         dataset_type: Optional[str] = None,
         provider: Optional[str] = None,
         dataset_id: Optional[str] = None,
+        metadata_only: Optional[bool] = False,
         **selector,
     ) -> DatasetCollection:
         if "selector" in selector:
@@ -81,6 +87,7 @@ class DatasetStore:
             dataset_type=dataset_type,
             dataset_id=dataset_id,
             provider=provider,
+            metadata_only=metadata_only,
             selector=selector,
         )
         return dataset_collection
@@ -107,7 +114,9 @@ class DatasetStore:
 
         return stream, storage_size, suffix
 
-    def _prepare_read_stream(self) -> tuple[Callable[[BinaryIO], BytesIO], str]:
+    def _prepare_read_stream(
+        self,
+    ) -> tuple[Callable[[BinaryIO], Awaitable[BytesIO]], str]:
         if self.storage_compression_method == "gzip":
 
             def reader(fh: BinaryIO) -> BytesIO:
@@ -168,7 +177,11 @@ class DatasetStore:
         return modified_files_
 
     def add_revision(
-        self, dataset: Dataset, files: Dict[str, DraftFile], description: str = "Update"
+        self,
+        dataset: Dataset,
+        files: Dict[str, DraftFile],
+        revision_source: RevisionSource,
+        description: str = "Update",
     ):
         """
         Create new revision first, so FileRepository can use
@@ -182,46 +195,53 @@ class DatasetStore:
             # It can happen an API tells us data is changed, but it was not changed. In this case
             # we decide to ignore it.
             # Make sure there are files changed before creating a new revision
-            dataset.add_revision(
-                Revision(
-                    revision_id=revision_id,
-                    created_at=created_at,
-                    description=description,
-                    modified_files=persisted_files_,
-                )
+            revision = Revision(
+                revision_id=revision_id,
+                created_at=created_at,
+                description=description,
+                modified_files=persisted_files_,
+                source=revision_source,
             )
 
+            dataset.add_revision(revision)
+
             self.dataset_repository.save(bucket=self.bucket, dataset=dataset)
             self.dispatch(RevisionAdded(dataset=dataset))
             logger.info(
                 f"Added a new revision to {dataset.identifier} -> {', '.join([file.file_id for file in persisted_files_])}"
             )
-            return True
         else:
             logger.info(
                 f"Ignoring a new revision without changed files -> {dataset.identifier}"
             )
-            return False
+            revision = None
+
+        return revision
 
     def update_dataset(
         self,
         dataset: Dataset,
-        dataset_resource: DatasetResource,
+        name: str,
+        state: DatasetState,
+        metadata: dict,
         files: Dict[str, DraftFile],
+        revision_source: RevisionSource,
     ):
         """The add_revision will also save the dataset."""
         metadata_changed = False
-        if dataset.update_from_resource(dataset_resource):
+        if dataset.update_metadata(name, metadata, state):
             self.dataset_repository.save(bucket=self.bucket, dataset=dataset)
             metadata_changed = True
 
-        self.add_revision(dataset, files)
+        revision = self.add_revision(dataset, files, revision_source)
 
         if metadata_changed:
             # Dispatch after revision added. Otherwise, the downstream handlers are not able to see
             # the new revision
             self.dispatch(MetadataUpdated(dataset=dataset))
 
+        return revision
+
     def destroy_dataset(self, dataset: Dataset):
         # TODO: remove files. Now we leave some orphaned files around
         self.dataset_repository.destroy(dataset)
@@ -235,6 +255,7 @@ class DatasetStore:
         state: DatasetState,
         metadata: dict,
         files: Dict[str, DraftFile],
+        revision_source: RevisionSource,
         description: str = "Create",
     ):
         now = utcnow()
@@ -251,9 +272,10 @@ class DatasetStore:
             created_at=now,
             updated_at=now,
         )
-        self.add_revision(dataset, files, description)
+        revision = self.add_revision(dataset, files, revision_source, description)
 
         self.dispatch(DatasetCreated(dataset=dataset))
+        return revision
 
     def load_files(
         self,
@@ -271,20 +293,9 @@ class DatasetStore:
                 continue
 
             def get_stream(file_):
-                revision_id = file_.revision_id
-                if revision_id is None:
-                    revision_id = current_revision.revision_id
-
                 return reader(
                     self.file_repository.load_content(
-                        bucket=self.bucket,
-                        dataset=dataset,
-                        # When file.revision_id is set we must use it.
-                        revision_id=revision_id,
-                        filename=file_.file_id
-                        + "."
-                        + file_.data_serialization_format
-                        + suffix,
+                        bucket=self.bucket, storage_path=file_.storage_path
                     )
                 )
 
@@ -302,8 +313,8 @@ class DatasetStore:
 
             try:
                 return statsbomb.load(
-                    event_data=files.get_file("events").stream,
-                    lineup_data=files.get_file("lineups").stream,
+                    event_data=(files.get_file("events")).stream,
+                    lineup_data=(files.get_file("lineups")).stream,
                     **kwargs,
                 )
             except Exception as e:
@@ -333,7 +344,7 @@ class DatasetStore:
     #         filename=filename,
     #     )
 
-    def map(
-        self, fn, dataset_collection: DatasetCollection, processes: Optional[int] = None
-    ):
-        return map_in_pool(fn, dataset_collection, processes)
+    # def map(
+    #     self, fn, dataset_collection: DatasetCollection, processes: Optional[int] = None
+    # ):
+    #     return map_in_pool(fn, dataset_collection, processes)

{ingestify-0.1.3 → ingestify-0.3.0}/ingestify/application/ingestion_engine.py

@@ -5,7 +5,7 @@ from typing import Optional, List
 
 from .loader import Loader
 from .dataset_store import DatasetStore
-from ..domain.models.extract_job import ExtractJob
+from ingestify.domain.models.ingestion.ingestion_plan import IngestionPlan
 
 logger = logging.getLogger(__name__)
 
@@ -18,8 +18,8 @@ class IngestionEngine:
         self.store = store
         self.loader = Loader(self.store)
 
-    def add_extract_job(self, extract_job: ExtractJob):
-        self.loader.add_extract_job(extract_job)
+    def add_ingestion_plan(self, ingestion_plan: IngestionPlan):
+        self.loader.add_ingestion_plan(ingestion_plan)
 
     def load(self, dry_run: bool = False, provider: Optional[str] = None):
         self.loader.collect_and_run(dry_run=dry_run, provider=provider)

ingestify-0.3.0/ingestify/application/loader.py

@@ -0,0 +1,165 @@
+import logging
+import platform
+import uuid
+from multiprocessing import set_start_method
+from typing import List, Optional
+
+from ingestify.domain.models import Selector
+from ingestify.utils import TaskExecutor
+
+from .dataset_store import DatasetStore
+from ingestify.domain.models.ingestion.ingestion_plan import IngestionPlan
+from ..domain.models.ingestion.ingestion_job import IngestionJob
+from ..exceptions import ConfigurationError
+
+if platform.system() == "Darwin":
+    set_start_method("fork", force=True)
+else:
+    set_start_method("spawn", force=True)
+
+
+logger = logging.getLogger(__name__)
+
+
+class Loader:
+    def __init__(self, store: DatasetStore):
+        self.store = store
+        self.ingestion_plans: List[IngestionPlan] = []
+
+    def add_ingestion_plan(self, ingestion_plan: IngestionPlan):
+        self.ingestion_plans.append(ingestion_plan)
+
+    def collect_and_run(self, dry_run: bool = False, provider: Optional[str] = None):
+        # First collect all selectors, before discovering datasets
+        selectors = {}
+        for ingestion_plan in self.ingestion_plans:
+            logger.info(f"Determining selectors for {ingestion_plan}")
+
+            if provider is not None:
+                if ingestion_plan.source.provider != provider:
+                    logger.info(
+                        f"Skipping {ingestion_plan} because provider doesn't match '{provider}'"
+                    )
+                    continue
+
+            static_selectors = [
+                selector
+                for selector in ingestion_plan.selectors
+                if not selector.is_dynamic
+            ]
+            dynamic_selectors = [
+                selector for selector in ingestion_plan.selectors if selector.is_dynamic
+            ]
+
+            no_selectors = len(static_selectors) == 1 and not bool(static_selectors[0])
+            if dynamic_selectors or no_selectors:
+                if hasattr(ingestion_plan.source, "discover_selectors"):
+                    logger.debug(
+                        f"Discovering selectors from {ingestion_plan.source.__class__.__name__}"
+                    )
+
+                    # TODO: consider making this lazy and fetch once per Source instead of
+                    #       once per IngestionPlan
+                    all_selectors = ingestion_plan.source.discover_selectors(
+                        ingestion_plan.dataset_type
+                    )
+                    if no_selectors:
+                        # When there were no selectors specified, just use all of them
+                        extra_static_selectors = [
+                            Selector.build(
+                                job_selector,
+                                data_spec_versions=ingestion_plan.data_spec_versions,
+                            )
+                            for job_selector in all_selectors
+                        ]
+                        static_selectors = []
+                    else:
+                        extra_static_selectors = []
+                        for dynamic_selector in dynamic_selectors:
+                            dynamic_job_selectors = [
+                                Selector.build(
+                                    job_selector,
+                                    data_spec_versions=ingestion_plan.data_spec_versions,
+                                )
+                                for job_selector in all_selectors
+                                if dynamic_selector.is_match(job_selector)
+                            ]
+                            extra_static_selectors.extend(dynamic_job_selectors)
+                            logger.info(f"Added {len(dynamic_job_selectors)} selectors")
+
+                    static_selectors.extend(extra_static_selectors)
+
+                    logger.info(
+                        f"Discovered {len(extra_static_selectors)} selectors from {ingestion_plan.source.__class__.__name__}"
+                    )
+                else:
+                    if not no_selectors:
+                        # When there are no selectors and no discover_selectors, just pass it through. It might break
+                        # later on
+                        raise ConfigurationError(
+                            f"Dynamic selectors cannot be used for "
+                            f"{ingestion_plan.source.__class__.__name__} because it doesn't support"
+                            f" selector discovery"
+                        )
+
+            # Merge selectors when source, dataset_type and actual selector is the same. This makes
+            # sure there will be only 1 dataset for this combination
+            for selector in static_selectors:
+                key = (
+                    ingestion_plan.source.name,
+                    ingestion_plan.dataset_type,
+                    selector.key,
+                )
+                if existing_selector := selectors.get(key):
+                    existing_selector[1].data_spec_versions.merge(
+                        selector.data_spec_versions
+                    )
+                else:
+                    selectors[key] = (ingestion_plan, selector)
+
+        """
+            Data is denormalized:
+            
+            It actually looks like:
+                - IngestionPlan #1
+                    - Selector 1.1
+                    - Selector 1.2
+                    - Selector 1.3
+                - IngestionPlan #2
+                    - Selector 2.1
+                    - Selector 2.2
+                    
+            We process this as:
+            - IngestionPlan #1, Selector 1.1
+            - IngestionPlan #1, Selector 1.2
+            - IngestionPlan #1, Selector 1.3
+            - IngestionPlan #2, Selector 2.1
+            - IngestionPlan #2, Selector 2.2 
+            
+            IngestionJobSummary holds the summary for an IngestionPlan and a single Selector
+        """
+        for ingestion_plan, selector in selectors.values():
+            logger.info(
+                f"Discovering datasets from {ingestion_plan.source.__class__.__name__} using selector {selector}"
+            )
+
+            ingestion_job = IngestionJob(
+                ingestion_job_id=str(uuid.uuid1()),
+                ingestion_plan=ingestion_plan,
+                selector=selector,
+            )
+
+            with TaskExecutor(dry_run=dry_run) as task_executor:
+                for ingestion_job_summary in ingestion_job.execute(
+                    self.store, task_executor=task_executor
+                ):
+                    # TODO: handle task_summaries
+                    #       Summarize to a IngestionJobSummary, and save to a database. This Summary can later be used in a
+                    #       next run to determine where to resume.
+                    # TODO 2: Do we want to add additional information from the summary back to the Task, so it can use
+                    #      extra information to determine how/where to resume
+                    ingestion_job_summary.output_report()
+                    logger.info(f"Storing IngestionJobSummary")
+                    self.store.save_ingestion_job_summary(ingestion_job_summary)
+
+        logger.info("Done")

{ingestify-0.1.3 → ingestify-0.3.0}/ingestify/domain/models/__init__.py

@@ -11,10 +11,8 @@ from .dataset import (
     LoadedFile,
     Selector,
     Revision,
-    dataset_repository_factory,
-    file_repository_factory,
 )
-from .sink import Sink, sink_factory
+from .sink import Sink
 from .source import Source
 from .task import Task, TaskSet
 from .data_spec_version_collection import DataSpecVersionCollection
@@ -35,11 +33,8 @@ __all__ = [
     "FileRepository",
     "FileCollection",
     "DatasetRepository",
-    "dataset_repository_factory",
-    "file_repository_factory",
     "TaskSet",
     "Task",
     "Sink",
-    "sink_factory",
     "DataSpecVersionCollection",
 ]

ingestify-0.3.0/ingestify/domain/models/base.py

@@ -0,0 +1,22 @@
+from functools import partial
+from typing import ClassVar, Any, Optional
+
+import pydantic
+from pydantic import BaseModel as PydanticBaseModel, ConfigDict
+
+
+# class BaseModel(PydanticBaseModel):
+#     model_config = ConfigDict(arbitrary_types_allowed=True)
+#
+#     _sa_instance_state: Optional[dict] = None
+from sqlalchemy.orm import MappedAsDataclass
+
+
+class BaseModel(
+    MappedAsDataclass,
+    # DeclarativeBase,
+    dataclass_callable=partial(
+        pydantic.dataclasses.dataclass, config=ConfigDict(arbitrary_types_allowed=True)
+    ),
+):
+    pass

{ingestify-0.1.3 → ingestify-0.3.0}/ingestify/domain/models/data_spec_version_collection.py

@@ -16,6 +16,12 @@ class DataSpecVersionCollection(dict):
 
         return cls(items_)
 
+    def to_dict(self):
+        return {
+            data_feed_key: list(data_spec_versions)
+            for data_feed_key, data_spec_versions in self.items()
+        }
+
     def copy(self):
         return DataSpecVersionCollection(copy.deepcopy(self))
 

{ingestify-0.1.3 → ingestify-0.3.0}/ingestify/domain/models/dataset/__init__.py

@@ -1,8 +1,8 @@
+from .file import DraftFile, File, LoadedFile
 from .collection import DatasetCollection
 from .dataset import Dataset
-from .dataset_repository import DatasetRepository, dataset_repository_factory
-from .file import DraftFile, File, LoadedFile
-from .file_repository import FileRepository, file_repository_factory
+from .dataset_repository import DatasetRepository
+from .file_repository import FileRepository
 from .file_collection import FileCollection
 from .identifier import Identifier
 from .selector import Selector
@@ -16,12 +16,10 @@ __all__ = [
     "Identifier",
     "DatasetCollection",
     "DatasetCreated",
-    "dataset_repository_factory",
     "File",
     "DraftFile",
     "LoadedFile",
     "DatasetRepository",
     "FileRepository",
-    "file_repository_factory",
     "FileCollection",
 ]

{ingestify-0.1.3 → ingestify-0.3.0}/ingestify/domain/models/dataset/dataset.py

@@ -1,70 +1,52 @@
-from dataclasses import dataclass, field
 from datetime import datetime
 from enum import Enum
 from typing import List, Optional
+from pydantic import Field
 
 from ingestify.utils import utcnow
-
+from .dataset_state import DatasetState
 from .file import DraftFile
 from .identifier import Identifier
-from .revision import Revision
-
-
-class DatasetState(Enum):
-    SCHEDULED = "SCHEDULED"
-    PARTIAL = "PARTIAL"
-    COMPLETE = "COMPLETE"
-
-    @property
-    def is_complete(self):
-        return self == DatasetState.COMPLETE
+from .revision import Revision, RevisionSource, SourceType
+from ..base import BaseModel
 
-    def __str__(self):
-        return self.value
 
-
-@dataclass
-class Dataset:
+class Dataset(BaseModel):
     bucket: str  # This must be set by the DatasetRepository
-
     dataset_id: str
     name: str
     state: DatasetState
-
     dataset_type: str
     provider: str
-
     identifier: Identifier
     metadata: dict
-
     created_at: datetime
     updated_at: datetime
-
-    revisions: List[Revision] = field(default_factory=list)
+    revisions: List[Revision] = Field(default_factory=list)
 
     @property
     def is_complete(self):
         return self.state.is_complete
 
-    def next_revision_id(self):
+    def next_revision_id(self) -> int:
         return len(self.revisions)
 
     def add_revision(self, revision: Revision):
         self.revisions.append(revision)
         self.updated_at = utcnow()
 
-    def update_from_resource(self, dataset_resource) -> bool:
+    def update_metadata(self, name: str, metadata: dict, state: DatasetState) -> bool:
         changed = False
-        if self.name != dataset_resource.name:
-            self.name = dataset_resource.name
+        if self.name != name:
+            self.name = name
             changed = True
 
-        if self.metadata != dataset_resource.metadata:
-            self.metadata = dataset_resource.metadata
+        if self.metadata != metadata:
+            self.metadata = metadata
             changed = True
 
-        if self.state != dataset_resource.state:
-            self.state = dataset_resource.state
+        if self.state != state:
+            self.state = state
             changed = True
 
         if changed:
@@ -101,4 +83,5 @@ class Dataset:
                 description="Squashed revision",
                 is_squashed=True,
                 modified_files=list(files.values()),
+                source=RevisionSource(source_type=SourceType.SQUASHED, source_id=""),
             )

{ingestify-0.1.3 → ingestify-0.3.0}/ingestify/domain/models/dataset/dataset_repository.py

@@ -1,16 +1,12 @@
 from abc import ABC, abstractmethod
 from typing import Optional, List, Union
 
-from ingestify.utils import ComponentFactory, ComponentRegistry
-
 from .collection import DatasetCollection
 from .dataset import Dataset
 from .selector import Selector
 
-dataset_repository_registry = ComponentRegistry()
-
 
-class DatasetRepository(ABC, metaclass=dataset_repository_registry.metaclass):
+class DatasetRepository(ABC):
     @abstractmethod
     def get_dataset_collection(
         self,
@@ -34,13 +30,3 @@ class DatasetRepository(ABC, metaclass=dataset_repository_registry.metaclass):
     @abstractmethod
     def next_identity(self):
         pass
-
-    @classmethod
-    @abstractmethod
-    def supports(cls, url: str) -> bool:
-        pass
-
-
-dataset_repository_factory = ComponentFactory.build_factory(
-    DatasetRepository, dataset_repository_registry
-)

ingestify-0.3.0/ingestify/domain/models/dataset/dataset_state.py

@@ -0,0 +1,11 @@
+from enum import Enum
+
+
+class DatasetState(str, Enum):
+    SCHEDULED = "SCHEDULED"
+    PARTIAL = "PARTIAL"
+    COMPLETE = "COMPLETE"
+
+    @property
+    def is_complete(self):
+        return self == DatasetState.COMPLETE

ingestify-0.3.0/ingestify/domain/models/dataset/events.py

@@ -0,0 +1,21 @@
+from typing import ClassVar
+
+from pydantic import BaseModel
+
+from ingestify.domain.models.event.domain_event import DomainEvent
+from .dataset import Dataset
+
+
+class DatasetCreated(DomainEvent):
+    dataset: Dataset
+    event_type: ClassVar[str] = "dataset_created"
+
+
+class RevisionAdded(DomainEvent):
+    dataset: Dataset
+    event_type: ClassVar[str] = "revision_added"
+
+
+class MetadataUpdated(DomainEvent):
+    dataset: Dataset
+    event_type: ClassVar[str] = "metadata_updated"