ingestify 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

ingestify/__init__.py

@@ -8,4 +8,4 @@ if not __INGESTIFY_SETUP__:
     from .infra import retrieve_http
     from .source_base import Source, DatasetResource
 
-__version__ = "0.2.0"
+__version__ = "0.3.0"

ingestify/application/dataset_store.py

@@ -66,6 +66,7 @@ class DatasetStore:
         dataset_type: Optional[str] = None,
         provider: Optional[str] = None,
         dataset_id: Optional[str] = None,
+        metadata_only: Optional[bool] = False,
         **selector,
     ) -> DatasetCollection:
         if "selector" in selector:
@@ -86,6 +87,7 @@ class DatasetStore:
             dataset_type=dataset_type,
             dataset_id=dataset_id,
             provider=provider,
+            metadata_only=metadata_only,
             selector=selector,
         )
         return dataset_collection
@@ -291,20 +293,9 @@ class DatasetStore:
                 continue
 
             def get_stream(file_):
-                revision_id = file_.revision_id
-                if revision_id is None:
-                    revision_id = current_revision.revision_id
-
                 return reader(
                     self.file_repository.load_content(
-                        bucket=self.bucket,
-                        dataset=dataset,
-                        # When file.revision_id is set we must use it.
-                        revision_id=revision_id,
-                        filename=file_.file_id
-                        + "."
-                        + file_.data_serialization_format
-                        + suffix,
+                        bucket=self.bucket, storage_path=file_.storage_path
                     )
                 )
 

ingestify/application/loader.py

@@ -33,6 +33,8 @@ class Loader:
         # First collect all selectors, before discovering datasets
         selectors = {}
         for ingestion_plan in self.ingestion_plans:
+            logger.info(f"Determining selectors for {ingestion_plan}")
+
             if provider is not None:
                 if ingestion_plan.source.provider != provider:
                     logger.info(
@@ -137,7 +139,7 @@ class Loader:
             IngestionJobSummary holds the summary for an IngestionPlan and a single Selector
         """
         for ingestion_plan, selector in selectors.values():
-            logger.debug(
+            logger.info(
                 f"Discovering datasets from {ingestion_plan.source.__class__.__name__} using selector {selector}"
             )
 
@@ -148,18 +150,16 @@ class Loader:
             )
 
             with TaskExecutor(dry_run=dry_run) as task_executor:
-                ingestion_job_summary = ingestion_job.execute(
+                for ingestion_job_summary in ingestion_job.execute(
                     self.store, task_executor=task_executor
-                )
-
-                # TODO: handle task_summaries
-                #       Summarize to a IngestionJobSummary, and save to a database. This Summary can later be used in a
-                #       next run to determine where to resume.
-                # TODO 2: Do we want to add additional information from the summary back to the Task, so it can use
-                #      extra information to determine how/where to resume
-                ingestion_job_summary.set_finished()
-
-            ingestion_job_summary.output_report()
-            self.store.save_ingestion_job_summary(ingestion_job_summary)
+                ):
+                    # TODO: handle task_summaries
+                    #       Summarize to a IngestionJobSummary, and save to a database. This Summary can later be used in a
+                    #       next run to determine where to resume.
+                    # TODO 2: Do we want to add additional information from the summary back to the Task, so it can use
+                    #      extra information to determine how/where to resume
+                    ingestion_job_summary.output_report()
+                    logger.info(f"Storing IngestionJobSummary")
+                    self.store.save_ingestion_job_summary(ingestion_job_summary)
 
         logger.info("Done")

ingestify/domain/models/dataset/file_repository.py

@@ -3,11 +3,34 @@ from pathlib import Path
 from typing import BinaryIO
 
 from .dataset import Dataset
+from ...services.identifier_key_transformer import IdentifierTransformer
 
 
 class FileRepository(ABC):
-    def __init__(self, url: str):
+    def __init__(self, url: str, identifier_transformer: IdentifierTransformer):
         self.base_dir = Path(url.split("://")[1])
+        self.identifier_transformer = identifier_transformer
+
+    def get_write_path(
+        self, bucket: str, dataset: Dataset, revision_id: int, filename: str
+    ) -> Path:
+        # TODO: use the IdentifierKeyTransformer
+        identifier_path = self.identifier_transformer.to_path(
+            provider=dataset.provider,
+            dataset_type=dataset.dataset_type,
+            identifier=dataset.identifier,
+        )
+
+        path = (
+            self.base_dir
+            / bucket
+            / f"provider={dataset.provider}"
+            / f"dataset_type={dataset.dataset_type}"
+            / identifier_path
+            / str(revision_id)
+            / filename
+        )
+        return path
 
     @abstractmethod
     def save_content(
@@ -20,10 +43,11 @@ class FileRepository(ABC):
     ) -> Path:
         pass
 
+    def get_read_path(self, storage_path: str) -> Path:
+        return self.base_dir / storage_path
+
     @abstractmethod
-    def load_content(
-        self, bucket: str, dataset: Dataset, revision_id: int, filename: str
-    ) -> BinaryIO:
+    def load_content(self, storage_path: str) -> BinaryIO:
         pass
 
     @classmethod
@@ -31,20 +55,6 @@ class FileRepository(ABC):
     def supports(cls, url: str) -> bool:
         pass
 
-    def get_path(
-        self, bucket: str, dataset: Dataset, revision_id: int, filename: str
-    ) -> Path:
-        path = (
-            self.base_dir
-            / bucket
-            / f"provider={dataset.provider}"
-            / f"dataset_type={dataset.dataset_type}"
-            / str(dataset.identifier)
-            / str(revision_id)
-            / filename
-        )
-        return path
-
     def get_relative_path(self, path: Path) -> Path:
         """Return the relative path to the base of the repository"""
         return path.relative_to(self.base_dir)

ingestify/domain/models/ingestion/ingestion_job.py

@@ -2,7 +2,7 @@ import itertools
 import json
 import logging
 import uuid
-from typing import Optional
+from typing import Optional, Iterator
 
 from ingestify import retrieve_http
 from ingestify.application.dataset_store import DatasetStore
@@ -22,7 +22,7 @@ from ingestify.utils import TaskExecutor, chunker
 logger = logging.getLogger(__name__)
 
 
-DEFAULT_CHUNK_SIZE = 1000
+DEFAULT_CHUNK_SIZE = 1_000
 
 
 def run_task(task):
@@ -32,14 +32,14 @@ def run_task(task):
 
 def to_batches(input_):
     if isinstance(input_, list):
-        batches = [input_]
+        batches = iter(input_)
     else:
         # Assume it's an iterator. Peek what's inside, and put it back
         try:
             peek = next(input_)
         except StopIteration:
             # Nothing to batch
-            return []
+            return iter([])
 
         input_ = itertools.chain([peek], input_)
 
@@ -184,6 +184,9 @@ class CreateDatasetTask(Task):
         return f"CreateDatasetTask({self.dataset_resource.provider} -> {self.dataset_resource.dataset_resource_id})"
 
 
+MAX_TASKS_PER_CHUNK = 10_000
+
+
 class IngestionJob:
     def __init__(
         self,
@@ -197,96 +200,126 @@ class IngestionJob:
 
     def execute(
         self, store: DatasetStore, task_executor: TaskExecutor
-    ) -> IngestionJobSummary:
-        with IngestionJobSummary.new(ingestion_job=self) as ingestion_job_summary:
-            with ingestion_job_summary.record_timing("get_dataset_collection"):
-                dataset_collection_metadata = store.get_dataset_collection(
-                    dataset_type=self.ingestion_plan.dataset_type,
-                    data_spec_versions=self.selector.data_spec_versions,
-                    selector=self.selector,
-                    metadata_only=True,
-                ).metadata
-
-            # There are two different, but similar flows here:
-            # 1. The discover_datasets returns a list, and the entire list can be processed at once
-            # 2. The discover_datasets returns an iterator of batches, in this case we need to process each batch
-            with ingestion_job_summary.record_timing("find_datasets"):
-                # Timing might be incorrect as it is an iterator
-                datasets = self.ingestion_plan.source.find_datasets(
-                    dataset_type=self.ingestion_plan.dataset_type,
-                    data_spec_versions=self.selector.data_spec_versions,
-                    dataset_collection_metadata=dataset_collection_metadata,
-                    **self.selector.custom_attributes,
-                )
+    ) -> Iterator[IngestionJobSummary]:
+        is_first_chunk = True
+        ingestion_job_summary = IngestionJobSummary.new(ingestion_job=self)
+        # Process all items in batches. Yield a IngestionJobSummary per batch
+
+        logger.info("Finding metadata")
+        with ingestion_job_summary.record_timing("get_dataset_collection"):
+            dataset_collection_metadata = store.get_dataset_collection(
+                dataset_type=self.ingestion_plan.dataset_type,
+                data_spec_versions=self.selector.data_spec_versions,
+                selector=self.selector,
+                metadata_only=True,
+            ).metadata
+        logger.info(f"Done: {dataset_collection_metadata}")
+
+        # There are two different, but similar flows here:
+        # 1. The discover_datasets returns a list, and the entire list can be processed at once
+        # 2. The discover_datasets returns an iterator of batches, in this case we need to process each batch
+        with ingestion_job_summary.record_timing("find_datasets"):
+            # Timing might be incorrect as it is an iterator
+            dataset_resources = self.ingestion_plan.source.find_datasets(
+                dataset_type=self.ingestion_plan.dataset_type,
+                data_spec_versions=self.selector.data_spec_versions,
+                dataset_collection_metadata=dataset_collection_metadata,
+                **self.selector.custom_attributes,
+            )
 
-            batches = to_batches(datasets)
+        finish_task_timer = ingestion_job_summary.start_timing("tasks")
 
-            with ingestion_job_summary.record_timing("tasks"):
-                for batch in batches:
-                    dataset_identifiers = [
-                        Identifier.create_from_selector(
-                            self.selector, **dataset_resource.dataset_resource_id
-                        )
-                        # We have to pass the data_spec_versions here as a Source can add some
-                        # extra data to the identifier which is retrieved in a certain data format
-                        for dataset_resource in batch
-                    ]
-
-                    # Load all available datasets based on the discovered dataset identifiers
-                    dataset_collection = store.get_dataset_collection(
-                        dataset_type=self.ingestion_plan.dataset_type,
-                        # Assume all DatasetResources share the same provider
-                        provider=batch[0].provider,
-                        selector=dataset_identifiers,
-                    )
+        batches = to_batches(dataset_resources)
 
-                    skip_count = 0
+        while True:
+            try:
+                batch = next(batches)
+            except StopIteration:
+                break
+            except Exception:
+                # TODO: handle exception on IngestionJob level
+                raise
 
-                    task_set = TaskSet()
-                    for dataset_resource in batch:
-                        dataset_identifier = Identifier.create_from_selector(
-                            self.selector, **dataset_resource.dataset_resource_id
-                        )
+            dataset_identifiers = [
+                Identifier.create_from_selector(
+                    self.selector, **dataset_resource.dataset_resource_id
+                )
+                # We have to pass the data_spec_versions here as a Source can add some
+                # extra data to the identifier which is retrieved in a certain data format
+                for dataset_resource in batch
+            ]
+
+            # Load all available datasets based on the discovered dataset identifiers
+            dataset_collection = store.get_dataset_collection(
+                dataset_type=self.ingestion_plan.dataset_type,
+                # Assume all DatasetResources share the same provider
+                provider=batch[0].provider,
+                selector=dataset_identifiers,
+            )
 
-                        if dataset := dataset_collection.get(dataset_identifier):
-                            if self.ingestion_plan.fetch_policy.should_refetch(
-                                dataset, dataset_resource
-                            ):
-                                task_set.add(
-                                    UpdateDatasetTask(
-                                        dataset=dataset,  # Current dataset from the database
-                                        dataset_resource=dataset_resource,  # Most recent dataset_resource
-                                        store=store,
-                                    )
-                                )
-                            else:
-                                skip_count += 1
-                        else:
-                            if self.ingestion_plan.fetch_policy.should_fetch(
-                                dataset_resource
-                            ):
-                                task_set.add(
-                                    CreateDatasetTask(
-                                        dataset_resource=dataset_resource,
-                                        store=store,
-                                    )
-                                )
-                            else:
-                                skip_count += 1
-
-                    if task_set:
-                        logger.info(
-                            f"Discovered {len(dataset_identifiers)} datasets from {self.ingestion_plan.source.__class__.__name__} "
-                            f"using selector {self.selector} => {len(task_set)} tasks. {skip_count} skipped."
-                        )
-                        logger.info(f"Running {len(task_set)} tasks")
-                        ingestion_job_summary.add_task_summaries(
-                            task_executor.run(run_task, task_set)
+            skipped_datasets = 0
+
+            task_set = TaskSet()
+            for dataset_resource in batch:
+                dataset_identifier = Identifier.create_from_selector(
+                    self.selector, **dataset_resource.dataset_resource_id
+                )
+
+                if dataset := dataset_collection.get(dataset_identifier):
+                    if self.ingestion_plan.fetch_policy.should_refetch(
+                        dataset, dataset_resource
+                    ):
+                        task_set.add(
+                            UpdateDatasetTask(
+                                dataset=dataset,  # Current dataset from the database
+                                dataset_resource=dataset_resource,  # Most recent dataset_resource
+                                store=store,
+                            )
                         )
                     else:
-                        logger.info(
-                            f"Discovered {len(dataset_identifiers)} datasets from {self.ingestion_plan.source.__class__.__name__} "
-                            f"using selector {self.selector} => nothing to do"
+                        skipped_datasets += 1
+                else:
+                    if self.ingestion_plan.fetch_policy.should_fetch(dataset_resource):
+                        task_set.add(
+                            CreateDatasetTask(
+                                dataset_resource=dataset_resource,
+                                store=store,
+                            )
                         )
+                    else:
+                        skipped_datasets += 1
+
+            if task_set:
+                logger.info(
+                    f"Discovered {len(dataset_identifiers)} datasets from {self.ingestion_plan.source.__class__.__name__} "
+                    f"using selector {self.selector} => {len(task_set)} tasks. {skipped_datasets} skipped."
+                )
+                logger.info(f"Running {len(task_set)} tasks")
+                ingestion_job_summary.add_task_summaries(
+                    task_executor.run(run_task, task_set)
+                )
+            else:
+                logger.info(
+                    f"Discovered {len(dataset_identifiers)} datasets from {self.ingestion_plan.source.__class__.__name__} "
+                    f"using selector {self.selector} => nothing to do"
+                )
+
+            ingestion_job_summary.increase_skipped_datasets(skipped_datasets)
+
+            if ingestion_job_summary.task_count() >= MAX_TASKS_PER_CHUNK:
+                finish_task_timer()
+                ingestion_job_summary.set_finished()
+                yield ingestion_job_summary
+
+                # Start a new one
+                is_first_chunk = False
+                ingestion_job_summary = IngestionJobSummary.new(ingestion_job=self)
+
+                # We will resume tasks, start timer right away
+                finish_task_timer = ingestion_job_summary.start_timing("tasks")
 
-        return ingestion_job_summary
+        if ingestion_job_summary.task_count() > 0 or is_first_chunk:
+            # When there is interesting information to store, or there was no data at all, store it
+            finish_task_timer()
+            ingestion_job_summary.set_finished()
+            yield ingestion_job_summary

ingestify/domain/models/ingestion/ingestion_job_summary.py

@@ -1,3 +1,4 @@
+import uuid
 from contextlib import contextmanager
 from datetime import datetime, timedelta
 from typing import Optional, List, TYPE_CHECKING
@@ -14,13 +15,15 @@ if TYPE_CHECKING:
 
 
 def format_duration(duration: timedelta):
-    return f"{duration.total_seconds():.2}sec"
+    return f"{duration.total_seconds():.2f}sec"
 
 
 class IngestionJobSummary(BaseModel):
+    ingestion_job_summary_id: str
     ingestion_job_id: str
 
     # From the IngestionPlan
+    provider: str
     source_name: str
     dataset_type: str
     data_spec_versions: DataSpecVersionCollection
@@ -31,6 +34,7 @@ class IngestionJobSummary(BaseModel):
     timings: List[Timing] = Field(default_factory=list)
     task_summaries: List[TaskSummary] = Field(default_factory=list)
 
+    skipped_datasets: int = 0
     failed_tasks: int = 0
     successful_tasks: int = 0
     ignored_successful_tasks: int = 0
@@ -38,7 +42,9 @@ class IngestionJobSummary(BaseModel):
     @classmethod
     def new(cls, ingestion_job: "IngestionJob"):
         args = dict(
+            ingestion_job_summary_id=str(uuid.uuid1()),
             ingestion_job_id=ingestion_job.ingestion_job_id,
+            provider=ingestion_job.ingestion_plan.source.provider,
             source_name=ingestion_job.ingestion_plan.source.name,
             dataset_type=ingestion_job.ingestion_plan.dataset_type,
             data_spec_versions=ingestion_job.ingestion_plan.data_spec_versions,
@@ -52,9 +58,23 @@ class IngestionJobSummary(BaseModel):
         yield
         self.timings.append(Timing(name=name, started_at=start, ended_at=utcnow()))
 
+    def start_timing(self, name):
+        start = utcnow()
+
+        def finish():
+            self.timings.append(Timing(name=name, started_at=start, ended_at=utcnow()))
+
+        return finish
+
     def add_task_summaries(self, task_summaries: List[TaskSummary]):
         self.task_summaries.extend(task_summaries)
 
+    def increase_skipped_datasets(self, skipped_datasets: int):
+        self.skipped_datasets += skipped_datasets
+
+    def task_count(self):
+        return len(self.task_summaries)
+
     def set_finished(self):
         self.failed_tasks = len(
             [task for task in self.task_summaries if task.status == TaskStatus.FAILED]
@@ -80,6 +100,7 @@ class IngestionJobSummary(BaseModel):
         print("--------------------")
         print(f"  - IngestionPlan:")
         print(f"        Source: {self.source_name}")
+        print(f"        Provider: {self.provider}")
         print(f"        DatasetType: {self.dataset_type}")
         print(f"  - Selector: {self.selector}")
         print(f"  - Timings: ")
@@ -89,14 +110,10 @@ class IngestionJobSummary(BaseModel):
             f"  - Tasks: {len(self.task_summaries)} - {(len(self.task_summaries) / self.duration.total_seconds()):.1f} tasks/sec"
         )
 
-        for status in [
-            TaskStatus.FAILED,
-            TaskStatus.FINISHED,
-            TaskStatus.FINISHED_IGNORED,
-        ]:
-            print(
-                f"    - {status.value.lower()}: {len([task for task in self.task_summaries if task.status == status])}"
-            )
+        print(f"    - Failed tasks: {self.failed_tasks}")
+        print(f"    - Successful tasks: {self.successful_tasks}")
+        print(f"    - Successful ignored tasks: {self.successful_tasks}")
+        print(f"    - Skipped datasets: {self.skipped_datasets}")
         print("--------------------")
 
     def __enter__(self):

ingestify/domain/services/identifier_key_transformer.py

@@ -0,0 +1,111 @@
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Callable, Optional, Union
+
+from ingestify.exceptions import IngestifyError
+
+
+class TransformationType(Enum):
+    IDENTITY = "IDENTITY"
+    BUCKET = "BUCKET"
+    RANGE = "RANGE"
+    CUSTOM = "CUSTOM"
+
+
+class Transformation(ABC):
+    @property
+    @abstractmethod
+    def transformation_type(self) -> TransformationType:
+        pass
+
+    def is_identity(self) -> bool:
+        return self.transformation_type == TransformationType.IDENTITY
+
+    @abstractmethod
+    def __call__(self, id_key_value: Union[str, int]) -> str:
+        pass
+
+    @classmethod
+    def from_dict(cls, config: dict) -> "Transformation":
+        type_ = config.pop("type")
+        if type_ == "bucket":
+            return BucketTransformation(**config)
+        else:
+            raise IngestifyError(f"Cannot build Transformation from {config}")
+
+
+class IdentityTransformation(Transformation):
+    transformation_type = TransformationType.IDENTITY
+
+    def __call__(self, id_key_value: Union[str, int]) -> str:
+        # Return the original value as a string
+        return str(id_key_value)
+
+
+class BucketTransformation(Transformation):
+    transformation_type = TransformationType.BUCKET
+
+    def __init__(self, bucket_size: int = None, bucket_count: int = None):
+        self.bucket_size = bucket_size
+        self.bucket_count = bucket_count
+
+    def __call__(self, id_key_value: Union[str, int]) -> str:
+        if self.bucket_count:
+            return str(int(id_key_value) % self.bucket_count)
+        elif self.bucket_size:
+            bucket_start = int(id_key_value) // self.bucket_size * self.bucket_size
+            bucket_end = bucket_start + self.bucket_size - 1
+            return f"{bucket_start}-{bucket_end}"
+        else:
+            raise IngestifyError("Invalid BucketTransformation")
+
+
+class IdentifierTransformer:
+    def __init__(self):
+        # Mapping of (provider, dataset_type, id_key) to the transformation
+        self.key_transformations: dict[tuple[str, str, str], Transformation] = {}
+
+    def register_transformation(
+        self,
+        provider: str,
+        dataset_type: str,
+        id_key: str,
+        transformation: Union[Transformation, dict],
+    ):
+        """
+        Registers a transformation for a specific (provider, dataset_type, id_key).
+        """
+        if isinstance(transformation, dict):
+            transformation = Transformation.from_dict(transformation)
+
+        self.key_transformations[(provider, dataset_type, id_key)] = transformation
+
+    def get_transformation(
+        self, provider: str, dataset_type: str, id_key: str
+    ) -> Transformation:
+        """
+        Retrieves the transformation for the given column or defaults to identity.
+        """
+        transformation = self.key_transformations.get((provider, dataset_type, id_key))
+        return transformation if transformation else IdentityTransformation()
+
+    def to_path(self, provider: str, dataset_type: str, identifier: dict) -> str:
+        """
+        Transforms the identifier into a path string using registered transformations.
+        For non-identity transformations, includes both transformed and original values,
+        with the transformed value appearing first and including the suffix.
+        """
+        path_parts = []
+        for key, value in identifier.items():
+            transformation = self.get_transformation(provider, dataset_type, key)
+            if not transformation.is_identity():
+                # Non-identity transformation: include both transformed and original
+                transformed_value = transformation(value)
+                suffix = transformation.transformation_type.value.lower()
+                path_parts.append(f"{key}_{suffix}={transformed_value}")
+
+            # Append the original value (either standalone for identity or alongside transformed)
+            path_parts.append(f"{key}={value}")
+
+        # Join the parts with `/` to form the full path
+        return "/".join(path_parts)

ingestify/infra/store/dataset/sqlalchemy/mapping.py

@@ -229,9 +229,11 @@ mapper_registry.map_imperatively(File, file_table)
 ingestion_job_summary = Table(
     "ingestion_job_summary",
     metadata,
-    Column("ingestion_job_id", String(255), primary_key=True),
+    Column("ingestion_job_summary_id", String(255), primary_key=True),
+    Column("ingestion_job_id", String(255), index=True),
     # From the IngestionPlan
     Column("source_name", String(255)),
+    Column("provider", String(255)),
     Column("dataset_type", String(255)),
     Column(
         "data_spec_versions",
@@ -250,6 +252,7 @@ ingestion_job_summary = Table(
     # Some task counters
     Column("successful_tasks", Integer),
     Column("ignored_successful_tasks", Integer),
+    Column("skipped_datasets", Integer),
     Column("failed_tasks", Integer),
     Column(
         "timings",
@@ -281,9 +284,9 @@ task_summary_table = Table(
     "task_summary",
     metadata,
     Column(
-        "ingestion_job_id",
+        "ingestion_job_summary_id",
         String(255),
-        ForeignKey("ingestion_job_summary.ingestion_job_id"),
+        ForeignKey("ingestion_job_summary.ingestion_job_summary_id"),
         primary_key=True,
     ),
     Column("task_id", Integer, primary_key=True),

ingestify/infra/store/dataset/sqlalchemy/repository.py

@@ -209,9 +209,7 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
             )
 
         if not metadata_only:
-            dataset_query = apply_query_filter(
-                self.session.query(Dataset)  # .options(joinedload(Dataset.revisions))
-            )
+            dataset_query = apply_query_filter(self.session.query(Dataset))
             datasets = list(dataset_query)
         else:
             datasets = []

ingestify/infra/store/file/local_file_repository.py

@@ -19,14 +19,12 @@ class LocalFileRepository(FileRepository):
         filename: str,
         stream: BinaryIO,
     ) -> Path:
-        path = self.get_path(bucket, dataset, revision_id, filename)
+        path = self.get_write_path(bucket, dataset, revision_id, filename)
         path.parent.mkdir(parents=True, exist_ok=True)
 
         with open(path, "wb") as fp:
             shutil.copyfileobj(stream, fp)
         return path
 
-    def load_content(
-        self, bucket: str, dataset: Dataset, revision_id: int, filename: str
-    ) -> BinaryIO:
-        return open(self.get_path(bucket, dataset, revision_id, filename), "rb")
+    def load_content(self, storage_path: str) -> BinaryIO:
+        return open(self.get_read_path(storage_path), "rb")

ingestify/infra/store/file/s3_file_repository.py

@@ -8,10 +8,7 @@ from ingestify.domain.models import FileRepository
 
 
 class S3FileRepository(FileRepository):
-    def __init__(self, url):
-        super().__init__(url)
-
-        self._s3 = None
+    _s3 = None
 
     @property
     def s3(self):
@@ -30,16 +27,14 @@ class S3FileRepository(FileRepository):
         filename: str,
         stream: BinaryIO,
     ) -> Path:
-        key = self.get_path(bucket, dataset, revision_id, filename)
+        key = self.get_write_path(bucket, dataset, revision_id, filename)
         s3_bucket = Path(key.parts[0])
 
         self.s3.Object(str(s3_bucket), str(key.relative_to(s3_bucket))).put(Body=stream)
         return key
 
-    def load_content(
-        self, bucket: str, dataset: Dataset, revision_id: int, filename: str
-    ) -> BinaryIO:
-        key = self.get_path(bucket, dataset, revision_id, filename)
+    def load_content(self, storage_path: str) -> BinaryIO:
+        key = self.get_read_path(storage_path)
         s3_bucket = Path(key.parts[0])
         return self.s3.Object(str(s3_bucket), str(key.relative_to(s3_bucket))).get()[
             "Body"

ingestify/main.py

@@ -19,6 +19,7 @@ from ingestify.domain.models.event import EventBus, Publisher, Subscriber
 
 from ingestify.domain.models.ingestion.ingestion_plan import IngestionPlan
 from ingestify.domain.models.fetch_policy import FetchPolicy
+from ingestify.domain.services.identifier_key_transformer import IdentifierTransformer
 from ingestify.exceptions import ConfigurationError
 from ingestify.infra import S3FileRepository, LocalFileRepository
 from ingestify.infra.store.dataset.sqlalchemy import SqlAlchemyDatasetRepository
@@ -60,11 +61,15 @@ def import_cls(name):
     return getattr(mod, components[-1])
 
 
-def build_file_repository(file_url: str) -> FileRepository:
+def build_file_repository(file_url: str, identifier_transformer) -> FileRepository:
     if file_url.startswith("s3://"):
-        repository = S3FileRepository(url=file_url)
+        repository = S3FileRepository(
+            url=file_url, identifier_transformer=identifier_transformer
+        )
     elif file_url.startswith("file://"):
-        repository = LocalFileRepository(url=file_url)
+        repository = LocalFileRepository(
+            url=file_url, identifier_transformer=identifier_transformer
+        )
     else:
         raise Exception(f"Cannot find repository to handle file {file_url}")
 
@@ -72,7 +77,7 @@ def build_file_repository(file_url: str) -> FileRepository:
 
 
 def get_dataset_store_by_urls(
-    metadata_url: str, file_url: str, bucket: str
+    metadata_url: str, file_url: str, bucket: str, dataset_types
 ) -> DatasetStore:
     """
     Initialize a DatasetStore by a DatasetRepository and a FileRepository
@@ -80,7 +85,19 @@ def get_dataset_store_by_urls(
     if not bucket:
         raise Exception("Bucket is not specified")
 
-    file_repository = build_file_repository(file_url)
+    identifier_transformer = IdentifierTransformer()
+    for dataset_type in dataset_types:
+        for id_key, id_config in dataset_type["identifier_keys"].items():
+            identifier_transformer.register_transformation(
+                provider=dataset_type["provider"],
+                dataset_type=dataset_type["dataset_type"],
+                id_key=id_key,
+                transformation=id_config["transformation"],
+            )
+
+    file_repository = build_file_repository(
+        file_url, identifier_transformer=identifier_transformer
+    )
 
     if secrets_manager.supports(metadata_url):
         metadata_url = secrets_manager.load_as_db_url(metadata_url)
@@ -103,14 +120,15 @@ def get_datastore(config_file, bucket: Optional[str] = None) -> DatasetStore:
     config = parse_config(config_file, default_value="")
 
     return get_dataset_store_by_urls(
-        dataset_url=config["main"]["dataset_url"],
+        metadata_url=config["main"]["metadata_url"],
         file_url=config["main"]["file_url"],
         bucket=bucket or config["main"].get("default_bucket"),
+        dataset_types=config.get("dataset_types", []),
     )
 
 
 def get_remote_datastore(url: str, bucket: str, **kwargs) -> DatasetStore:
-    return get_dataset_store_by_urls(dataset_url=url, file_url=url, bucket=bucket)
+    return get_dataset_store_by_urls(metadata_url=url, file_url=url, bucket=bucket)
 
 
 def get_source_cls(key: str) -> Type[Source]:
@@ -173,6 +191,7 @@ def get_engine(config_file, bucket: Optional[str] = None) -> IngestionEngine:
         metadata_url=config["main"]["metadata_url"],
         file_url=config["main"]["file_url"],
         bucket=bucket or config["main"].get("default_bucket"),
+        dataset_types=config.get("dataset_types", []),
     )
 
     # Setup an EventBus and wire some more components
@@ -188,7 +207,7 @@ def get_engine(config_file, bucket: Optional[str] = None) -> IngestionEngine:
         store=store,
     )
 
-    logger.info("Determining tasks...")
+    logger.info("Adding IngestionPlans...")
 
     fetch_policy = FetchPolicy()
 

{ingestify-0.2.0.dist-info → ingestify-0.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.2.0
+Version: 0.3.0
 Summary: Standardizing soccer tracking- and event data
 Author: Koen Vossen
 Author-email: info@koenvossen.nl

{ingestify-0.2.0.dist-info → ingestify-0.3.0.dist-info}/RECORD

@@ -1,14 +1,14 @@
-ingestify/__init__.py,sha256=rzYt6rUUedAUB4VDxDENn6bzWpACW34yfbQKVjTzgQg,301
+ingestify/__init__.py,sha256=DnPPEtJT32gAPuUKXgIsqUE4fIvc6QA96vrcKr6nz6A,301
 ingestify/cmdline.py,sha256=bIuyPgGEw4wIglNzpG9zp7TsJozsP8NSVsCe4eAyWUg,7189
 ingestify/exceptions.py,sha256=wMMuajl4AkQRfW60TLN7btJmQaH8-lUczXyW_2g9kOU,143
-ingestify/main.py,sha256=Lo8bCwOz3AOeO1pSTYhd7VjSZ8tcc9eSz0GLlwyy6DI,7632
+ingestify/main.py,sha256=0sTNoLcS7euOavIAviQIMTolRnXsvOvNbmFdXgXgxhE,8516
 ingestify/server.py,sha256=OVrf_XtpAQIn88MzqQzShXgsA9_jbnqYvD8YPBjn3cs,2413
 ingestify/source_base.py,sha256=GXAFCoT11Zov9M2v-fqQr9gFCXbtVfEIEH32V7r2oE8,382
 ingestify/utils.py,sha256=HETGhAoUlutLG0cQR63nac2JbFei9gnktDHeBQoYWfU,5692
 ingestify/application/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ingestify/application/dataset_store.py,sha256=LccTpvsMWCIV0ewzS5sIXKk2kaQcZhnXGFT8Eao3U3Q,12074
+ingestify/application/dataset_store.py,sha256=6xMHa_ShyPOyegIKl2xwmRl3BlV5i21z95cpKW3oARw,11712
 ingestify/application/ingestion_engine.py,sha256=PtMjKMpvfqB802G5zfKLzyamdH7qFOXl3x6_97y8w60,2288
-ingestify/application/loader.py,sha256=nqLKtwu48mJVumB9BtgTv79soCOtW9pzg-pvTvc66bc,7031
+ingestify/application/loader.py,sha256=v8ZcpMDEml9k_uFPFqT4WaCjXED_OIpAr7g0Pz5Hp6Y,7153
 ingestify/application/secrets_manager.py,sha256=5qCbPfUvRGP1Xbq6xPkMfpgYl8uPzF_0NbiKeRY5zxU,1757
 ingestify/domain/__init__.py,sha256=M7_fVTJjQUx53P4UQUPhowRKPKsIIjx4JYZL1yjHKsM,46
 ingestify/domain/models/__init__.py,sha256=cjQmdSDFA-saXjdF1mLPNWILFHIFgdj20J_fC5FmFsI,770
@@ -27,7 +27,7 @@ ingestify/domain/models/dataset/dataset_state.py,sha256=O95mea5N34HDXw7XsYzxHna4
 ingestify/domain/models/dataset/events.py,sha256=58VacQejQt-WPh9BywP4st5McauM3gXBQo0kaDnSekY,481
 ingestify/domain/models/dataset/file.py,sha256=nuoZI9GI5OysYwWCCyNsHMlm1Z9A1GbEKd38jvBzJ4E,4119
 ingestify/domain/models/dataset/file_collection.py,sha256=yaQmqFlmbajLCkU5QnjgqCvKzvVEZJrXVvinx5UGHcM,1193
-ingestify/domain/models/dataset/file_repository.py,sha256=ntzLiWZleZQFmrVsFvDSwfbOT86WtAXLbqgA8HlV56Q,1248
+ingestify/domain/models/dataset/file_repository.py,sha256=9EQprch9isAH2pbK7e7tfOKl6ulip4Ij1kBCTbO_rTc,1721
 ingestify/domain/models/dataset/identifier.py,sha256=EJYsxt0OS_43Y989DZQq8U9NjwmtvnHGYGMe6-hOBlI,575
 ingestify/domain/models/dataset/revision.py,sha256=O_1HG2S2EmYdWqI2K282S_D-d6IhRh_f4Q3wV8MEhkk,1311
 ingestify/domain/models/dataset/selector.py,sha256=kEGpU8pIyjZ0zwE9n2uo_NY5xrNanWiTTgapyMAUEsw,1039
@@ -39,8 +39,8 @@ ingestify/domain/models/event/event_bus.py,sha256=iseourbCwdUg-ODM5bM_u6cageJmce
 ingestify/domain/models/event/publisher.py,sha256=TOAawYYiPQCLR2Gm17LumMEzeapMDYcAYeklLFmwqAY,620
 ingestify/domain/models/event/subscriber.py,sha256=tP1ZFSvpJWKUITnATYekRxJzepz85UY7egBTMiP-dwg,1039
 ingestify/domain/models/ingestion/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ingestify/domain/models/ingestion/ingestion_job.py,sha256=GnBQVnTU3FdKdSElXEISUrQz-orGIHchnNAo20Qg0DY,11511
-ingestify/domain/models/ingestion/ingestion_job_summary.py,sha256=YygBv0GgU396HRe-exQqW2QmitBEnAh2VG_xkW3wdyQ,3645
+ingestify/domain/models/ingestion/ingestion_job.py,sha256=U6B62c7NGeHBAjmKhgOa4uHeul34xyR66WtWaPSRNTU,12276
+ingestify/domain/models/ingestion/ingestion_job_summary.py,sha256=1l9O3QJkYLs74HhrwAijwNEriPMwHN9OFG64Iz4z3uI,4262
 ingestify/domain/models/ingestion/ingestion_plan.py,sha256=KAvITBMQt3zmMFokESQJyp3rMuz1Hxr6msfZK1_faZM,648
 ingestify/domain/models/resources/__init__.py,sha256=ZuY9DPRfwk-aLB3Lj6DYP_NqMkcQfcYjZp4VejTtcbU,46
 ingestify/domain/models/resources/dataset_resource.py,sha256=NRnN029ct3P_Eg2d9Unb1t7A12Ksv_emBGhoe9DpPwM,3118
@@ -49,6 +49,7 @@ ingestify/domain/models/task/set.py,sha256=04txDYgS5rotXofD9TqChKdW0VZIYshrkfPIp
 ingestify/domain/models/task/task.py,sha256=OwLZQi9GGe0O8m1dKvJdN2Rham5oilI49KyKc5uV20A,161
 ingestify/domain/models/task/task_summary.py,sha256=ovzqKPstngRVzVA_JboQMluq5uQjKVJDsWNNcfcadhU,3774
 ingestify/domain/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ingestify/domain/services/identifier_key_transformer.py,sha256=y4GS9u9Ej1MO2jUhAxWbifp0mrE_MqTHvVVcoQzSKb4,4034
 ingestify/domain/services/transformers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/domain/services/transformers/kloppy_to_pandas.py,sha256=NcN6nTBGVn9gz-_hWZJTMcduS1Gg7EM4X95Cqxi1QIM,809
 ingestify/infra/__init__.py,sha256=V0hpLzPVTcOHRVh0gguF6FT30YIgEOUd5v87xUHkfZ4,88
@@ -63,11 +64,11 @@ ingestify/infra/source/wyscout.py,sha256=DxCzdkzYpVRHTfV9GpF8pe3FzwIk-WHYUlea6nO
 ingestify/infra/store/__init__.py,sha256=3dA6NWfB6FS5SFdQiSlJ0ZghBfnUAUuGIP5Vr4rkCqk,43
 ingestify/infra/store/dataset/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/infra/store/dataset/sqlalchemy/__init__.py,sha256=Z5JHWGO_hwT6rO-ecMOOAmOKjFFJi449KZvJTQgt6vQ,52
-ingestify/infra/store/dataset/sqlalchemy/mapping.py,sha256=-iTkC4_YGkkFrIsEZVTW2eoaofj4c7QZFaq7tl1r2G4,9288
-ingestify/infra/store/dataset/sqlalchemy/repository.py,sha256=O2g7g_clNz43g9gXjjBJZsIGvRTntQ6rJpQeDT8yQ7c,7141
+ingestify/infra/store/dataset/sqlalchemy/mapping.py,sha256=UlEIfNusSOEWOxPi_ORrdLSylbi6-TO1qwEmcrBLwog,9447
+ingestify/infra/store/dataset/sqlalchemy/repository.py,sha256=-eSR_F9tS9Hd3JNEpoJoDAb5RY38rFaKLMI3eBedjx8,7068
 ingestify/infra/store/file/__init__.py,sha256=DuEekZa2pmDuRCFiulbgoGotN0wGv3OrRXSvokY0PhY,104
-ingestify/infra/store/file/local_file_repository.py,sha256=0oIzjjKO5U_7gPXhsBJFUqQBarQTFQS499ZK7HNxMxo,893
-ingestify/infra/store/file/s3_file_repository.py,sha256=txDviBrY9EHn3soqLFvTrjSPkyh548RxUgx4T83j0QY,1331
+ingestify/infra/store/file/local_file_repository.py,sha256=1hhLqds5LlppJq2QBB0oN0Q98j6aXreCtYQYz3Q1P8g,819
+ingestify/infra/store/file/s3_file_repository.py,sha256=_sekV1rfEbwIaSGhKRnFQlj92E9qNgONiwXt6ZLCyGg,1188
 ingestify/static/templates/statsbomb_github/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/static/templates/statsbomb_github/config.yaml.jinja2,sha256=_gAuAipfBL3ddLacyS1IBP5JluvPS2vmrb8GGaFtcUM,386
 ingestify/static/templates/statsbomb_github/query.py,sha256=wjAOMoKvhX-BzCRqEm1SJp6YAcF8Fsq7ddrOaOpAeOk,364
@@ -78,8 +79,8 @@ ingestify/static/templates/wyscout/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRk
 ingestify/static/templates/wyscout/config.yaml.jinja2,sha256=0zQXuvJVwd0oL2OJsPMZ8sOvRbdfRbieSGLQ44ezmYc,379
 ingestify/static/templates/wyscout/query.py,sha256=wjAOMoKvhX-BzCRqEm1SJp6YAcF8Fsq7ddrOaOpAeOk,364
 ingestify/static/templates/wyscout/database/README.md,sha256=7IuzjKo7Pqkx5wkmOETRZDljVOslqfA3ALuHMONq5dg,32
-ingestify-0.2.0.dist-info/METADATA,sha256=8974JGisSq9_Q-4M1cFYY_AU5zBW7n_UZ8NKjj_ZBDM,18853
-ingestify-0.2.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-ingestify-0.2.0.dist-info/entry_points.txt,sha256=czYYXeX2ul4zdeB6bKlz3HaUF7zyVVcj9E_sRNDisI0,53
-ingestify-0.2.0.dist-info/top_level.txt,sha256=Lwnjgns4KequS7KiicXhh6mLUvcdfjzLyPI4qf_s4A0,10
-ingestify-0.2.0.dist-info/RECORD,,
+ingestify-0.3.0.dist-info/METADATA,sha256=-QlChdV6OYWkqSyXUmkQTG4deBliRsSmmZMTWKeURnI,18853
+ingestify-0.3.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+ingestify-0.3.0.dist-info/entry_points.txt,sha256=czYYXeX2ul4zdeB6bKlz3HaUF7zyVVcj9E_sRNDisI0,53
+ingestify-0.3.0.dist-info/top_level.txt,sha256=Lwnjgns4KequS7KiicXhh6mLUvcdfjzLyPI4qf_s4A0,10
+ingestify-0.3.0.dist-info/RECORD,,