ingestify 0.1.3__py3-none-any.whl → 0.3.0__py3-none-any.whl

ingestify/domain/models/task/task_summary.py

@@ -0,0 +1,118 @@
+import logging
+import traceback
+from contextlib import contextmanager
+from datetime import datetime
+from enum import Enum
+from typing import Optional, List
+from pydantic import Field, field_validator
+
+from ingestify.domain.models.base import BaseModel
+from ingestify.domain.models.dataset.identifier import Identifier
+from ingestify.domain.models.timing import Timing
+from ingestify.exceptions import IngestifyError
+from ingestify.utils import utcnow
+
+
+logger = logging.getLogger(__name__)
+
+
+class TaskStatus(str, Enum):
+    RUNNING = "RUNNING"
+    FINISHED = "FINISHED"
+    FINISHED_IGNORED = "FINISHED_IGNORED"  # Finished, but didn't produce any new data
+    FAILED = "FAILED"
+
+
+class Operation(str, Enum):
+    CREATE = "CREATE"
+    UPDATE = "UPDATE"
+
+
+class TaskSummary(BaseModel):
+    task_id: str
+    started_at: datetime
+    operation: Operation
+    dataset_identifier: Identifier
+    ended_at: Optional[datetime] = None
+    persisted_file_count: int = 0
+    bytes_retrieved: int = 0
+    last_modified: Optional[datetime] = None
+    status: TaskStatus = TaskStatus.RUNNING
+    timings: List[Timing] = Field(default_factory=list)
+
+    @field_validator("dataset_identifier", mode="before")
+    @classmethod
+    def ensure_list(cls, value) -> Identifier:
+        if not isinstance(value, Identifier):
+            return Identifier(**value)
+        return value
+
+    def record_load_file(self, fn, metadata: dict):
+        start = utcnow()
+        try:
+            result = None
+            return fn()
+        except Exception as e:
+            result = {
+                "type": type(e).__name__,
+                "message": str(e),
+                "traceback": traceback.format_exc(),
+            }
+            raise e
+        finally:
+            metadata = dict(result=result, **metadata)
+            self.timings.append(
+                Timing(
+                    name=f"Load of {metadata.get('file_id', 'file')}",
+                    started_at=start,
+                    ended_at=utcnow(),
+                    metadata=metadata,
+                )
+            )
+
+    @classmethod
+    @contextmanager
+    def new(cls, task_id: str, operation: Operation, dataset_identifier: Identifier):
+        start = utcnow()
+        task_summary = cls(
+            task_id=task_id,
+            started_at=start,
+            operation=operation,
+            dataset_identifier=dataset_identifier,
+        )
+        try:
+            yield task_summary
+
+            task_summary.set_status(TaskStatus.FINISHED)
+        except Exception as e:
+            logger.exception(f"Failed to execute task.")
+            task_summary.set_status(TaskStatus.FAILED)
+
+            # When the error comes from our own code, make sure it will be raised to the highest level
+            # raise
+            if isinstance(e, IngestifyError):
+                raise
+        finally:
+            task_summary.ended_at = utcnow()
+
+    @classmethod
+    def update(cls, task_id: str, dataset_identifier: Identifier):
+        return cls.new(task_id, Operation.UPDATE, dataset_identifier)
+
+    @classmethod
+    def create(cls, task_id: str, dataset_identifier: Identifier):
+        return cls.new(task_id, Operation.CREATE, dataset_identifier)
+
+    def set_stats_from_revision(self, revision: Optional["Revision"]):
+        if revision:
+            self.persisted_file_count = len(revision.modified_files)
+            self.bytes_retrieved = sum(file.size for file in revision.modified_files)
+            self.last_modified = max(
+                file.modified_at for file in revision.modified_files
+            )
+        else:
+            self.status = TaskStatus.FINISHED_IGNORED
+
+    def set_status(self, status: TaskStatus):
+        if self.status == TaskStatus.RUNNING:
+            self.status = status

ingestify/domain/models/timing.py

@@ -0,0 +1,16 @@
+from datetime import datetime
+from typing import Optional, Any
+from pydantic import BaseModel, ConfigDict
+
+
+class Timing(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    name: str
+    started_at: datetime
+    ended_at: datetime
+    metadata: Optional[dict[str, Any]] = None
+
+    @property
+    def duration(self):
+        return self.ended_at - self.started_at

ingestify/domain/services/identifier_key_transformer.py

@@ -0,0 +1,111 @@
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Callable, Optional, Union
+
+from ingestify.exceptions import IngestifyError
+
+
+class TransformationType(Enum):
+    IDENTITY = "IDENTITY"
+    BUCKET = "BUCKET"
+    RANGE = "RANGE"
+    CUSTOM = "CUSTOM"
+
+
+class Transformation(ABC):
+    @property
+    @abstractmethod
+    def transformation_type(self) -> TransformationType:
+        pass
+
+    def is_identity(self) -> bool:
+        return self.transformation_type == TransformationType.IDENTITY
+
+    @abstractmethod
+    def __call__(self, id_key_value: Union[str, int]) -> str:
+        pass
+
+    @classmethod
+    def from_dict(cls, config: dict) -> "Transformation":
+        type_ = config.pop("type")
+        if type_ == "bucket":
+            return BucketTransformation(**config)
+        else:
+            raise IngestifyError(f"Cannot build Transformation from {config}")
+
+
+class IdentityTransformation(Transformation):
+    transformation_type = TransformationType.IDENTITY
+
+    def __call__(self, id_key_value: Union[str, int]) -> str:
+        # Return the original value as a string
+        return str(id_key_value)
+
+
+class BucketTransformation(Transformation):
+    transformation_type = TransformationType.BUCKET
+
+    def __init__(self, bucket_size: int = None, bucket_count: int = None):
+        self.bucket_size = bucket_size
+        self.bucket_count = bucket_count
+
+    def __call__(self, id_key_value: Union[str, int]) -> str:
+        if self.bucket_count:
+            return str(int(id_key_value) % self.bucket_count)
+        elif self.bucket_size:
+            bucket_start = int(id_key_value) // self.bucket_size * self.bucket_size
+            bucket_end = bucket_start + self.bucket_size - 1
+            return f"{bucket_start}-{bucket_end}"
+        else:
+            raise IngestifyError("Invalid BucketTransformation")
+
+
+class IdentifierTransformer:
+    def __init__(self):
+        # Mapping of (provider, dataset_type, id_key) to the transformation
+        self.key_transformations: dict[tuple[str, str, str], Transformation] = {}
+
+    def register_transformation(
+        self,
+        provider: str,
+        dataset_type: str,
+        id_key: str,
+        transformation: Union[Transformation, dict],
+    ):
+        """
+        Registers a transformation for a specific (provider, dataset_type, id_key).
+        """
+        if isinstance(transformation, dict):
+            transformation = Transformation.from_dict(transformation)
+
+        self.key_transformations[(provider, dataset_type, id_key)] = transformation
+
+    def get_transformation(
+        self, provider: str, dataset_type: str, id_key: str
+    ) -> Transformation:
+        """
+        Retrieves the transformation for the given column or defaults to identity.
+        """
+        transformation = self.key_transformations.get((provider, dataset_type, id_key))
+        return transformation if transformation else IdentityTransformation()
+
+    def to_path(self, provider: str, dataset_type: str, identifier: dict) -> str:
+        """
+        Transforms the identifier into a path string using registered transformations.
+        For non-identity transformations, includes both transformed and original values,
+        with the transformed value appearing first and including the suffix.
+        """
+        path_parts = []
+        for key, value in identifier.items():
+            transformation = self.get_transformation(provider, dataset_type, key)
+            if not transformation.is_identity():
+                # Non-identity transformation: include both transformed and original
+                transformed_value = transformation(value)
+                suffix = transformation.transformation_type.value.lower()
+                path_parts.append(f"{key}_{suffix}={transformed_value}")
+
+            # Append the original value (either standalone for identity or alongside transformed)
+            path_parts.append(f"{key}={value}")
+
+        # Join the parts with `/` to form the full path
+        return "/".join(path_parts)

ingestify/infra/fetch/http.py

@@ -69,7 +69,12 @@ def retrieve_http(
         else:
             raise Exception(f"Don't know how to use {key}")
 
+    ignore_not_found = http_kwargs.pop("ignore_not_found", False)
+
     response = get_session().get(url, headers=headers, **http_kwargs)
+    if response.status_code == 404 and ignore_not_found:
+        return None
+
     response.raise_for_status()
     if response.status_code == 304:
         # Not modified

ingestify/infra/source/statsbomb_github.py

@@ -1,22 +1,22 @@
-import json
 from datetime import datetime
 
 import requests
 
-from ingestify import Source, retrieve_http
-from ingestify.domain import DraftFile
+from ingestify import Source, DatasetResource
 from ingestify.domain.models.dataset.dataset import DatasetState
 
 BASE_URL = "https://raw.githubusercontent.com/statsbomb/open-data/master/data"
+DATA_SPEC_VERSION = "v1-open-data"
 
 
 class StatsbombGithub(Source):
     provider = "statsbomb"
 
-    def discover_selectors(self, dataset_type: str, data_spec_versions: None = None):
+    def discover_selectors(self, dataset_type: str):
         assert dataset_type == "match"
 
         competitions = requests.get(f"{BASE_URL}/competitions.json").json()
+
         return [
             dict(
                 competition_id=competition["competition_id"],
@@ -25,68 +25,81 @@ class StatsbombGithub(Source):
             for competition in competitions
         ]
 
-    def discover_datasets(
+    def find_datasets(
         self,
-        dataset_type,
-        competition_id: str = None,
-        season_id: str = None,
+        dataset_type: str,
+        competition_id: str,
+        season_id: str,
+        match_id: str = None,
         data_spec_versions=None,
+        dataset_collection_metadata=None,
     ):
         assert dataset_type == "match"
 
-        datasets = []
-
         matches = requests.get(
             f"{BASE_URL}/matches/{competition_id}/{season_id}.json"
         ).json()
 
         for match in matches:
-            last_updated = match["last_updated"]
-            if "Z" not in last_updated:
-                # Assume UTC
-                last_updated += "Z"
-
-            last_modified = datetime.fromisoformat(last_updated.replace("Z", "+00:00"))
-
-            dataset = dict(
-                competition_id=competition_id,
-                season_id=season_id,
-                match_id=match["match_id"],
-                _last_modified=last_modified,
-                _match=match,
-                _metadata=match,
-                _state=DatasetState.COMPLETE,
-            )
-            datasets.append(dataset)
-        return datasets
+            if match_id:
+                if match["match_id"] != match_id:
+                    continue
 
-    def fetch_dataset_files(
-        self, dataset_type, identifier, current_revision, data_spec_versions
-    ):
-        assert dataset_type == "match"
+            last_modified = datetime.fromisoformat(match["last_updated"] + "+00:00")
 
-        current_files = current_revision.modified_files_map if current_revision else {}
-        files = {}
-        for filename, url in [
-            ("lineups.json", f"{BASE_URL}/lineups/{identifier.match_id}.json"),
-            ("events.json", f"{BASE_URL}/events/{identifier.match_id}.json"),
-        ]:
-            data_feed_key = filename.split(".")[0]
-            file_id = data_feed_key + "__v1"
-            files[file_id] = retrieve_http(
-                url,
-                current_files.get(filename),
-                file_data_feed_key=data_feed_key,
-                file_data_spec_version="v1",
-                file_data_serialization_format="json",
+            # Open data is always complete.. I guess?
+            state = DatasetState.COMPLETE
+
+            name = (
+                f"{match['match_date']} / "
+                f"{match['home_team']['home_team_name']} - {match['away_team']['away_team_name']}"
+            )
+
+            dataset_resource = DatasetResource(
+                dataset_resource_id=dict(
+                    competition_id=competition_id,
+                    season_id=season_id,
+                    match_id=match["match_id"],
+                ),
+                dataset_type=dataset_type,
+                provider=self.provider,
+                name=name,
+                metadata=match,
+                state=state,
             )
 
-        files["match__v1"] = DraftFile.from_input(
-            json.dumps(identifier._match, indent=4),
-            data_feed_key="match",
-            data_spec_version="v1",
-            data_serialization_format="json",
-            modified_at=None,
-        )
+            dataset_resource.add_file(
+                last_modified=last_modified,
+                data_feed_key="match",
+                data_spec_version=DATA_SPEC_VERSION,
+                json_content=match,
+            )
 
-        return files
+            if state.is_complete:
+                name += f" / {match['home_score']}-{match['away_score']}"
+
+                for data_feed_key in ["lineups", "events"]:
+                    dataset_resource.add_file(
+                        last_modified=last_modified,
+                        data_feed_key=data_feed_key,
+                        data_spec_version=DATA_SPEC_VERSION,
+                        url=f"{BASE_URL}/{data_feed_key}/{match['match_id']}.json",
+                        data_serialization_format="json",
+                    )
+
+                if (
+                    match["last_updated_360"]
+                    and match["match_status_360"] == "available"
+                ):
+                    dataset_resource.add_file(
+                        last_modified=datetime.fromisoformat(
+                            match["last_updated_360"] + "+00:00"
+                        ),
+                        data_feed_key="360-frames",
+                        data_spec_version=DATA_SPEC_VERSION,
+                        url=f"{BASE_URL}/three-sixty/{match['match_id']}.json",
+                        data_serialization_format="json",
+                        http_options={"ignore_not_found": True},
+                    )
+
+            yield dataset_resource

ingestify/infra/store/dataset/__init__.py

@@ -1,2 +0,0 @@
-from .local_dataset_repository import LocalDatasetRepository
-from .sqlalchemy import SqlAlchemyDatasetRepository

ingestify/infra/store/dataset/sqlalchemy/mapping.py

@@ -1,5 +1,7 @@
 import datetime
+from dataclasses import is_dataclass, asdict
 from pathlib import Path
+from typing import Optional
 
 from sqlalchemy import (
     JSON,
@@ -13,11 +15,37 @@ from sqlalchemy import (
     String,
     Table,
     TypeDecorator,
+    Boolean,
 )
 from sqlalchemy.orm import registry, relationship
 
+from ingestify.domain import Selector, Identifier, DataSpecVersionCollection
 from ingestify.domain.models import Dataset, File, Revision
 from ingestify.domain.models.dataset.dataset import DatasetState
+from ingestify.domain.models.ingestion.ingestion_job_summary import (
+    IngestionJobSummary,
+)
+from ingestify.domain.models.task.task_summary import TaskSummary, Operation, TaskStatus
+from ingestify.domain.models.timing import Timing
+from ingestify.domain.models.dataset.revision import RevisionState
+
+
+def JSONType(serializer=None, deserializer=None):
+    class _JsonType(TypeDecorator):
+        cache_ok = True
+        impl = JSON
+
+        def process_bind_param(self, value, dialect):
+            if serializer is not None:
+                return serializer(value)
+            return value
+
+        def process_result_value(self, value, dialect):
+            if deserializer is not None:
+                return deserializer(value)
+            return value
+
+    return _JsonType
 
 
 class TZDateTime(TypeDecorator):
@@ -25,7 +53,10 @@ class TZDateTime(TypeDecorator):
     LOCAL_TIMEZONE = datetime.datetime.utcnow().astimezone().tzinfo
     cache_ok = True
 
-    def process_bind_param(self, value: datetime, dialect):
+    def process_bind_param(self, value: Optional[datetime.datetime], dialect):
+        if not value:
+            return None
+
         if value.tzinfo is None:
             value = value.astimezone(self.LOCAL_TIMEZONE)
 
@@ -67,6 +98,45 @@ class DatasetStateString(TypeDecorator):
         return DatasetState[value]
 
 
+class RevisionStateString(TypeDecorator):
+    impl = String(255)
+
+    def process_bind_param(self, value: RevisionState, dialect):
+        return value.value
+
+    def process_result_value(self, value, dialect):
+        if not value:
+            return value
+
+        return RevisionState[value]
+
+
+class OperationString(TypeDecorator):
+    impl = String(255)
+
+    def process_bind_param(self, value: Operation, dialect):
+        return value.value
+
+    def process_result_value(self, value, dialect):
+        if not value:
+            return value
+
+        return Operation[value]
+
+
+class TaskStatusString(TypeDecorator):
+    impl = String(255)
+
+    def process_bind_param(self, value: TaskStatus, dialect):
+        return value.value
+
+    def process_result_value(self, value, dialect):
+        if not value:
+            return value
+
+        return TaskStatus[value]
+
+
 mapper_registry = registry()
 
 metadata = MetaData()
@@ -80,7 +150,7 @@ dataset_table = Table(
     Column("dataset_type", String(255)),
     Column("state", DatasetStateString),
     Column("name", String(255)),
-    Column("identifier", JSON),
+    Column("identifier", JSONType(deserializer=lambda item: Identifier(**item))),
     Column("metadata", JSON),
     Column("created_at", TZDateTime(6)),
     Column("updated_at", TZDateTime(6)),
@@ -95,7 +165,10 @@ revision_table = Table(
     Column("revision_id", Integer, primary_key=True),
     Column("description", String(255)),
     Column("created_at", TZDateTime(6)),
+    Column("state", RevisionStateString, default=RevisionState.PENDING_VALIDATION),
+    Column("source", JSONType()),
 )
+
 file_table = Table(
     "file",
     metadata,
@@ -129,7 +202,7 @@ mapper_registry.map_imperatively(
             Revision,
             backref="dataset",
             order_by=revision_table.c.revision_id,
-            lazy="joined",
+            lazy="selectin",
             cascade="all, delete-orphan",
         ),
     },
@@ -143,7 +216,7 @@ mapper_registry.map_imperatively(
             File,
             order_by=file_table.c.file_id,
             primaryjoin="and_(Revision.revision_id==File.revision_id, Revision.dataset_id==File.dataset_id)",
-            lazy="joined",
+            lazy="selectin",
             cascade="all, delete-orphan",
         )
     },
@@ -151,3 +224,113 @@ mapper_registry.map_imperatively(
 
 
 mapper_registry.map_imperatively(File, file_table)
+
+
+ingestion_job_summary = Table(
+    "ingestion_job_summary",
+    metadata,
+    Column("ingestion_job_summary_id", String(255), primary_key=True),
+    Column("ingestion_job_id", String(255), index=True),
+    # From the IngestionPlan
+    Column("source_name", String(255)),
+    Column("provider", String(255)),
+    Column("dataset_type", String(255)),
+    Column(
+        "data_spec_versions",
+        JSONType(
+            serializer=lambda data_spec_versions: data_spec_versions.to_dict(),
+            deserializer=lambda data_spec_versions: DataSpecVersionCollection.from_dict(
+                data_spec_versions
+            ),
+        ),
+    ),
+    Column(
+        "selector", JSONType(serializer=lambda selector: selector.filtered_attributes)
+    ),
+    Column("started_at", TZDateTime(6)),
+    Column("finished_at", TZDateTime(6)),
+    # Some task counters
+    Column("successful_tasks", Integer),
+    Column("ignored_successful_tasks", Integer),
+    Column("skipped_datasets", Integer),
+    Column("failed_tasks", Integer),
+    Column(
+        "timings",
+        JSONType(
+            serializer=lambda timings: [
+                timing.model_dump(mode="json") for timing in timings
+            ],
+            deserializer=lambda timings: [
+                Timing.model_validate(timing) for timing in timings
+            ],
+        ),
+    ),
+    # Column(
+    #     "task_summaries",
+    #     JSONType(
+    #         serializer=lambda task_summaries: [
+    #             task_summary.model_dump(mode="json") for task_summary in task_summaries
+    #         ],
+    #         deserializer=lambda task_summaries: [
+    #             TaskSummary.model_validate(task_summary)
+    #             for task_summary in task_summaries
+    #         ],
+    #     ),
+    # ),
+)
+
+
+task_summary_table = Table(
+    "task_summary",
+    metadata,
+    Column(
+        "ingestion_job_summary_id",
+        String(255),
+        ForeignKey("ingestion_job_summary.ingestion_job_summary_id"),
+        primary_key=True,
+    ),
+    Column("task_id", Integer, primary_key=True),
+    Column("started_at", TZDateTime(6)),
+    Column("ended_at", TZDateTime(6)),
+    Column("operation", OperationString),
+    Column(
+        "dataset_identifier", JSONType(deserializer=lambda item: Identifier(**item))
+    ),
+    Column("persisted_file_count", Integer),
+    Column("bytes_retrieved", Integer),
+    Column("last_modified", TZDateTime(6)),
+    Column("status", TaskStatusString),
+    Column(
+        "timings",
+        JSONType(
+            serializer=lambda timings: [
+                timing.model_dump(mode="json") for timing in timings
+            ],
+            deserializer=lambda timings: [
+                Timing.model_validate(timing) for timing in timings
+            ],
+        ),
+    ),
+    # Column("description", String(255)),
+    # Column("created_at", TZDateTime(6)),
+    # Column("state", RevisionStateString, default=RevisionState.PENDING_VALIDATION),
+    # Column("source", JSONType()),
+)
+
+
+mapper_registry.map_imperatively(
+    IngestionJobSummary,
+    ingestion_job_summary,
+    properties={
+        "task_summaries": relationship(
+            TaskSummary,
+            backref="ingestion_job_summary",
+            # order_by=task_summary_table.c.revision_id,
+            lazy="selectin",
+            cascade="all, delete-orphan",
+        ),
+    },
+)
+
+
+mapper_registry.map_imperatively(TaskSummary, task_summary_table)