ingestify 0.1.0__py3-none-any.whl

ingestify/domain/models/event/_old_event.py

@@ -0,0 +1,21 @@
+from dataclasses import dataclass
+from typing import Protocol
+
+from ingestify.domain import DatasetCreated
+
+
+#
+# class EventRepository:
+#     def __init__(self):
+#         self.events = []
+#
+#     def save(self, event):
+#         self.events.append(event)
+#
+#
+# class EventWriter:
+#     def __init__(self, event_repository: EventRepository):
+#         self.event_repository = event_repository
+#
+#     def dispatch(self, event):
+#         self.event_repository.save(event)

ingestify/domain/models/event/dispatcher.py

@@ -0,0 +1,8 @@
+from typing import Protocol
+
+from .domain_event import DomainEvent
+
+
+class Dispatcher(Protocol):
+    def dispatch(self, event: DomainEvent):
+        pass

ingestify/domain/models/event/domain_event.py

@@ -0,0 +1,10 @@
+from abc import abstractmethod, ABC
+from dataclasses import dataclass
+
+
+@dataclass
+class DomainEvent(ABC):
+    @property
+    @abstractmethod
+    def event_type(self) -> str:
+        pass

ingestify/domain/models/event/event_bus.py

@@ -0,0 +1,24 @@
+import logging
+
+
+from .dispatcher import Dispatcher
+
+
+logger = logging.getLogger(__name__)
+
+
+class EventBus:
+    def __init__(self):
+        self.dispatchers: list[Dispatcher] = []
+
+    def register(self, dispatcher: Dispatcher):
+        self.dispatchers.append(dispatcher)
+
+    def dispatch(self, event):
+
+        for dispatcher in self.dispatchers:
+            try:
+                dispatcher.dispatch(event)
+            except Exception as e:
+                logger.exception(f"Failed to handle {event}")
+                raise Exception(f"Failed to handle {event}") from e

ingestify/domain/models/event/publisher.py

@@ -0,0 +1,23 @@
+import logging
+
+from .dispatcher import Dispatcher
+from .domain_event import DomainEvent
+from .subscriber import Subscriber
+
+
+logger = logging.getLogger(__name__)
+
+
+class Publisher(Dispatcher):
+    def __init__(self):
+        self.subscribers: list[Subscriber] = []
+
+    def dispatch(self, event: DomainEvent):
+        for subscriber in self.subscribers:
+            try:
+                subscriber.handle(event)
+            except Exception:
+                logger.exception(f"Failed to handle {event} by {subscriber}")
+
+    def add_subscriber(self, subscriber: Subscriber):
+        self.subscribers.append(subscriber)

ingestify/domain/models/event/subscriber.py

@@ -0,0 +1,39 @@
+from typing import TYPE_CHECKING
+
+from .domain_event import DomainEvent
+
+if TYPE_CHECKING:
+    from ingestify.domain.models.dataset.events import (
+        DatasetCreated,
+        MetadataUpdated,
+        RevisionAdded,
+    )
+
+
+class Subscriber:
+    def __init__(self, store):
+        self.store = store
+
+    def on_dataset_created(self, event: "DatasetCreated"):
+        pass
+
+    def on_metadata_updated(self, event: "MetadataUpdated"):
+        pass
+
+    def on_revision_added(self, event: "RevisionAdded"):
+        pass
+
+    def handle(self, event: DomainEvent):
+        # TODO: fix the circular dependencies
+        from ingestify.domain.models.dataset.events import (
+            DatasetCreated,
+            MetadataUpdated,
+            RevisionAdded,
+        )
+
+        if isinstance(event, DatasetCreated):
+            self.on_dataset_created(event)
+        elif isinstance(event, MetadataUpdated):
+            self.on_metadata_updated(event)
+        elif isinstance(event, RevisionAdded):
+            self.on_revision_added(event)

ingestify/domain/models/extract_job.py

@@ -0,0 +1,23 @@
+from dataclasses import dataclass
+from typing import List
+
+from ingestify.domain.models import Source, Selector
+from ingestify.domain.models.data_spec_version_collection import (
+    DataSpecVersionCollection,
+)
+from ingestify.domain.models.fetch_policy import FetchPolicy
+
+
+@dataclass
+class ExtractJob:
+    source: Source
+    selectors: List[Selector]
+    fetch_policy: FetchPolicy
+    dataset_type: str
+    data_spec_versions: DataSpecVersionCollection
+
+    def __repr__(self):
+        return f'<ExtractJob source="{self.source.name}" dataset_type="{self.dataset_type}">'
+
+    def __str__(self):
+        return repr(self)

ingestify/domain/models/fetch_policy.py

@@ -0,0 +1,40 @@
+from datetime import timedelta
+
+from ingestify.domain import Dataset, Identifier, DatasetResource
+from ingestify.utils import utcnow
+
+
+class FetchPolicy:
+    def __init__(self):
+        # refresh all data that changed less than two day ago
+        self.min_age = utcnow() - timedelta(days=2)
+        self.last_change = utcnow() - timedelta(days=1)
+
+    def should_fetch(self, dataset_resource: DatasetResource) -> bool:
+        # this is called when dataset does not exist yet
+        return True
+
+    def should_refetch(
+        self, dataset: Dataset, dataset_resource: DatasetResource
+    ) -> bool:
+        current_revision = dataset.current_revision
+        if not dataset.revisions:
+            # TODO: this is weird? Dataset without any data. Fetch error?
+            return True
+        elif current_revision:
+            files_last_modified = {
+                file.file_id: file.last_modified
+                for file in dataset_resource.files.values()
+            }
+            if current_revision.is_changed(files_last_modified):
+                return True
+
+            # We don't set last_modified on Dataset level anymore, only on file level
+            # else:
+            #     if (
+            #         identifier.last_modified
+            #         and current_revision.created_at < identifier.last_modified
+            #     ):
+            #         return True
+
+        return False

ingestify/domain/models/resources/__init__.py

@@ -0,0 +1 @@
+from .dataset_resource import DatasetResource

ingestify/domain/models/resources/dataset_resource.py

@@ -0,0 +1,99 @@
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Optional, Callable, TYPE_CHECKING
+
+from ingestify.exceptions import DuplicateFile
+
+if TYPE_CHECKING:
+    from ingestify.domain import DraftFile, File
+    from ingestify.domain.models.dataset.dataset import DatasetState
+
+
+@dataclass(frozen=True)
+class FileResource:
+    dataset_resource: "DatasetResource"
+    file_id: str
+    last_modified: datetime
+    data_feed_key: str
+    data_spec_version: str
+
+    # DataSerializationFormat is "json" in case of json_content, otherwise file_loader will return it
+    # data_serialization_format: str
+
+    json_content: Optional[dict] = None
+
+    url: Optional[str] = None
+    http_options: Optional[dict] = None
+    data_serialization_format: Optional[str] = None
+
+    file_loader: Optional[
+        Callable[["FileResource", Optional["File"]], Optional["DraftFile"]]
+    ] = None
+
+    def __post_init__(self):
+        if self.json_content is None and not self.url and not self.file_loader:
+            raise TypeError(
+                "You need to specify `json_content`, `url` or a custom `file_loader`"
+            )
+
+
+class DatasetResource:
+    def __init__(
+        self,
+        dataset_resource_id: dict,
+        /,
+        dataset_type: str,
+        provider: str,
+        name: str,
+        metadata: Optional[dict] = None,
+        state: Optional["DatasetState"] = None,
+    ):
+        from ingestify.domain.models.dataset.dataset import DatasetState
+
+        self.dataset_type = dataset_type
+        self.provider = provider
+        self.dataset_resource_id = dataset_resource_id
+        self.name = name
+        self.metadata = metadata or {}
+        self.state = state or DatasetState.COMPLETE
+
+        self.files = {}
+
+    def add_file(
+        self,
+        last_modified: datetime,
+        data_feed_key: str,
+        # Some sources might not have a DataSpecVersion. Set a default
+        data_spec_version: str = "v1",
+        json_content: Optional[dict] = None,
+        url: Optional[str] = None,
+        http_options: Optional[dict] = None,
+        data_serialization_format: Optional[str] = None,
+        file_loader: Optional[
+            Callable[
+                ["FileResource", Optional["File"]],
+                Optional["DraftFile"],
+            ]
+        ] = None,
+    ):
+        file_id = f"{data_feed_key}__{data_spec_version}"
+        if file_id in self.files:
+            raise DuplicateFile(f"File with id {file_id} already exists.")
+
+        file_resource = FileResource(
+            dataset_resource=self,
+            file_id=file_id,
+            data_feed_key=data_feed_key,
+            data_spec_version=data_spec_version,
+            last_modified=last_modified,
+            json_content=json_content,
+            url=url,
+            http_options=http_options,
+            data_serialization_format=data_serialization_format,
+            file_loader=file_loader,
+        )
+
+        self.files[file_id] = file_resource
+
+        # Allow chaining
+        return self

ingestify/domain/models/sink.py

@@ -0,0 +1,16 @@
+from abc import ABC, abstractmethod
+
+from ingestify.utils import ComponentFactory, ComponentRegistry
+
+from .dataset import Dataset
+
+sink_registry = ComponentRegistry()
+
+
+class Sink(ABC, metaclass=sink_registry.metaclass):
+    @abstractmethod
+    def upsert(self, dataset: Dataset, data, params: dict):
+        pass
+
+
+sink_factory = ComponentFactory.build_factory(Sink, sink_registry)

ingestify/domain/models/source.py

@@ -0,0 +1,34 @@
+from abc import ABC, abstractmethod
+from typing import Dict, List, Optional, Iterable, Iterator, Union
+
+from .data_spec_version_collection import DataSpecVersionCollection
+from .dataset.collection_metadata import DatasetCollectionMetadata
+from .resources.dataset_resource import DatasetResource
+
+
+class Source(ABC):
+    def __init__(self, name: str, **kwargs):
+        self.name = name
+
+    @property
+    @abstractmethod
+    def provider(self) -> str:
+        raise NotImplemented
+
+    # TODO: consider making this required...
+    # @abstractmethod
+    # def discover_selectors(self, dataset_type: str) -> List[Dict]:
+    #     pass
+
+    @abstractmethod
+    def find_datasets(
+        self,
+        dataset_type: str,
+        data_spec_versions: DataSpecVersionCollection,
+        dataset_collection_metadata: DatasetCollectionMetadata,
+        **kwargs
+    ) -> Iterator[List[DatasetResource]]:
+        pass
+
+    def __repr__(self):
+        return self.__class__.__name__

ingestify/domain/models/task/__init__.py

@@ -0,0 +1,4 @@
+from .set import TaskSet
+from .task import Task
+
+__all__ = ["Task", "TaskSet"]

ingestify/domain/models/task/set.py

@@ -0,0 +1,21 @@
+from .task import Task
+
+
+class TaskSet:
+    def __init__(self, tasks=None):
+        self.tasks = tasks or []
+
+    def add(self, task: Task):
+        self.tasks.append(task)
+
+    def __len__(self):
+        return len(self.tasks)
+
+    def __iter__(self):
+        return iter(self.tasks)
+
+    def __add__(self, other: "TaskSet"):
+        return TaskSet(self.tasks + other.tasks)
+
+    def __bool__(self):
+        return len(self) > 0

ingestify/domain/models/task/task.py

@@ -0,0 +1,7 @@
+from abc import ABC, abstractmethod
+
+
+class Task(ABC):
+    @abstractmethod
+    def run(self):
+        pass

ingestify/domain/services/transformers/kloppy_to_pandas.py

@@ -0,0 +1,25 @@
+from typing import Dict, List
+
+import pandas as pd
+from kloppy import StatsBombSerializer
+
+from ingestify.domain.models import Dataset, LoadedFile
+
+
+class KloppyToPandasTransformer(Transformer):
+    def transform(
+        self, dataset: Dataset, loaded_files: Dict[str, LoadedFile]
+    ) -> pd.DataFrame:
+        if dataset.provider == "statsbomb":
+            serializer = StatsBombSerializer()
+            kloppy_dataset = serializer.deserialize(
+                inputs=dict(
+                    event_data=loaded_files["events.json"].stream,
+                    lineup_data=loaded_files["lineup.json"].stream,
+                ),
+                options={},
+            )
+        else:
+            raise Exception(f"Dataset provider {dataset.provider} not known")
+
+        return kloppy_dataset.to_pandas()

ingestify/exceptions.py

@@ -0,0 +1,10 @@
+class IngestifyError(Exception):
+    pass
+
+
+class ConfigurationError(IngestifyError):
+    pass
+
+
+class DuplicateFile(IngestifyError):
+    pass

ingestify/infra/__init__.py

@@ -0,0 +1,4 @@
+from .fetch.http import retrieve_http
+from .store import *
+
+__all__ = ["retrieve_http"]

ingestify/infra/fetch/http.py

@@ -0,0 +1,100 @@
+import json
+from datetime import datetime
+from email.utils import format_datetime, parsedate
+from hashlib import sha1
+from io import BytesIO
+from typing import Optional, Callable, Tuple
+
+import requests
+
+from ingestify.domain.models import DraftFile, File
+from ingestify.utils import utcnow
+
+
+def retrieve_http(
+    url,
+    current_file: Optional[File] = None,
+    headers: Optional[dict] = None,
+    pager: Optional[Tuple[str, Callable[[str, dict], Optional[str]]]] = None,
+    last_modified: Optional[datetime] = None,
+    **kwargs,
+) -> Optional[DraftFile]:
+    headers = headers or {}
+    if current_file:
+        if last_modified and current_file.modified_at >= last_modified:
+            # Not changed
+            return None
+        # else:
+        #     print(f"{current_file.modified_at=} {last_modified=}")
+        # headers["if-modified-since"] = (
+        #     format_datetime(current_file.modified_at, usegmt=True),
+        # )
+        headers["if-none-match"] = current_file.tag
+
+    http_kwargs = {}
+    file_attributes = {}
+    for key, item in kwargs.items():
+        if key.startswith("http_"):
+            http_kwargs[key[5:]] = item
+        elif key.startswith("file_"):
+            file_attributes[key[5:]] = item
+        else:
+            raise Exception(f"Don't know how to use {key}")
+
+    response = requests.get(url, headers=headers, **http_kwargs)
+    response.raise_for_status()
+    if response.status_code == 304:
+        # Not modified
+        return None
+
+    if last_modified:
+        # From metadata received from api in discover_datasets
+        modified_at = last_modified
+    elif "last-modified" in response.headers:
+        # Received from the webserver
+        modified_at = parsedate(response.headers["last-modified"])
+    else:
+        modified_at = utcnow()
+
+    tag = response.headers.get("etag")
+    # content_length = int(response.headers.get("content-length", 0))
+
+    if pager:
+        """
+        A pager helps with responses that return the data in pages.
+        """
+        data_path, pager_fn = pager
+        data = []
+        while True:
+            current_page_data = response.json()
+            data.extend(current_page_data[data_path])
+            next_url = pager_fn(url, current_page_data)
+            if not next_url:
+                break
+            else:
+                response = requests.get(next_url, headers=headers, **http_kwargs)
+
+        content = json.dumps({data_path: data}).encode("utf-8")
+    else:
+        content = response.content
+
+    if not tag:
+        tag = sha1(content).hexdigest()
+
+    # if not content_length: - Don't use http header as it might be wrong
+    # for example in case of compressed data
+    content_length = len(content)
+
+    if current_file and current_file.tag == tag:
+        # Not changed. Don't keep it
+        return None
+
+    return DraftFile(
+        created_at=utcnow(),
+        modified_at=modified_at,
+        tag=tag,
+        size=content_length,
+        content_type=response.headers.get("content-type"),
+        stream=BytesIO(content),
+        **file_attributes,
+    )

ingestify/infra/serialization/__init__.py

@@ -0,0 +1,50 @@
+import json
+from datetime import datetime
+from typing import Type, Any, TypeVar
+
+from dataclass_factory import Schema, Factory, NameStyle
+from dataclass_factory.schema_helpers import type_checker
+
+from ingestify.domain import DatasetCreated, Identifier
+from ingestify.domain.models.dataset.events import MetadataUpdated, RevisionAdded
+
+isotime_schema = Schema(
+    parser=lambda x: datetime.fromisoformat(x.replace("Z", "+00:00")),  # type: ignore
+    serializer=lambda x: datetime.isoformat(x).replace("+00:00", "Z"),
+)
+
+identifier_schema = Schema(
+    # json.loads(x) for backwards compatibility
+    parser=lambda x: Identifier(x if isinstance(x, dict) else json.loads(x)),
+    serializer=lambda x: dict(x),
+)
+
+factory = Factory(
+    schemas={
+        datetime: isotime_schema,
+        Identifier: identifier_schema,
+        DatasetCreated: Schema(
+            pre_parse=type_checker(DatasetCreated.event_type, "event_type")
+        ),
+        MetadataUpdated: Schema(
+            pre_parse=type_checker(MetadataUpdated.event_type, "event_type")
+        ),
+        RevisionAdded: Schema(
+            pre_parse=type_checker(RevisionAdded.event_type, "event_type")
+        ),
+        # ClipSelectionContent: Schema(pre_parse=type_checker(ClipSelectionContent.content_type, field="contentType")),
+        # TeamInfoImageContent: Schema(pre_parse=type_checker(TeamInfoImageContent.content_type, field="contentType")),
+        # StaticVideoContent: Schema(pre_parse=type_checker(StaticVideoContent.content_type, field="contentType"))
+    },
+    default_schema=Schema(),
+)
+
+T = TypeVar("T")
+
+
+def serialize(data: T, class_: Type[T] = None) -> Any:
+    return factory.dump(data, class_)
+
+
+def unserialize(data: Any, class_: Type[T]) -> T:
+    return factory.load(data, class_)

ingestify/infra/sink/postgresql.py

@@ -0,0 +1,50 @@
+from io import StringIO
+
+import pandas as pd
+from sqlalchemy import create_engine, text
+
+from ingestify.domain.models import Dataset, Sink
+
+
+# https://stackoverflow.com/questions/13947327/to-ignore-duplicate-keys-during-copy-from-in-postgresql
+def _copy_to(conn, tablename, data, extra_columns):
+    if isinstance(data, pd.DataFrame):
+        for k, v in extra_columns:
+            data[k] = v
+        tsv_file = data.to_csv(sep="\t", header=True, index=False)
+        stream = StringIO(tsv_file)
+    else:
+        raise Exception("Dont know how to handle data")
+
+    raw_connection = conn.connection
+    driver = conn.engine.dialect.driver
+
+    sql = f"COPY {tablename} FROM STDIN WITH CSV DELIMITER '\t' HEADER"
+    if driver == "pg8000":
+        # https://github.com/tlocke/pg8000/blob/13bc039e805e8a2cd8d816b939362b40018ea8ef/test/native/test_copy.py
+        raw_connection.run(sql=sql, stream=stream)
+    elif driver == "pgcopy2":
+        # https://github.com/psycopg/psycopg2/blob/1d3a89a0bba621dc1cc9b32db6d241bd2da85ad1/tests/test_copy.py
+        with raw_connection.cursor() as cursor:
+            cursor.copy_expert(sql=sql, file=stream)
+
+
+class PostgresSQLSink(Sink):
+    def __init__(self, url: str):
+        self.engine = create_engine(url)
+
+    def upsert(self, dataset: Dataset, data, params: dict):
+        if not isinstance(data, pd.DataFrame):
+            raise TypeError(
+                f"Data {type(data)} is not supported by the PostgresSQLSink"
+            )
+
+        table_name = params["table_name"]
+
+        with self.engine.begin() as conn:
+            conn.query(
+                text(
+                    f"DELETE FROM {table_name} WHERE dataset_id = {dataset.dataset_id}"
+                )
+            )
+            _copy_to(conn, table_name, data, dict(dataset_id=dataset.dataset_id))