ingestify 0.1.0__py3-none-any.whl

ingestify/domain/models/__init__.py

@@ -0,0 +1,45 @@
+from .dataset import (
+    Dataset,
+    DatasetCollection,
+    DatasetRepository,
+    DatasetCreated,
+    DraftFile,
+    File,
+    FileRepository,
+    FileCollection,
+    Identifier,
+    LoadedFile,
+    Selector,
+    Revision,
+    dataset_repository_factory,
+    file_repository_factory,
+)
+from .sink import Sink, sink_factory
+from .source import Source
+from .task import Task, TaskSet
+from .data_spec_version_collection import DataSpecVersionCollection
+from .resources import DatasetResource
+
+__all__ = [
+    "Selector",
+    "Identifier",
+    "Source",
+    "Revision",
+    "Dataset",
+    "DatasetCollection",
+    "DatasetResource",
+    "File",
+    "DraftFile",
+    "DatasetCreated",
+    "LoadedFile",
+    "FileRepository",
+    "FileCollection",
+    "DatasetRepository",
+    "dataset_repository_factory",
+    "file_repository_factory",
+    "TaskSet",
+    "Task",
+    "Sink",
+    "sink_factory",
+    "DataSpecVersionCollection",
+]

ingestify/domain/models/data_spec_version_collection.py

@@ -0,0 +1,33 @@
+import copy
+from dataclasses import dataclass
+from typing import Dict, Union, List, Set, Optional
+
+
+class DataSpecVersionCollection(dict):
+    @classmethod
+    def from_dict(cls, items: Dict[str, Union[str, List[str], Set[str]]]):
+        items_ = {}
+        for data_feed_key, data_spec_versions in items.items():
+            if isinstance(data_spec_versions, str):
+                data_spec_versions = {data_spec_versions}
+            elif isinstance(data_spec_versions, list):
+                data_spec_versions = set(data_spec_versions)
+            items_[data_feed_key] = data_spec_versions
+
+        return cls(items_)
+
+    def copy(self):
+        return DataSpecVersionCollection(copy.deepcopy(self))
+
+    def merge(self, other: "DataSpecVersionCollection"):
+        for data_feed_key, data_spec_versions in other.items():
+            if data_feed_key in self:
+                self[data_feed_key].update(data_spec_versions)
+            else:
+                self[data_feed_key] = data_spec_versions
+
+    def get_version(self, data_feed_key: str, default: Optional[str] = None):
+        items = self.get(data_feed_key)
+        if not items:
+            return default
+        return list(items)[0]

ingestify/domain/models/dataset/__init__.py

@@ -0,0 +1,27 @@
+from .collection import DatasetCollection
+from .dataset import Dataset
+from .dataset_repository import DatasetRepository, dataset_repository_factory
+from .file import DraftFile, File, LoadedFile
+from .file_repository import FileRepository, file_repository_factory
+from .file_collection import FileCollection
+from .identifier import Identifier
+from .selector import Selector
+from .revision import Revision
+from .events import DatasetCreated
+
+__all__ = [
+    "Selector",
+    "Revision",
+    "Dataset",
+    "Identifier",
+    "DatasetCollection",
+    "DatasetCreated",
+    "dataset_repository_factory",
+    "File",
+    "DraftFile",
+    "LoadedFile",
+    "DatasetRepository",
+    "FileRepository",
+    "file_repository_factory",
+    "FileCollection",
+]

ingestify/domain/models/dataset/collection.py

@@ -0,0 +1,44 @@
+from typing import List, Optional
+
+from .collection_metadata import DatasetCollectionMetadata
+from .dataset import Dataset
+from .identifier import Identifier
+
+
+class DatasetCollection:
+    def __init__(
+        self,
+        metadata: Optional[DatasetCollectionMetadata] = None,
+        datasets: Optional[List[Dataset]] = None,
+    ):
+        datasets = datasets or []
+
+        # TODO: this fails when datasets contains different dataset_types with overlapping identifiers
+        self.datasets: dict[str, Dataset] = {
+            dataset.identifier.key: dataset for dataset in datasets
+        }
+        self.metadata = metadata
+
+    def loaded(self):
+        return self.metadata.count == len(self.datasets)
+
+    def get(self, dataset_identifier: Identifier) -> Dataset:
+        return self.datasets.get(dataset_identifier.key)
+
+    def __len__(self):
+        return len(self.datasets)
+
+    def __iter__(self):
+        return iter(self.datasets.values())
+
+    def get_dataset_by_id(self, dataset_id):
+        for dataset in self:
+            if dataset.dataset_id == dataset_id:
+                return dataset
+        return None
+
+    def first(self):
+        try:
+            return next(iter(self.datasets.values()))
+        except StopIteration:
+            raise Exception("No items in the collection")

ingestify/domain/models/dataset/collection_metadata.py

@@ -0,0 +1,13 @@
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Optional
+
+
+@dataclass
+class DatasetCollectionMetadata:
+    # This can be useful to figure out if a backfill is required
+    first_modified: Optional[datetime]
+
+    # Use the last modified to only retrieve datasets that are changed
+    last_modified: Optional[datetime]
+    row_count: int

ingestify/domain/models/dataset/dataset.py

@@ -0,0 +1,104 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import List, Optional
+
+from ingestify.utils import utcnow
+
+from .file import DraftFile
+from .identifier import Identifier
+from .revision import Revision
+
+
+class DatasetState(Enum):
+    SCHEDULED = "SCHEDULED"
+    PARTIAL = "PARTIAL"
+    COMPLETE = "COMPLETE"
+
+    @property
+    def is_complete(self):
+        return self == DatasetState.COMPLETE
+
+    def __str__(self):
+        return self.value
+
+
+@dataclass
+class Dataset:
+    bucket: str  # This must be set by the DatasetRepository
+
+    dataset_id: str
+    name: str
+    state: DatasetState
+
+    dataset_type: str
+    provider: str
+
+    identifier: Identifier
+    metadata: dict
+
+    created_at: datetime
+    updated_at: datetime
+
+    revisions: List[Revision] = field(default_factory=list)
+
+    @property
+    def is_complete(self):
+        return self.state.is_complete
+
+    def next_revision_id(self):
+        return len(self.revisions)
+
+    def add_revision(self, revision: Revision):
+        self.revisions.append(revision)
+        self.updated_at = utcnow()
+
+    def update_from_resource(self, dataset_resource) -> bool:
+        changed = False
+        if self.name != dataset_resource.name:
+            self.name = dataset_resource.name
+            changed = True
+
+        if self.metadata != dataset_resource.metadata:
+            self.metadata = dataset_resource.metadata
+            changed = True
+
+        if self.state != dataset_resource.state:
+            self.state = dataset_resource.state
+            changed = True
+
+        if changed:
+            self.updated_at = utcnow()
+
+        return changed
+
+    @property
+    def current_revision(self) -> Optional[Revision]:
+        """
+        When multiple versions are available, squash versions into one single version which
+        contents all most recent files.
+        """
+        if not self.revisions:
+            return None
+        elif len(self.revisions) == 1:
+            return self.revisions[0]
+        else:
+            files = {}
+
+            for revision in self.revisions:
+                for file_id, file in revision.modified_files_map.items():
+                    if isinstance(file, DraftFile):
+                        raise Exception(
+                            f"Cannot squash draft file. Revision: {revision}. FileId: {file_id}"
+                        )
+                    files[file_id] = file
+                    files[file_id].revision_id = revision.revision_id
+
+            return Revision(
+                revision_id=self.revisions[-1].revision_id,
+                created_at=self.revisions[-1].created_at,
+                # created_at=max([file.modified_at for file in files.values()]),
+                description="Squashed revision",
+                is_squashed=True,
+                modified_files=list(files.values()),
+            )

ingestify/domain/models/dataset/dataset_repository.py

@@ -0,0 +1,46 @@
+from abc import ABC, abstractmethod
+from typing import Optional, List, Union
+
+from ingestify.utils import ComponentFactory, ComponentRegistry
+
+from .collection import DatasetCollection
+from .dataset import Dataset
+from .selector import Selector
+
+dataset_repository_registry = ComponentRegistry()
+
+
+class DatasetRepository(ABC, metaclass=dataset_repository_registry.metaclass):
+    @abstractmethod
+    def get_dataset_collection(
+        self,
+        bucket: str,
+        dataset_type: Optional[str] = None,
+        dataset_id: Optional[Union[str, List[str]]] = None,
+        provider: Optional[str] = None,
+        selector: Optional[Union[Selector, List[Selector]]] = None,
+        metadata_only: bool = False,
+    ) -> DatasetCollection:
+        pass
+
+    @abstractmethod
+    def destroy(self, dataset: Dataset):
+        pass
+
+    @abstractmethod
+    def save(self, bucket: str, dataset: Dataset):
+        pass
+
+    @abstractmethod
+    def next_identity(self):
+        pass
+
+    @classmethod
+    @abstractmethod
+    def supports(cls, url: str) -> bool:
+        pass
+
+
+dataset_repository_factory = ComponentFactory.build_factory(
+    DatasetRepository, dataset_repository_registry
+)

ingestify/domain/models/dataset/events.py

@@ -0,0 +1,31 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+
+from ingestify.domain.models.event.domain_event import DomainEvent
+from ingestify.utils import utcnow
+
+from .dataset import Dataset
+
+
+@dataclass
+class DatasetCreated(DomainEvent):
+    dataset: Dataset
+
+    event_type: str = "dataset_created"
+    occurred_at: datetime = field(default_factory=utcnow)
+
+
+@dataclass
+class RevisionAdded(DomainEvent):
+    dataset: Dataset
+
+    event_type: str = "revision_added"
+    occurred_at: datetime = field(default_factory=utcnow)
+
+
+@dataclass
+class MetadataUpdated(DomainEvent):
+    dataset: Dataset
+
+    event_type: str = "metadata_updated"
+    occurred_at: datetime = field(default_factory=utcnow)

ingestify/domain/models/dataset/file.py

@@ -0,0 +1,146 @@
+import hashlib
+import mimetypes
+
+from dataclasses import dataclass
+from datetime import datetime
+from io import BytesIO, StringIO
+from pathlib import Path
+from typing import BinaryIO, Optional, Union, Callable
+
+from ingestify.utils import utcnow
+
+
+@dataclass
+class DraftFile:
+    created_at: datetime
+    modified_at: datetime
+    tag: str
+    size: int
+    content_type: Optional[str]
+
+    data_feed_key: str  # Example: 'events'
+    data_spec_version: str  # Example: 'v3'
+    data_serialization_format: str  # Example: 'json'
+
+    stream: BinaryIO
+
+    @classmethod
+    def from_input(
+        cls,
+        file_,
+        data_feed_key,
+        data_spec_version="v1",
+        data_serialization_format="txt",
+        modified_at=None,
+    ):
+        # Pass-through for these types
+        if isinstance(file_, DraftFile) or file_ is None:
+            return file_
+        elif isinstance(file_, str):
+            stream = BytesIO(file_.encode("utf-8"))
+        elif isinstance(file_, bytes):
+            stream = BytesIO(file_)
+        elif isinstance(file_, StringIO):
+            stream = BytesIO(file_.read().encode("utf-8"))
+        elif isinstance(file_, BytesIO):
+            stream = file_
+        else:
+            raise Exception(f"Not possible to create DraftFile from {type(file_)}")
+
+        data = stream.read()
+        size = len(data)
+        tag = hashlib.sha1(data).hexdigest()
+        stream.seek(0)
+
+        now = utcnow()
+
+        return DraftFile(
+            created_at=now,
+            modified_at=modified_at or now,
+            tag=tag,
+            size=size,
+            stream=stream,
+            content_type=None,
+            data_feed_key=data_feed_key,
+            data_spec_version=data_spec_version,
+            data_serialization_format=data_serialization_format,
+        )
+
+
+@dataclass
+class File:
+    file_id: str
+    created_at: datetime
+    modified_at: datetime
+    tag: str
+    size: int
+    content_type: Optional[str]
+
+    data_feed_key: str  # Example: 'events'
+    data_spec_version: str  # Example: 'v3'
+    data_serialization_format: str  # Example: 'json'
+
+    storage_size: int
+    storage_compression_method: Optional[str]  # Example: 'gzip'
+    storage_path: Path
+
+    # This can be used when a Version is squashed
+    revision_id: Optional[int] = None
+
+    @classmethod
+    def from_draft(
+        cls,
+        draft_file: DraftFile,
+        file_id: str,
+        storage_size: int,
+        storage_compression_method,
+        path: Path,
+    ) -> "File":
+        return cls(
+            file_id=file_id,
+            created_at=draft_file.created_at,
+            modified_at=draft_file.modified_at,
+            tag=draft_file.tag,
+            size=draft_file.size,
+            data_feed_key=draft_file.data_feed_key,
+            data_spec_version=draft_file.data_spec_version,
+            data_serialization_format=draft_file.data_serialization_format,
+            content_type=draft_file.content_type,
+            storage_size=storage_size,
+            storage_compression_method=storage_compression_method,
+            storage_path=path,
+        )
+
+
+@dataclass
+class LoadedFile:
+    # Unique key to identify this File within a Dataset
+    file_id: str
+    created_at: datetime
+    modified_at: datetime
+    tag: str
+    size: int
+    storage_size: int
+    content_type: Optional[str]
+
+    data_feed_key: str  # Example: 'events'
+    data_spec_version: str  # Example: 'v3'
+    data_serialization_format: Optional[str]  # Example: 'gzip'
+
+    storage_size: int
+    storage_compression_method: Optional[str]  # Example: 'gzip'
+    storage_path: Path
+
+    _stream: Union[BinaryIO, Callable[[], BinaryIO]]
+
+    # This can be used when a Revision is squashed
+    revision_id: Optional[int] = None
+
+    @property
+    def stream(self):
+        if callable(self._stream):
+            self._stream = self._stream(self)
+        return self._stream
+
+
+__all__ = ["File", "DraftFile", "LoadedFile"]

ingestify/domain/models/dataset/file_collection.py

@@ -0,0 +1,35 @@
+from typing import Optional
+
+from .file import LoadedFile
+
+
+class FileCollection(dict):
+    def __init__(self, seq, auto_rewind: bool = True, **kwargs):
+        super().__init__(seq, **kwargs)
+
+        self._auto_rewind = auto_rewind
+
+    def get_file(
+        self,
+        data_feed_key: Optional[str] = None,
+        data_spec_version: Optional[str] = None,
+        auto_rewind: Optional[bool] = None,
+    ) -> Optional[LoadedFile]:
+        if not data_feed_key and not data_spec_version:
+            raise ValueError(
+                "You have to specify `data_feed_key` or `data_spec_version`"
+            )
+
+        for file in self.values():
+            if (not data_feed_key or file.data_feed_key == data_feed_key) and (
+                not data_spec_version or file.data_spec_version == data_spec_version
+            ):
+                should_auto_rewind = auto_rewind
+                if should_auto_rewind is None:
+                    should_auto_rewind = self._auto_rewind
+
+                if should_auto_rewind and file.stream.tell() > 0:
+                    file.stream.seek(0)
+                return file
+
+        return None

ingestify/domain/models/dataset/file_repository.py

@@ -0,0 +1,59 @@
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import BinaryIO
+
+from ingestify.utils import ComponentFactory, ComponentRegistry
+
+from .dataset import Dataset
+
+file_repository_registry = ComponentRegistry()
+
+
+class FileRepository(ABC, metaclass=file_repository_registry.metaclass):
+    def __init__(self, url: str):
+        self.base_dir = Path(url.split("://")[1])
+
+    @abstractmethod
+    def save_content(
+        self,
+        bucket: str,
+        dataset: Dataset,
+        revision_id: int,
+        filename: str,
+        stream: BinaryIO,
+    ) -> Path:
+        pass
+
+    @abstractmethod
+    def load_content(
+        self, bucket: str, dataset: Dataset, revision_id: int, filename: str
+    ) -> BinaryIO:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def supports(cls, url: str) -> bool:
+        pass
+
+    def get_path(
+        self, bucket: str, dataset: Dataset, revision_id: int, filename: str
+    ) -> Path:
+        path = (
+            self.base_dir
+            / bucket
+            / f"provider={dataset.provider}"
+            / f"dataset_type={dataset.dataset_type}"
+            / str(dataset.identifier)
+            / str(revision_id)
+            / filename
+        )
+        return path
+
+    def get_relative_path(self, path: Path) -> Path:
+        """Return the relative path to the base of the repository"""
+        return path.relative_to(self.base_dir)
+
+
+file_repository_factory = ComponentFactory.build_factory(
+    FileRepository, file_repository_registry
+)

ingestify/domain/models/dataset/identifier.py

@@ -0,0 +1,24 @@
+from typing import TYPE_CHECKING
+
+from ingestify.utils import key_from_dict
+
+if TYPE_CHECKING:
+    from ingestify.domain import Selector
+
+
+class Identifier(dict):
+    @classmethod
+    def create_from_selector(cls, selector: "Selector", **kwargs):
+        identifier = cls(**selector.filtered_attributes)
+        identifier.update(kwargs)
+        return identifier
+
+    @property
+    def key(self):
+        return key_from_dict(self)
+
+    def __hash__(self):
+        return hash(self.key)
+
+    def __str__(self):
+        return "/".join([f"{k}={v}" for k, v in self.items()])

ingestify/domain/models/dataset/revision.py

@@ -0,0 +1,29 @@
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Dict, List
+
+from .file import File
+
+
+@dataclass
+class Revision:
+    revision_id: int
+    created_at: datetime
+    description: str
+    modified_files: List[File]
+    is_squashed: bool = False
+
+    @property
+    def modified_files_map(self) -> Dict[str, File]:
+        return {file.file_id: file for file in self.modified_files}
+
+    def is_changed(self, files: Dict[str, datetime]) -> bool:
+        modified_files_map = self.modified_files_map
+        for file_id, last_modified in files.items():
+            if file_id not in modified_files_map:
+                return True
+
+            if modified_files_map[file_id].modified_at < last_modified:
+                return True
+
+        return False

ingestify/domain/models/dataset/selector.py

@@ -0,0 +1,37 @@
+from ingestify.domain.models.data_spec_version_collection import (
+    DataSpecVersionCollection,
+)
+from ingestify.utils import AttributeBag
+
+
+class Selector(AttributeBag):
+    def __bool__(self):
+        return len(self.filtered_attributes) > 0
+
+    @classmethod
+    def build(cls, attributes, data_spec_versions: DataSpecVersionCollection):
+        if callable(attributes):
+            return cls(
+                _data_spec_versions=data_spec_versions.copy(), _matcher=attributes
+            )
+        else:
+            return cls(_data_spec_versions=data_spec_versions.copy(), **attributes)
+
+    @property
+    def is_dynamic(self):
+        return "_matcher" in self.attributes
+
+    def is_match(self, selector: dict):
+        return self._matcher(selector)
+
+    @property
+    def data_spec_versions(self):
+        return self._data_spec_versions
+
+    @property
+    def custom_attributes(self):
+        return {
+            k: v
+            for k, v in self.items()
+            if k not in ("_matcher", "_data_spec_versions")
+        }

ingestify/domain/models/event/__init__.py

@@ -0,0 +1,4 @@
+from .publisher import Publisher
+from .domain_event import DomainEvent
+from .subscriber import Subscriber
+from .event_bus import EventBus