ingestify 0.1.3__py3-none-any.whl → 0.3.0__py3-none-any.whl

ingestify/domain/models/data_spec_version_collection.py

@@ -16,6 +16,12 @@ class DataSpecVersionCollection(dict):
 
         return cls(items_)
 
+    def to_dict(self):
+        return {
+            data_feed_key: list(data_spec_versions)
+            for data_feed_key, data_spec_versions in self.items()
+        }
+
     def copy(self):
         return DataSpecVersionCollection(copy.deepcopy(self))
 

ingestify/domain/models/dataset/__init__.py

@@ -1,8 +1,8 @@
+from .file import DraftFile, File, LoadedFile
 from .collection import DatasetCollection
 from .dataset import Dataset
-from .dataset_repository import DatasetRepository, dataset_repository_factory
-from .file import DraftFile, File, LoadedFile
-from .file_repository import FileRepository, file_repository_factory
+from .dataset_repository import DatasetRepository
+from .file_repository import FileRepository
 from .file_collection import FileCollection
 from .identifier import Identifier
 from .selector import Selector
@@ -16,12 +16,10 @@ __all__ = [
     "Identifier",
     "DatasetCollection",
     "DatasetCreated",
-    "dataset_repository_factory",
     "File",
     "DraftFile",
     "LoadedFile",
     "DatasetRepository",
     "FileRepository",
-    "file_repository_factory",
     "FileCollection",
 ]

ingestify/domain/models/dataset/dataset.py

@@ -1,70 +1,52 @@
-from dataclasses import dataclass, field
 from datetime import datetime
 from enum import Enum
 from typing import List, Optional
+from pydantic import Field
 
 from ingestify.utils import utcnow
-
+from .dataset_state import DatasetState
 from .file import DraftFile
 from .identifier import Identifier
-from .revision import Revision
-
-
-class DatasetState(Enum):
-    SCHEDULED = "SCHEDULED"
-    PARTIAL = "PARTIAL"
-    COMPLETE = "COMPLETE"
-
-    @property
-    def is_complete(self):
-        return self == DatasetState.COMPLETE
+from .revision import Revision, RevisionSource, SourceType
+from ..base import BaseModel
 
-    def __str__(self):
-        return self.value
 
-
-@dataclass
-class Dataset:
+class Dataset(BaseModel):
     bucket: str  # This must be set by the DatasetRepository
-
     dataset_id: str
     name: str
     state: DatasetState
-
     dataset_type: str
     provider: str
-
     identifier: Identifier
     metadata: dict
-
     created_at: datetime
     updated_at: datetime
-
-    revisions: List[Revision] = field(default_factory=list)
+    revisions: List[Revision] = Field(default_factory=list)
 
     @property
     def is_complete(self):
         return self.state.is_complete
 
-    def next_revision_id(self):
+    def next_revision_id(self) -> int:
         return len(self.revisions)
 
     def add_revision(self, revision: Revision):
         self.revisions.append(revision)
         self.updated_at = utcnow()
 
-    def update_from_resource(self, dataset_resource) -> bool:
+    def update_metadata(self, name: str, metadata: dict, state: DatasetState) -> bool:
         changed = False
-        if self.name != dataset_resource.name:
-            self.name = dataset_resource.name
+        if self.name != name:
+            self.name = name
             changed = True
 
-        if self.metadata != dataset_resource.metadata:
-            self.metadata = dataset_resource.metadata
+        if self.metadata != metadata:
+            self.metadata = metadata
             changed = True
 
-        if self.state != dataset_resource.state:
-            self.state = dataset_resource.state
+        if self.state != state:
+            self.state = state
             changed = True
 
         if changed:
@@ -101,4 +83,5 @@ class Dataset:
                 description="Squashed revision",
                 is_squashed=True,
                 modified_files=list(files.values()),
+                source=RevisionSource(source_type=SourceType.SQUASHED, source_id=""),
             )

ingestify/domain/models/dataset/dataset_repository.py

@@ -1,16 +1,12 @@
 from abc import ABC, abstractmethod
 from typing import Optional, List, Union
 
-from ingestify.utils import ComponentFactory, ComponentRegistry
-
 from .collection import DatasetCollection
 from .dataset import Dataset
 from .selector import Selector
 
-dataset_repository_registry = ComponentRegistry()
-
 
-class DatasetRepository(ABC, metaclass=dataset_repository_registry.metaclass):
+class DatasetRepository(ABC):
     @abstractmethod
     def get_dataset_collection(
         self,
@@ -34,13 +30,3 @@ class DatasetRepository(ABC, metaclass=dataset_repository_registry.metaclass):
     @abstractmethod
     def next_identity(self):
         pass
-
-    @classmethod
-    @abstractmethod
-    def supports(cls, url: str) -> bool:
-        pass
-
-
-dataset_repository_factory = ComponentFactory.build_factory(
-    DatasetRepository, dataset_repository_registry
-)

ingestify/domain/models/dataset/dataset_state.py

@@ -0,0 +1,11 @@
+from enum import Enum
+
+
+class DatasetState(str, Enum):
+    SCHEDULED = "SCHEDULED"
+    PARTIAL = "PARTIAL"
+    COMPLETE = "COMPLETE"
+
+    @property
+    def is_complete(self):
+        return self == DatasetState.COMPLETE

ingestify/domain/models/dataset/events.py

@@ -1,31 +1,21 @@
-from dataclasses import dataclass, field
-from datetime import datetime
+from typing import ClassVar
 
-from ingestify.domain.models.event.domain_event import DomainEvent
-from ingestify.utils import utcnow
+from pydantic import BaseModel
 
+from ingestify.domain.models.event.domain_event import DomainEvent
 from .dataset import Dataset
 
 
-@dataclass
 class DatasetCreated(DomainEvent):
     dataset: Dataset
-
-    event_type: str = "dataset_created"
-    occurred_at: datetime = field(default_factory=utcnow)
+    event_type: ClassVar[str] = "dataset_created"
 
 
-@dataclass
 class RevisionAdded(DomainEvent):
     dataset: Dataset
+    event_type: ClassVar[str] = "revision_added"
 
-    event_type: str = "revision_added"
-    occurred_at: datetime = field(default_factory=utcnow)
 
-
-@dataclass
 class MetadataUpdated(DomainEvent):
     dataset: Dataset
-
-    event_type: str = "metadata_updated"
-    occurred_at: datetime = field(default_factory=utcnow)
+    event_type: ClassVar[str] = "metadata_updated"

ingestify/domain/models/dataset/file.py

@@ -1,37 +1,32 @@
-import hashlib
-import mimetypes
-
-from dataclasses import dataclass
 from datetime import datetime
-from io import BytesIO, StringIO
 from pathlib import Path
-from typing import BinaryIO, Optional, Union, Callable
+from typing import BinaryIO, Optional, Union, Callable, Awaitable
+from io import BytesIO, StringIO
+import hashlib
 
+from ingestify.domain.models.base import BaseModel
 from ingestify.utils import utcnow
 
 
-@dataclass
-class DraftFile:
+class DraftFile(BaseModel):
     created_at: datetime
     modified_at: datetime
     tag: str
     size: int
     content_type: Optional[str]
-
     data_feed_key: str  # Example: 'events'
     data_spec_version: str  # Example: 'v3'
     data_serialization_format: str  # Example: 'json'
-
-    stream: BinaryIO
+    stream: BytesIO
 
     @classmethod
     def from_input(
         cls,
         file_,
-        data_feed_key,
-        data_spec_version="v1",
-        data_serialization_format="txt",
-        modified_at=None,
+        data_feed_key: str,
+        data_spec_version: str = "v1",
+        data_serialization_format: str = "txt",
+        modified_at: Optional[datetime] = None,
     ):
         # Pass-through for these types
         if isinstance(file_, DraftFile) or file_ is None:
@@ -67,25 +62,20 @@ class DraftFile:
         )
 
 
-@dataclass
-class File:
+class File(BaseModel):
     file_id: str
     created_at: datetime
     modified_at: datetime
     tag: str
     size: int
     content_type: Optional[str]
-
     data_feed_key: str  # Example: 'events'
     data_spec_version: str  # Example: 'v3'
     data_serialization_format: str  # Example: 'json'
-
     storage_size: int
     storage_compression_method: Optional[str]  # Example: 'gzip'
     storage_path: Path
-
-    # This can be used when a Version is squashed
-    revision_id: Optional[int] = None
+    revision_id: Optional[int] = None  # This can be used when a Version is squashed
 
     @classmethod
     def from_draft(
@@ -93,7 +83,7 @@ class File:
         draft_file: DraftFile,
         file_id: str,
         storage_size: int,
-        storage_compression_method,
+        storage_compression_method: str,
         path: Path,
     ) -> "File":
         return cls(
@@ -112,8 +102,7 @@ class File:
         )
 
 
-@dataclass
-class LoadedFile:
+class LoadedFile(BaseModel):
     # Unique key to identify this File within a Dataset
     file_id: str
     created_at: datetime
@@ -122,24 +111,22 @@ class LoadedFile:
     size: int
     storage_size: int
     content_type: Optional[str]
-
     data_feed_key: str  # Example: 'events'
     data_spec_version: str  # Example: 'v3'
-    data_serialization_format: Optional[str]  # Example: 'gzip'
-
-    storage_size: int
+    data_serialization_format: Optional[str]  # Example: 'json'
     storage_compression_method: Optional[str]  # Example: 'gzip'
     storage_path: Path
+    _stream: Union[BinaryIO, Callable[[], Awaitable[BinaryIO]]]
+    revision_id: Optional[int] = None  # This can be used when a Revision is squashed
 
-    _stream: Union[BinaryIO, Callable[[], BinaryIO]]
-
-    # This can be used when a Revision is squashed
-    revision_id: Optional[int] = None
+    def load_stream(self):
+        if callable(self._stream):
+            self._stream = self._stream(self)
 
     @property
     def stream(self):
         if callable(self._stream):
-            self._stream = self._stream(self)
+            raise Exception("You should load the stream first using `load_stream`")
         return self._stream
 
 

ingestify/domain/models/dataset/file_collection.py

@@ -3,7 +3,7 @@ from typing import Optional
 from .file import LoadedFile
 
 
-class FileCollection(dict):
+class FileCollection(dict[str, LoadedFile]):
     def __init__(self, seq, auto_rewind: bool = True, **kwargs):
         super().__init__(seq, **kwargs)
 
@@ -28,6 +28,8 @@ class FileCollection(dict):
                 if should_auto_rewind is None:
                     should_auto_rewind = self._auto_rewind
 
+                file.load_stream()
+
                 if should_auto_rewind and file.stream.tell() > 0:
                     file.stream.seek(0)
                 return file

ingestify/domain/models/dataset/file_repository.py

@@ -2,16 +2,35 @@ from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import BinaryIO
 
-from ingestify.utils import ComponentFactory, ComponentRegistry
-
 from .dataset import Dataset
-
-file_repository_registry = ComponentRegistry()
+from ...services.identifier_key_transformer import IdentifierTransformer
 
 
-class FileRepository(ABC, metaclass=file_repository_registry.metaclass):
-    def __init__(self, url: str):
+class FileRepository(ABC):
+    def __init__(self, url: str, identifier_transformer: IdentifierTransformer):
         self.base_dir = Path(url.split("://")[1])
+        self.identifier_transformer = identifier_transformer
+
+    def get_write_path(
+        self, bucket: str, dataset: Dataset, revision_id: int, filename: str
+    ) -> Path:
+        # TODO: use the IdentifierKeyTransformer
+        identifier_path = self.identifier_transformer.to_path(
+            provider=dataset.provider,
+            dataset_type=dataset.dataset_type,
+            identifier=dataset.identifier,
+        )
+
+        path = (
+            self.base_dir
+            / bucket
+            / f"provider={dataset.provider}"
+            / f"dataset_type={dataset.dataset_type}"
+            / identifier_path
+            / str(revision_id)
+            / filename
+        )
+        return path
 
     @abstractmethod
     def save_content(
@@ -24,10 +43,11 @@ class FileRepository(ABC, metaclass=file_repository_registry.metaclass):
     ) -> Path:
         pass
 
+    def get_read_path(self, storage_path: str) -> Path:
+        return self.base_dir / storage_path
+
     @abstractmethod
-    def load_content(
-        self, bucket: str, dataset: Dataset, revision_id: int, filename: str
-    ) -> BinaryIO:
+    def load_content(self, storage_path: str) -> BinaryIO:
         pass
 
     @classmethod
@@ -35,25 +55,6 @@ class FileRepository(ABC, metaclass=file_repository_registry.metaclass):
     def supports(cls, url: str) -> bool:
         pass
 
-    def get_path(
-        self, bucket: str, dataset: Dataset, revision_id: int, filename: str
-    ) -> Path:
-        path = (
-            self.base_dir
-            / bucket
-            / f"provider={dataset.provider}"
-            / f"dataset_type={dataset.dataset_type}"
-            / str(dataset.identifier)
-            / str(revision_id)
-            / filename
-        )
-        return path
-
     def get_relative_path(self, path: Path) -> Path:
         """Return the relative path to the base of the repository"""
         return path.relative_to(self.base_dir)
-
-
-file_repository_factory = ComponentFactory.build_factory(
-    FileRepository, file_repository_registry
-)

ingestify/domain/models/dataset/revision.py

@@ -1,17 +1,40 @@
-from dataclasses import dataclass
 from datetime import datetime
+from enum import Enum
 from typing import Dict, List
 
+from typing_extensions import TypedDict
+
 from .file import File
+from ..base import BaseModel
+
+
+class SourceType(str, Enum):
+    TASK = "TASK"
+    MANUAL = "MANUAL"
+    SQUASHED = "SQUASHED"
+
+
+class RevisionSource(TypedDict):
+    source_type: SourceType
+    source_id: str
+
+
+class RevisionState(str, Enum):
+    PENDING_VALIDATION = "PENDING_VALIDATION"
+    VALIDATING = "VALIDATING"
+    VALIDATION_FAILED = "VALIDATION_FAILED"
+    APPROVED = "APPROVED"
+    REJECTED = "REJECTED"
 
 
-@dataclass
-class Revision:
+class Revision(BaseModel):
     revision_id: int
     created_at: datetime
     description: str
     modified_files: List[File]
+    source: RevisionSource
     is_squashed: bool = False
+    state: RevisionState = RevisionState.PENDING_VALIDATION
 
     @property
     def modified_files_map(self) -> Dict[str, File]:

ingestify/domain/models/event/domain_event.py

@@ -1,9 +1,13 @@
-from abc import abstractmethod, ABC
-from dataclasses import dataclass
+from abc import ABC, abstractmethod
+from datetime import datetime
+from pydantic import BaseModel, Field
 
+from ingestify.utils import utcnow
+
+
+class DomainEvent(BaseModel, ABC):
+    occurred_at: datetime = Field(default_factory=utcnow)
 
-@dataclass
-class DomainEvent(ABC):
     @property
     @abstractmethod
     def event_type(self) -> str: