ingestify 0.3.0__tar.gz → 0.3.2__tar.gz

{ingestify-0.3.0 → ingestify-0.3.2}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.3.0
-Summary: Standardizing soccer tracking- and event data
+Version: 0.3.2
+Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl
 License: AGPL

{ingestify-0.3.0 → ingestify-0.3.2}/ingestify/__init__.py

@@ -8,4 +8,4 @@ if not __INGESTIFY_SETUP__:
     from .infra import retrieve_http
     from .source_base import Source, DatasetResource
 
-__version__ = "0.3.0"
+__version__ = "0.3.2"

{ingestify-0.3.0 → ingestify-0.3.2}/ingestify/application/dataset_store.py

@@ -294,9 +294,7 @@ class DatasetStore:
 
             def get_stream(file_):
                 return reader(
-                    self.file_repository.load_content(
-                        bucket=self.bucket, storage_path=file_.storage_path
-                    )
+                    self.file_repository.load_content(storage_path=file_.storage_path)
                 )
 
             loaded_file = LoadedFile(

{ingestify-0.3.0 → ingestify-0.3.2}/ingestify/domain/models/dataset/file.py

@@ -116,7 +116,7 @@ class LoadedFile(BaseModel):
     data_serialization_format: Optional[str]  # Example: 'json'
     storage_compression_method: Optional[str]  # Example: 'gzip'
     storage_path: Path
-    _stream: Union[BinaryIO, Callable[[], Awaitable[BinaryIO]]]
+    _stream: Union[BinaryIO, BytesIO, Callable[[], Awaitable[Union[BinaryIO, BytesIO]]]]
     revision_id: Optional[int] = None  # This can be used when a Revision is squashed
 
     def load_stream(self):

{ingestify-0.3.0 → ingestify-0.3.2}/ingestify/domain/models/ingestion/ingestion_job_summary.py

@@ -112,7 +112,7 @@ class IngestionJobSummary(BaseModel):
 
         print(f"    - Failed tasks: {self.failed_tasks}")
         print(f"    - Successful tasks: {self.successful_tasks}")
-        print(f"    - Successful ignored tasks: {self.successful_tasks}")
+        print(f"    - Successful ignored tasks: {self.ignored_successful_tasks}")
         print(f"    - Skipped datasets: {self.skipped_datasets}")
         print("--------------------")
 

ingestify-0.3.2/ingestify/infra/store/file/dummy_file_repository.py

@@ -0,0 +1,31 @@
+import logging
+from pathlib import Path
+from typing import BinaryIO
+
+from ingestify.domain.models import Dataset, FileRepository
+
+
+logger = logging.getLogger(__name__)
+
+
+class DummyFileRepository(FileRepository):
+    @classmethod
+    def supports(cls, url: str) -> bool:
+        return url.startswith("dummy://")
+
+    def save_content(
+        self,
+        bucket: str,
+        dataset: Dataset,
+        revision_id: int,
+        filename: str,
+        stream: BinaryIO,
+    ) -> Path:
+        path = self.get_write_path(bucket, dataset, revision_id, filename)
+
+        logger.info(f"Dummy save content to {path}")
+
+        return path
+
+    def load_content(self, storage_path: str) -> BinaryIO:
+        return BinaryIO()

{ingestify-0.3.0 → ingestify-0.3.2}/ingestify/main.py

@@ -26,6 +26,7 @@ from ingestify.infra.store.dataset.sqlalchemy import SqlAlchemyDatasetRepository
 from ingestify.infra.store.dataset.sqlalchemy.repository import (
     SqlAlchemySessionProvider,
 )
+from ingestify.infra.store.file.dummy_file_repository import DummyFileRepository
 
 logger = logging.getLogger(__name__)
 
@@ -70,6 +71,10 @@ def build_file_repository(file_url: str, identifier_transformer) -> FileReposito
         repository = LocalFileRepository(
             url=file_url, identifier_transformer=identifier_transformer
         )
+    elif file_url.startswith("dummy://"):
+        repository = DummyFileRepository(
+            url=file_url, identifier_transformer=identifier_transformer
+        )
     else:
         raise Exception(f"Cannot find repository to handle file {file_url}")
 

{ingestify-0.3.0 → ingestify-0.3.2}/ingestify.egg-info/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.3.0
-Summary: Standardizing soccer tracking- and event data
+Version: 0.3.2
+Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl
 License: AGPL

{ingestify-0.3.0 → ingestify-0.3.2}/ingestify.egg-info/SOURCES.txt

@@ -75,6 +75,7 @@ ingestify/infra/store/dataset/sqlalchemy/__init__.py
 ingestify/infra/store/dataset/sqlalchemy/mapping.py
 ingestify/infra/store/dataset/sqlalchemy/repository.py
 ingestify/infra/store/file/__init__.py
+ingestify/infra/store/file/dummy_file_repository.py
 ingestify/infra/store/file/local_file_repository.py
 ingestify/infra/store/file/s3_file_repository.py
 ingestify/static/templates/statsbomb_github/README.md

{ingestify-0.3.0 → ingestify-0.3.2}/setup.py

@@ -26,7 +26,7 @@ def setup_package():
         author="Koen Vossen",
         author_email="info@koenvossen.nl",
         license="AGPL",
-        description="Standardizing soccer tracking- and event data",
+        description="Data Ingestion Framework",
         long_description=readme,
         long_description_content_type="text/markdown",
         packages=setuptools.find_packages(exclude=["tests"]),