ingestify 0.9.1__tar.gz → 0.9.3__tar.gz

{ingestify-0.9.1 → ingestify-0.9.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestify
-Version: 0.9.1
+Version: 0.9.3
 Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl

{ingestify-0.9.1 → ingestify-0.9.3}/ingestify/__init__.py

@@ -9,4 +9,4 @@ if not __INGESTIFY_SETUP__:
     from .source_base import Source, DatasetResource
     from .main import debug_source
 
-__version__ = "0.9.1"
+__version__ = "0.9.3"

{ingestify-0.9.1 → ingestify-0.9.3}/ingestify/application/dataset_store.py

@@ -383,6 +383,7 @@ class DatasetStore:
         files: Dict[str, DraftFile],
         revision_source: RevisionSource,
         description: str = "Update",
+        force_save: bool = False,
     ):
         """
         Create new revision first, so FileRepository can use
@@ -392,7 +393,7 @@ class DatasetStore:
         created_at = utcnow()
 
         persisted_files_ = self._persist_files(dataset, revision_id, files)
-        if persisted_files_:
+        if persisted_files_ or force_save:
             # It can happen an API tells us data is changed, but it was not changed. In this case
             # we decide to ignore it.
             # Make sure there are files changed before creating a new revision
@@ -487,7 +488,9 @@ class DatasetStore:
             updated_at=now,
             last_modified_at=None,  # Not known at this moment
         )
-        revision = self.add_revision(dataset, files, revision_source, description)
+        revision = self.add_revision(
+            dataset, files, revision_source, description, force_save=True
+        )
 
         self.dispatch(DatasetCreated(dataset=dataset))
         return revision

{ingestify-0.9.1 → ingestify-0.9.3}/ingestify/domain/models/dataset/revision.py

@@ -38,7 +38,9 @@ class Revision(BaseModel):
 
     @property
     def last_modified_at(self):
-        return max(file.modified_at for file in self.modified_files)
+        if self.modified_files:
+            return max(file.modified_at for file in self.modified_files)
+        return None
 
     @property
     def modified_files_map(self) -> Dict[str, File]:

{ingestify-0.9.1 → ingestify-0.9.3}/ingestify/domain/models/task/task_summary.py

@@ -86,9 +86,7 @@ class TaskSummary(BaseModel, HasTiming):
         if revision:
             self.persisted_file_count = len(revision.modified_files)
             self.bytes_retrieved = sum(file.size for file in revision.modified_files)
-            self.last_modified = max(
-                file.modified_at for file in revision.modified_files
-            )
+            self.last_modified = revision.last_modified_at
         else:
             self.state = TaskState.FINISHED_IGNORED
 

{ingestify-0.9.1 → ingestify-0.9.3}/ingestify/infra/fetch/http.py

@@ -58,7 +58,9 @@ def retrieve_http(
             )
         # else:
         #     print(f"{current_file.modified_at=} {last_modified=}")
-        headers["if-modified-since"] = format_datetime(current_file.modified_at, usegmt=True)
+        headers["if-modified-since"] = format_datetime(
+            current_file.modified_at, usegmt=True
+        )
         headers["if-none-match"] = current_file.tag
 
     http_kwargs = {}

{ingestify-0.9.1 → ingestify-0.9.3}/ingestify/infra/store/dataset/sqlalchemy/repository.py

@@ -375,7 +375,9 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
                     dataset_ids_cte.c.dataset_id == self.revision_table.c.dataset_id,
                 )
             )
-            .order_by(self.revision_table.c.dataset_id)
+            .order_by(
+                self.revision_table.c.dataset_id, self.revision_table.c.revision_id
+            )
         )
 
         for dataset_id, revisions in itertools.groupby(
@@ -468,9 +470,14 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
                 dataset_ids = [row.dataset_id for row in dataset_query]
                 datasets = self._load_datasets(dataset_ids)
 
+                last_modified_values = [
+                    dataset.last_modified_at
+                    for dataset in datasets
+                    if dataset.last_modified_at is not None
+                ]
                 dataset_collection_metadata = DatasetCollectionMetadata(
-                    last_modified=max(dataset.last_modified_at for dataset in datasets)
-                    if datasets
+                    last_modified=max(last_modified_values)
+                    if last_modified_values
                     else None,
                     row_count=len(datasets),
                 )
@@ -560,22 +567,22 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
             try:
                 # Delete modified files related to the dataset
                 connection.execute(
-                    file_table.delete().where(
-                        file_table.c.dataset_id == dataset.dataset_id
+                    self.file_table.delete().where(
+                        self.file_table.c.dataset_id == dataset.dataset_id
                     )
                 )
 
                 # Delete revisions related to the dataset
                 connection.execute(
-                    revision_table.delete().where(
-                        revision_table.c.dataset_id == dataset.dataset_id
+                    self.revision_table.delete().where(
+                        self.revision_table.c.dataset_id == dataset.dataset_id
                     )
                 )
 
                 # Delete the dataset itself
                 connection.execute(
-                    dataset_table.delete().where(
-                        dataset_table.c.dataset_id == dataset.dataset_id
+                    self.dataset_table.delete().where(
+                        self.dataset_table.c.dataset_id == dataset.dataset_id
                     )
                 )
 

{ingestify-0.9.1 → ingestify-0.9.3}/ingestify/tests/test_engine.py

@@ -251,6 +251,28 @@ class FailingJobSource(Source):
         raise Exception("some failure")
 
 
+class NoFilesSource(Source):
+    provider = "fake"
+
+    def find_datasets(
+        self,
+        dataset_type: str,
+        data_spec_versions: DataSpecVersionCollection,
+        dataset_collection_metadata: DatasetCollectionMetadata,
+        competition_id,
+        season_id,
+        **kwargs,
+    ):
+        yield DatasetResource(
+            dataset_resource_id=dict(
+                competition_id=competition_id, season_id=season_id, match_id=1
+            ),
+            provider="fake",
+            dataset_type="match",
+            name="Dataset Without Files",
+        )
+
+
 def test_engine(config_file):
     engine = get_engine(config_file, "main")
 
@@ -499,3 +521,27 @@ def test_post_load_files_hook(config_file):
     engine.load()
     dataset2 = engine.store.get_dataset_collection().first()
     assert dataset2.state == DatasetState.COMPLETE
+
+
+def test_force_save_creates_revision(config_file):
+    """Test that datasets get a revision even when no files are persisted."""
+    engine = get_engine(config_file, "main")
+
+    # Create one dataset with files and one without
+    add_ingestion_plan(
+        engine, SimpleFakeSource("fake-source"), competition_id=1, season_id=2
+    )
+    add_ingestion_plan(
+        engine, NoFilesSource("fake-source"), competition_id=1, season_id=3
+    )
+
+    engine.load()
+
+    # This should not fail even though one dataset has no last_modified_at
+    datasets = engine.store.get_dataset_collection()
+    assert len(datasets) == 2
+
+    # Verify the dataset without files still has a revision
+    dataset_without_files = engine.store.get_dataset_collection(season_id=3).first()
+    assert len(dataset_without_files.revisions) == 1
+    assert len(dataset_without_files.current_revision.modified_files) == 0

{ingestify-0.9.1 → ingestify-0.9.3}/ingestify.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestify
-Version: 0.9.1
+Version: 0.9.3
 Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl