ingestify 0.9.2__py3-none-any.whl → 0.9.4__py3-none-any.whl

ingestify/__init__.py

@@ -9,4 +9,4 @@ if not __INGESTIFY_SETUP__:
     from .source_base import Source, DatasetResource
     from .main import debug_source
 
-__version__ = "0.9.2"
+__version__ = "0.9.4"

ingestify/domain/models/ingestion/ingestion_job.py

@@ -137,7 +137,7 @@ class UpdateDatasetTask(Task):
                 for file_id, file_resource in self.dataset_resource.files.items()
             }
 
-            self.dataset_resource.run_post_load_files(files)
+            self.dataset_resource.run_post_load_files(files, self.dataset)
 
             try:
                 revision = self.store.update_dataset(

ingestify/domain/models/resources/dataset_resource.py

@@ -3,6 +3,7 @@ from typing import Optional, Callable, Any, Protocol, TYPE_CHECKING, Dict  # noq
 from pydantic import Field
 
 from ingestify.domain.models.base import BaseModel
+from ingestify.domain.models.dataset import Dataset
 from ingestify.domain.models.dataset.dataset_state import DatasetState
 from ingestify.exceptions import DuplicateFile
 
@@ -51,17 +52,19 @@ class DatasetResource(BaseModel):
     state: DatasetState = Field(default_factory=lambda: DatasetState.COMPLETE)
     files: dict[str, FileResource] = Field(default_factory=dict)
     post_load_files: Optional[
-        Callable[["DatasetResource", Dict[str, DraftFile]], None]
+        Callable[["DatasetResource", Dict[str, DraftFile], Optional[Dataset]], None]
     ] = None
 
-    def run_post_load_files(self, files: Dict[str, DraftFile]):
+    def run_post_load_files(
+        self, files: Dict[str, DraftFile], existing_dataset: Optional[Dataset] = None
+    ):
         """Hook to modify dataset attributes based on loaded file content.
 
         Useful for setting state based on file content, e.g., keep state=SCHEDULED
         when files contain '{}', change to COMPLETE when they contain actual data.
         """
         if self.post_load_files:
-            self.post_load_files(self, files)
+            self.post_load_files(self, files, existing_dataset)
 
     def add_file(
         self,

ingestify/infra/store/dataset/sqlalchemy/repository.py

@@ -470,9 +470,14 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
                 dataset_ids = [row.dataset_id for row in dataset_query]
                 datasets = self._load_datasets(dataset_ids)
 
+                last_modified_values = [
+                    dataset.last_modified_at
+                    for dataset in datasets
+                    if dataset.last_modified_at is not None
+                ]
                 dataset_collection_metadata = DatasetCollectionMetadata(
-                    last_modified=max(dataset.last_modified_at for dataset in datasets)
-                    if datasets
+                    last_modified=max(last_modified_values)
+                    if last_modified_values
                     else None,
                     row_count=len(datasets),
                 )

ingestify/tests/test_engine.py

@@ -460,7 +460,9 @@ def test_dev_engine():
     assert datasets.first().name == "Test Dataset"
 
 
-def post_load_hook(dataset_resource: DatasetResource, files: dict[str, DraftFile]):
+def post_load_hook(
+    dataset_resource: DatasetResource, files: dict[str, DraftFile], existing_dataset
+):
     # Change state to COMPLETE if file content is not '{}'
     for file in files.values():
         if file.size > 2:
@@ -526,12 +528,32 @@ def test_post_load_files_hook(config_file):
 def test_force_save_creates_revision(config_file):
     """Test that datasets get a revision even when no files are persisted."""
     engine = get_engine(config_file, "main")
+
+    # Create one dataset with files and one without
+    add_ingestion_plan(
+        engine, SimpleFakeSource("fake-source"), competition_id=1, season_id=2
+    )
     add_ingestion_plan(
-        engine, NoFilesSource("fake-source"), competition_id=1, season_id=2
+        engine, NoFilesSource("fake-source"), competition_id=1, season_id=3
     )
 
     engine.load()
-    dataset = engine.store.get_dataset_collection().first()
 
-    assert len(dataset.revisions) == 1
-    assert len(dataset.current_revision.modified_files) == 0
+    # This should not fail even though one dataset has no last_modified_at
+    datasets = engine.store.get_dataset_collection()
+    assert len(datasets) == 2
+
+    # Verify the dataset without files still has a revision
+    dataset_without_files = engine.store.get_dataset_collection(season_id=3).first()
+    assert len(dataset_without_files.revisions) == 1
+    assert len(dataset_without_files.current_revision.modified_files) == 0
+
+    dataset_with_last_modified = engine.store.get_dataset_collection(
+        season_id=2
+    ).first()
+
+    dataset_without_files = engine.store.get_dataset_collection(metadata_only=True)
+    assert (
+        dataset_without_files.metadata.last_modified
+        == dataset_with_last_modified.last_modified_at
+    )

{ingestify-0.9.2.dist-info → ingestify-0.9.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestify
-Version: 0.9.2
+Version: 0.9.4
 Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl

{ingestify-0.9.2.dist-info → ingestify-0.9.4.dist-info}/RECORD

@@ -1,4 +1,4 @@
-ingestify/__init__.py,sha256=zFlvArmYGeebwjI5szpq8FLAYq8kpDb0ri9uh-5Yjrs,336
+ingestify/__init__.py,sha256=LJSJBoIR12MN0q1qS4q-tYq9_yc0n67c-hKzVgYyuUo,336
 ingestify/cmdline.py,sha256=Rs1_lSKSIJrcygH5fvtOGicOl_e0sZYW7deqp4_jGbY,6233
 ingestify/exceptions.py,sha256=izRzaLQmMy-4P8ZqGqVZyf4k6LFYOYqwYLuRaUH8BJw,187
 ingestify/main.py,sha256=mMXDNzSl1dzN03BUiS97uP3XwFMdgadxP0hJlONsZ_g,15789
@@ -39,11 +39,11 @@ ingestify/domain/models/event/event_bus.py,sha256=feVXsbBcRNkbWYvXbmz-Yi9-3R690y
 ingestify/domain/models/event/publisher.py,sha256=TOAawYYiPQCLR2Gm17LumMEzeapMDYcAYeklLFmwqAY,620
 ingestify/domain/models/event/subscriber.py,sha256=tP1ZFSvpJWKUITnATYekRxJzepz85UY7egBTMiP-dwg,1039
 ingestify/domain/models/ingestion/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ingestify/domain/models/ingestion/ingestion_job.py,sha256=Ou8v_FXDNnbrzPjHYiLoXMEg7ZRNFPjK1BMk9DY7L2E,15574
+ingestify/domain/models/ingestion/ingestion_job.py,sha256=3Icbgte5QDbho7bk7mPATsflWAtL95wOWa0WKtd8hEE,15588
 ingestify/domain/models/ingestion/ingestion_job_summary.py,sha256=7dmkWEvE7lMSv1ILWcSvys1bUGuGe_s-YbOFC6eYMBI,4794
 ingestify/domain/models/ingestion/ingestion_plan.py,sha256=KAvITBMQt3zmMFokESQJyp3rMuz1Hxr6msfZK1_faZM,648
 ingestify/domain/models/resources/__init__.py,sha256=ZuY9DPRfwk-aLB3Lj6DYP_NqMkcQfcYjZp4VejTtcbU,46
-ingestify/domain/models/resources/dataset_resource.py,sha256=zhTCM4bX6Wf4iWG2g8_SBx2U05YFxTxiZK3f6EqwD6I,3598
+ingestify/domain/models/resources/dataset_resource.py,sha256=UJYd3UVY-NP2PF0oYAiGp1aOuyxKfLJZKJE5Lxd44yY,3745
 ingestify/domain/models/task/__init__.py,sha256=BdlyIPvE07Xax_IzLgO9DUw0wsz9OZutxnxdDNyRlys,79
 ingestify/domain/models/task/set.py,sha256=04txDYgS5rotXofD9TqChKdW0VZIYshrkfPIpXtlhW4,430
 ingestify/domain/models/task/task.py,sha256=OwLZQi9GGe0O8m1dKvJdN2Rham5oilI49KyKc5uV20A,161
@@ -66,7 +66,7 @@ ingestify/infra/source/statsbomb/match.py,sha256=8Zpdys6-bB_ral2AmjGKhF4BnXW3F0Y
 ingestify/infra/store/__init__.py,sha256=3dA6NWfB6FS5SFdQiSlJ0ZghBfnUAUuGIP5Vr4rkCqk,43
 ingestify/infra/store/dataset/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/infra/store/dataset/sqlalchemy/__init__.py,sha256=Z5JHWGO_hwT6rO-ecMOOAmOKjFFJi449KZvJTQgt6vQ,52
-ingestify/infra/store/dataset/sqlalchemy/repository.py,sha256=v-JrX09zqheUklevXryev2_H1m0flYxJQFQBkZhVS24,24255
+ingestify/infra/store/dataset/sqlalchemy/repository.py,sha256=2z_a1ICIokx5yUGz1Jn-DBvFzK1pryAjPYLQTP3pxAw,24447
 ingestify/infra/store/dataset/sqlalchemy/tables.py,sha256=1FewCsN7jdk1ITzL_neOwJWtHD03NxCS9E8dhZcz4oY,12236
 ingestify/infra/store/file/__init__.py,sha256=DuEekZa2pmDuRCFiulbgoGotN0wGv3OrRXSvokY0PhY,104
 ingestify/infra/store/file/dummy_file_repository.py,sha256=azUq9c43Mz9-GWk9j0E97BaqyUKu-ZMrcuaIednLq5E,723
@@ -75,14 +75,14 @@ ingestify/infra/store/file/s3_file_repository.py,sha256=tz_EZ_gun7W2qJMlI3j_R03i
 ingestify/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/tests/conftest.py,sha256=Cr768nLMWUfIP6FMR7aFhUCY4uQ9Tz_bXOq0udpiDEE,411
 ingestify/tests/test_auto_ingest.py,sha256=coMOzJBTbeDwUSYDLnqmkaCXpG-6WQeKqf_nj9XFiA4,14502
-ingestify/tests/test_engine.py,sha256=ooqy4EjDSBFo3aHIlnqjokj9r8PnNcYxzl0rhPc0fVo,16783
+ingestify/tests/test_engine.py,sha256=iOjAoPvJSdR1WCbNjTSTnzFMVdusi_MZWN6InUqi9Ew,17558
 ingestify/tests/test_events.py,sha256=A1f8H4HRyn52SWo3wV_MgSeb6IbT_lNi9wWAK8EGsK4,7806
 ingestify/tests/test_file_cache.py,sha256=Xbh_VLLDH-KQXE3MeujDeOjjYYbAnjGR6wsHwMInKco,3049
 ingestify/tests/test_pagination.py,sha256=uAKDMsM6fYSa4NcAlXDllu2y-8lnh0AclhPZ5MWJKn8,5539
 ingestify/tests/test_store_version.py,sha256=4czUG8LtaGxgjW4trw7BzYJA8blQp3-HM8w-7HjqFl0,2508
 ingestify/tests/test_table_prefix.py,sha256=6N42T6hfulqTlsUlrwhNmZ-TK-ZOt4U8Jx9NxKyLS4I,2844
-ingestify-0.9.2.dist-info/METADATA,sha256=wbLlX2aaVowBJRKKOvz817ipB48gZmMeE48EcJgMycs,8263
-ingestify-0.9.2.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
-ingestify-0.9.2.dist-info/entry_points.txt,sha256=czYYXeX2ul4zdeB6bKlz3HaUF7zyVVcj9E_sRNDisI0,53
-ingestify-0.9.2.dist-info/top_level.txt,sha256=Lwnjgns4KequS7KiicXhh6mLUvcdfjzLyPI4qf_s4A0,10
-ingestify-0.9.2.dist-info/RECORD,,
+ingestify-0.9.4.dist-info/METADATA,sha256=sIvu3pk8p4F2cpNDHA7RQV8fmRfr01Ifc_DsIcM3PL4,8263
+ingestify-0.9.4.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+ingestify-0.9.4.dist-info/entry_points.txt,sha256=czYYXeX2ul4zdeB6bKlz3HaUF7zyVVcj9E_sRNDisI0,53
+ingestify-0.9.4.dist-info/top_level.txt,sha256=Lwnjgns4KequS7KiicXhh6mLUvcdfjzLyPI4qf_s4A0,10
+ingestify-0.9.4.dist-info/RECORD,,