ingestify 0.9.3__py3-none-any.whl → 0.9.4__py3-none-any.whl

ingestify/__init__.py

@@ -9,4 +9,4 @@ if not __INGESTIFY_SETUP__:
     from .source_base import Source, DatasetResource
     from .main import debug_source
 
-__version__ = "0.9.3"
+__version__ = "0.9.4"

ingestify/domain/models/ingestion/ingestion_job.py

@@ -137,7 +137,7 @@ class UpdateDatasetTask(Task):
                 for file_id, file_resource in self.dataset_resource.files.items()
             }
 
-            self.dataset_resource.run_post_load_files(files)
+            self.dataset_resource.run_post_load_files(files, self.dataset)
 
             try:
                 revision = self.store.update_dataset(

ingestify/domain/models/resources/dataset_resource.py

@@ -3,6 +3,7 @@ from typing import Optional, Callable, Any, Protocol, TYPE_CHECKING, Dict  # noq
 from pydantic import Field
 
 from ingestify.domain.models.base import BaseModel
+from ingestify.domain.models.dataset import Dataset
 from ingestify.domain.models.dataset.dataset_state import DatasetState
 from ingestify.exceptions import DuplicateFile
 
@@ -51,17 +52,19 @@ class DatasetResource(BaseModel):
     state: DatasetState = Field(default_factory=lambda: DatasetState.COMPLETE)
     files: dict[str, FileResource] = Field(default_factory=dict)
     post_load_files: Optional[
-        Callable[["DatasetResource", Dict[str, DraftFile]], None]
+        Callable[["DatasetResource", Dict[str, DraftFile], Optional[Dataset]], None]
     ] = None
 
-    def run_post_load_files(self, files: Dict[str, DraftFile]):
+    def run_post_load_files(
+        self, files: Dict[str, DraftFile], existing_dataset: Optional[Dataset] = None
+    ):
         """Hook to modify dataset attributes based on loaded file content.
 
         Useful for setting state based on file content, e.g., keep state=SCHEDULED
         when files contain '{}', change to COMPLETE when they contain actual data.
         """
         if self.post_load_files:
-            self.post_load_files(self, files)
+            self.post_load_files(self, files, existing_dataset)
 
     def add_file(
         self,

ingestify/tests/test_engine.py

@@ -460,7 +460,9 @@ def test_dev_engine():
     assert datasets.first().name == "Test Dataset"
 
 
-def post_load_hook(dataset_resource: DatasetResource, files: dict[str, DraftFile]):
+def post_load_hook(
+    dataset_resource: DatasetResource, files: dict[str, DraftFile], existing_dataset
+):
     # Change state to COMPLETE if file content is not '{}'
     for file in files.values():
         if file.size > 2:
@@ -545,3 +547,13 @@ def test_force_save_creates_revision(config_file):
     dataset_without_files = engine.store.get_dataset_collection(season_id=3).first()
     assert len(dataset_without_files.revisions) == 1
     assert len(dataset_without_files.current_revision.modified_files) == 0
+
+    dataset_with_last_modified = engine.store.get_dataset_collection(
+        season_id=2
+    ).first()
+
+    dataset_without_files = engine.store.get_dataset_collection(metadata_only=True)
+    assert (
+        dataset_without_files.metadata.last_modified
+        == dataset_with_last_modified.last_modified_at
+    )

{ingestify-0.9.3.dist-info → ingestify-0.9.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestify
-Version: 0.9.3
+Version: 0.9.4
 Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl

{ingestify-0.9.3.dist-info → ingestify-0.9.4.dist-info}/RECORD

@@ -1,4 +1,4 @@
-ingestify/__init__.py,sha256=fKX1lkxZhmjQ4aSvq_0_PN3XTXBwPP4jVJE-Ad2x8LM,336
+ingestify/__init__.py,sha256=LJSJBoIR12MN0q1qS4q-tYq9_yc0n67c-hKzVgYyuUo,336
 ingestify/cmdline.py,sha256=Rs1_lSKSIJrcygH5fvtOGicOl_e0sZYW7deqp4_jGbY,6233
 ingestify/exceptions.py,sha256=izRzaLQmMy-4P8ZqGqVZyf4k6LFYOYqwYLuRaUH8BJw,187
 ingestify/main.py,sha256=mMXDNzSl1dzN03BUiS97uP3XwFMdgadxP0hJlONsZ_g,15789
@@ -39,11 +39,11 @@ ingestify/domain/models/event/event_bus.py,sha256=feVXsbBcRNkbWYvXbmz-Yi9-3R690y
 ingestify/domain/models/event/publisher.py,sha256=TOAawYYiPQCLR2Gm17LumMEzeapMDYcAYeklLFmwqAY,620
 ingestify/domain/models/event/subscriber.py,sha256=tP1ZFSvpJWKUITnATYekRxJzepz85UY7egBTMiP-dwg,1039
 ingestify/domain/models/ingestion/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ingestify/domain/models/ingestion/ingestion_job.py,sha256=Ou8v_FXDNnbrzPjHYiLoXMEg7ZRNFPjK1BMk9DY7L2E,15574
+ingestify/domain/models/ingestion/ingestion_job.py,sha256=3Icbgte5QDbho7bk7mPATsflWAtL95wOWa0WKtd8hEE,15588
 ingestify/domain/models/ingestion/ingestion_job_summary.py,sha256=7dmkWEvE7lMSv1ILWcSvys1bUGuGe_s-YbOFC6eYMBI,4794
 ingestify/domain/models/ingestion/ingestion_plan.py,sha256=KAvITBMQt3zmMFokESQJyp3rMuz1Hxr6msfZK1_faZM,648
 ingestify/domain/models/resources/__init__.py,sha256=ZuY9DPRfwk-aLB3Lj6DYP_NqMkcQfcYjZp4VejTtcbU,46
-ingestify/domain/models/resources/dataset_resource.py,sha256=zhTCM4bX6Wf4iWG2g8_SBx2U05YFxTxiZK3f6EqwD6I,3598
+ingestify/domain/models/resources/dataset_resource.py,sha256=UJYd3UVY-NP2PF0oYAiGp1aOuyxKfLJZKJE5Lxd44yY,3745
 ingestify/domain/models/task/__init__.py,sha256=BdlyIPvE07Xax_IzLgO9DUw0wsz9OZutxnxdDNyRlys,79
 ingestify/domain/models/task/set.py,sha256=04txDYgS5rotXofD9TqChKdW0VZIYshrkfPIpXtlhW4,430
 ingestify/domain/models/task/task.py,sha256=OwLZQi9GGe0O8m1dKvJdN2Rham5oilI49KyKc5uV20A,161
@@ -75,14 +75,14 @@ ingestify/infra/store/file/s3_file_repository.py,sha256=tz_EZ_gun7W2qJMlI3j_R03i
 ingestify/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/tests/conftest.py,sha256=Cr768nLMWUfIP6FMR7aFhUCY4uQ9Tz_bXOq0udpiDEE,411
 ingestify/tests/test_auto_ingest.py,sha256=coMOzJBTbeDwUSYDLnqmkaCXpG-6WQeKqf_nj9XFiA4,14502
-ingestify/tests/test_engine.py,sha256=5EKBTZLms38hjkgDjmRH_ATiZI6FWou4Bt5IJIsE7os,17217
+ingestify/tests/test_engine.py,sha256=iOjAoPvJSdR1WCbNjTSTnzFMVdusi_MZWN6InUqi9Ew,17558
 ingestify/tests/test_events.py,sha256=A1f8H4HRyn52SWo3wV_MgSeb6IbT_lNi9wWAK8EGsK4,7806
 ingestify/tests/test_file_cache.py,sha256=Xbh_VLLDH-KQXE3MeujDeOjjYYbAnjGR6wsHwMInKco,3049
 ingestify/tests/test_pagination.py,sha256=uAKDMsM6fYSa4NcAlXDllu2y-8lnh0AclhPZ5MWJKn8,5539
 ingestify/tests/test_store_version.py,sha256=4czUG8LtaGxgjW4trw7BzYJA8blQp3-HM8w-7HjqFl0,2508
 ingestify/tests/test_table_prefix.py,sha256=6N42T6hfulqTlsUlrwhNmZ-TK-ZOt4U8Jx9NxKyLS4I,2844
-ingestify-0.9.3.dist-info/METADATA,sha256=XKCP3xKunZ9ENLwv8Bn3q8K8yDEZQEI6E0ro--N95xE,8263
-ingestify-0.9.3.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
-ingestify-0.9.3.dist-info/entry_points.txt,sha256=czYYXeX2ul4zdeB6bKlz3HaUF7zyVVcj9E_sRNDisI0,53
-ingestify-0.9.3.dist-info/top_level.txt,sha256=Lwnjgns4KequS7KiicXhh6mLUvcdfjzLyPI4qf_s4A0,10
-ingestify-0.9.3.dist-info/RECORD,,
+ingestify-0.9.4.dist-info/METADATA,sha256=sIvu3pk8p4F2cpNDHA7RQV8fmRfr01Ifc_DsIcM3PL4,8263
+ingestify-0.9.4.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+ingestify-0.9.4.dist-info/entry_points.txt,sha256=czYYXeX2ul4zdeB6bKlz3HaUF7zyVVcj9E_sRNDisI0,53
+ingestify-0.9.4.dist-info/top_level.txt,sha256=Lwnjgns4KequS7KiicXhh6mLUvcdfjzLyPI4qf_s4A0,10
+ingestify-0.9.4.dist-info/RECORD,,