ingestify 0.9.1__py3-none-any.whl → 0.9.2__py3-none-any.whl

ingestify/__init__.py

@@ -9,4 +9,4 @@ if not __INGESTIFY_SETUP__:
     from .source_base import Source, DatasetResource
     from .main import debug_source
 
-__version__ = "0.9.1"
+__version__ = "0.9.2"

ingestify/application/dataset_store.py

@@ -383,6 +383,7 @@ class DatasetStore:
         files: Dict[str, DraftFile],
         revision_source: RevisionSource,
         description: str = "Update",
+        force_save: bool = False,
     ):
         """
         Create new revision first, so FileRepository can use
@@ -392,7 +393,7 @@ class DatasetStore:
         created_at = utcnow()
 
         persisted_files_ = self._persist_files(dataset, revision_id, files)
-        if persisted_files_:
+        if persisted_files_ or force_save:
             # It can happen an API tells us data is changed, but it was not changed. In this case
             # we decide to ignore it.
             # Make sure there are files changed before creating a new revision
@@ -487,7 +488,9 @@ class DatasetStore:
             updated_at=now,
             last_modified_at=None,  # Not known at this moment
         )
-        revision = self.add_revision(dataset, files, revision_source, description)
+        revision = self.add_revision(
+            dataset, files, revision_source, description, force_save=True
+        )
 
         self.dispatch(DatasetCreated(dataset=dataset))
         return revision

ingestify/domain/models/dataset/revision.py

@@ -38,7 +38,9 @@ class Revision(BaseModel):
 
     @property
     def last_modified_at(self):
-        return max(file.modified_at for file in self.modified_files)
+        if self.modified_files:
+            return max(file.modified_at for file in self.modified_files)
+        return None
 
     @property
     def modified_files_map(self) -> Dict[str, File]:

ingestify/domain/models/task/task_summary.py

@@ -86,9 +86,7 @@ class TaskSummary(BaseModel, HasTiming):
         if revision:
             self.persisted_file_count = len(revision.modified_files)
             self.bytes_retrieved = sum(file.size for file in revision.modified_files)
-            self.last_modified = max(
-                file.modified_at for file in revision.modified_files
-            )
+            self.last_modified = revision.last_modified_at
         else:
             self.state = TaskState.FINISHED_IGNORED
 

ingestify/infra/fetch/http.py

@@ -58,7 +58,9 @@ def retrieve_http(
             )
         # else:
         #     print(f"{current_file.modified_at=} {last_modified=}")
-        headers["if-modified-since"] = format_datetime(current_file.modified_at, usegmt=True)
+        headers["if-modified-since"] = format_datetime(
+            current_file.modified_at, usegmt=True
+        )
         headers["if-none-match"] = current_file.tag
 
     http_kwargs = {}

ingestify/infra/store/dataset/sqlalchemy/repository.py

@@ -375,7 +375,9 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
                     dataset_ids_cte.c.dataset_id == self.revision_table.c.dataset_id,
                 )
             )
-            .order_by(self.revision_table.c.dataset_id)
+            .order_by(
+                self.revision_table.c.dataset_id, self.revision_table.c.revision_id
+            )
         )
 
         for dataset_id, revisions in itertools.groupby(
@@ -560,22 +562,22 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
             try:
                 # Delete modified files related to the dataset
                 connection.execute(
-                    file_table.delete().where(
-                        file_table.c.dataset_id == dataset.dataset_id
+                    self.file_table.delete().where(
+                        self.file_table.c.dataset_id == dataset.dataset_id
                     )
                 )
 
                 # Delete revisions related to the dataset
                 connection.execute(
-                    revision_table.delete().where(
-                        revision_table.c.dataset_id == dataset.dataset_id
+                    self.revision_table.delete().where(
+                        self.revision_table.c.dataset_id == dataset.dataset_id
                     )
                 )
 
                 # Delete the dataset itself
                 connection.execute(
-                    dataset_table.delete().where(
-                        dataset_table.c.dataset_id == dataset.dataset_id
+                    self.dataset_table.delete().where(
+                        self.dataset_table.c.dataset_id == dataset.dataset_id
                     )
                 )
 

ingestify/tests/test_engine.py

@@ -251,6 +251,28 @@ class FailingJobSource(Source):
         raise Exception("some failure")
 
 
+class NoFilesSource(Source):
+    provider = "fake"
+
+    def find_datasets(
+        self,
+        dataset_type: str,
+        data_spec_versions: DataSpecVersionCollection,
+        dataset_collection_metadata: DatasetCollectionMetadata,
+        competition_id,
+        season_id,
+        **kwargs,
+    ):
+        yield DatasetResource(
+            dataset_resource_id=dict(
+                competition_id=competition_id, season_id=season_id, match_id=1
+            ),
+            provider="fake",
+            dataset_type="match",
+            name="Dataset Without Files",
+        )
+
+
 def test_engine(config_file):
     engine = get_engine(config_file, "main")
 
@@ -499,3 +521,17 @@ def test_post_load_files_hook(config_file):
     engine.load()
     dataset2 = engine.store.get_dataset_collection().first()
     assert dataset2.state == DatasetState.COMPLETE
+
+
+def test_force_save_creates_revision(config_file):
+    """Test that datasets get a revision even when no files are persisted."""
+    engine = get_engine(config_file, "main")
+    add_ingestion_plan(
+        engine, NoFilesSource("fake-source"), competition_id=1, season_id=2
+    )
+
+    engine.load()
+    dataset = engine.store.get_dataset_collection().first()
+
+    assert len(dataset.revisions) == 1
+    assert len(dataset.current_revision.modified_files) == 0

{ingestify-0.9.1.dist-info → ingestify-0.9.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestify
-Version: 0.9.1
+Version: 0.9.2
 Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl

{ingestify-0.9.1.dist-info → ingestify-0.9.2.dist-info}/RECORD

@@ -1,4 +1,4 @@
-ingestify/__init__.py,sha256=MZ_6jH21WxPiBmvxbqdaDhr-gC_pq7zbVhZkjpk31v8,336
+ingestify/__init__.py,sha256=zFlvArmYGeebwjI5szpq8FLAYq8kpDb0ri9uh-5Yjrs,336
 ingestify/cmdline.py,sha256=Rs1_lSKSIJrcygH5fvtOGicOl_e0sZYW7deqp4_jGbY,6233
 ingestify/exceptions.py,sha256=izRzaLQmMy-4P8ZqGqVZyf4k6LFYOYqwYLuRaUH8BJw,187
 ingestify/main.py,sha256=mMXDNzSl1dzN03BUiS97uP3XwFMdgadxP0hJlONsZ_g,15789
@@ -6,7 +6,7 @@ ingestify/server.py,sha256=OVrf_XtpAQIn88MzqQzShXgsA9_jbnqYvD8YPBjn3cs,2413
 ingestify/source_base.py,sha256=GXAFCoT11Zov9M2v-fqQr9gFCXbtVfEIEH32V7r2oE8,382
 ingestify/utils.py,sha256=tsoo-GgeSrwK161WCqW793BAm5bjvnGwI8yGgLTJ1lk,6486
 ingestify/application/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ingestify/application/dataset_store.py,sha256=GP6wGjVirefEn6hlqWIkOBqdELad9L_mmTpdHdzj18M,20353
+ingestify/application/dataset_store.py,sha256=9JKx260vA07Pv9hpW_1R2S2DbXcMyQsirvDpPRakyWE,20440
 ingestify/application/ingestion_engine.py,sha256=we16yiDS9QGOlAUiP1vidDycihjWK3B2jo64uqKmrXE,11246
 ingestify/application/loader.py,sha256=K99ZJuHMEJFO6CIlxoyHKGSQtXw63JgOYu3moUD6sR0,13400
 ingestify/application/secrets_manager.py,sha256=5qCbPfUvRGP1Xbq6xPkMfpgYl8uPzF_0NbiKeRY5zxU,1757
@@ -29,7 +29,7 @@ ingestify/domain/models/dataset/file.py,sha256=cXDjSw19HRMCGFpVN4u1oejxE1V8SMQpt
 ingestify/domain/models/dataset/file_collection.py,sha256=yaQmqFlmbajLCkU5QnjgqCvKzvVEZJrXVvinx5UGHcM,1193
 ingestify/domain/models/dataset/file_repository.py,sha256=9EQprch9isAH2pbK7e7tfOKl6ulip4Ij1kBCTbO_rTc,1721
 ingestify/domain/models/dataset/identifier.py,sha256=EJYsxt0OS_43Y989DZQq8U9NjwmtvnHGYGMe6-hOBlI,575
-ingestify/domain/models/dataset/revision.py,sha256=e8-NsRS8AILrNjWqCxqANF55oY091CN3fBmIiVS9wz0,2049
+ingestify/domain/models/dataset/revision.py,sha256=KK-RPQgRVgNWJcjpfYPt9r6tk_0Z4FfT6oM2QuLWlIA,2105
 ingestify/domain/models/dataset/selector.py,sha256=qGRA22gDAHhjDAhMWzOjZPz3Rrs1V-DZ32z75NARoTQ,1448
 ingestify/domain/models/event/__init__.py,sha256=OdPTpE9bj5QqdGmrYqRTLPX1f-LR9GWJYlGMPPEsuL8,138
 ingestify/domain/models/event/_old_event.py,sha256=RktgCAj9SMdtqkAc_bOwoghEb2Z6m4r5_xWXin9wqx4,472
@@ -47,14 +47,14 @@ ingestify/domain/models/resources/dataset_resource.py,sha256=zhTCM4bX6Wf4iWG2g8_
 ingestify/domain/models/task/__init__.py,sha256=BdlyIPvE07Xax_IzLgO9DUw0wsz9OZutxnxdDNyRlys,79
 ingestify/domain/models/task/set.py,sha256=04txDYgS5rotXofD9TqChKdW0VZIYshrkfPIpXtlhW4,430
 ingestify/domain/models/task/task.py,sha256=OwLZQi9GGe0O8m1dKvJdN2Rham5oilI49KyKc5uV20A,161
-ingestify/domain/models/task/task_summary.py,sha256=T9BSGhOZjKCPfym34VUdBXuMy0o6E832GAI4WMtjGao,3181
+ingestify/domain/models/task/task_summary.py,sha256=T6fvAxsxq861gKj6DFbo6WMt7GJtg_fhQ5pbujqCskk,3119
 ingestify/domain/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/domain/services/identifier_key_transformer.py,sha256=y4GS9u9Ej1MO2jUhAxWbifp0mrE_MqTHvVVcoQzSKb4,4034
 ingestify/domain/services/transformers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/domain/services/transformers/kloppy_to_pandas.py,sha256=NcN6nTBGVn9gz-_hWZJTMcduS1Gg7EM4X95Cqxi1QIM,809
 ingestify/infra/__init__.py,sha256=V0hpLzPVTcOHRVh0gguF6FT30YIgEOUd5v87xUHkfZ4,88
 ingestify/infra/fetch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ingestify/infra/fetch/http.py,sha256=M7DQgr67sqYDcltZMDQb0IH6USSGrjvECIjL-mDP8L8,4622
+ingestify/infra/fetch/http.py,sha256=_ZuRsVplpya8GdhNl7SqVBXOtCnAU3V5Umt0vtaxGQk,4644
 ingestify/infra/serialization/__init__.py,sha256=UqXWJmKTp7Mi58ZyDASGguPFlqdVWVUbm_sg9GWx9eI,702
 ingestify/infra/sink/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/infra/sink/postgresql.py,sha256=SxuM3LntfYcpCriUpqJhMvgAf0s9cohXf6WkxSEDYDY,1816
@@ -66,7 +66,7 @@ ingestify/infra/source/statsbomb/match.py,sha256=8Zpdys6-bB_ral2AmjGKhF4BnXW3F0Y
 ingestify/infra/store/__init__.py,sha256=3dA6NWfB6FS5SFdQiSlJ0ZghBfnUAUuGIP5Vr4rkCqk,43
 ingestify/infra/store/dataset/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/infra/store/dataset/sqlalchemy/__init__.py,sha256=Z5JHWGO_hwT6rO-ecMOOAmOKjFFJi449KZvJTQgt6vQ,52
-ingestify/infra/store/dataset/sqlalchemy/repository.py,sha256=6WPHKxA6UhgzO3P4Sdbq7W14tTMxisC8Js3MZilPoNc,24160
+ingestify/infra/store/dataset/sqlalchemy/repository.py,sha256=v-JrX09zqheUklevXryev2_H1m0flYxJQFQBkZhVS24,24255
 ingestify/infra/store/dataset/sqlalchemy/tables.py,sha256=1FewCsN7jdk1ITzL_neOwJWtHD03NxCS9E8dhZcz4oY,12236
 ingestify/infra/store/file/__init__.py,sha256=DuEekZa2pmDuRCFiulbgoGotN0wGv3OrRXSvokY0PhY,104
 ingestify/infra/store/file/dummy_file_repository.py,sha256=azUq9c43Mz9-GWk9j0E97BaqyUKu-ZMrcuaIednLq5E,723
@@ -75,14 +75,14 @@ ingestify/infra/store/file/s3_file_repository.py,sha256=tz_EZ_gun7W2qJMlI3j_R03i
 ingestify/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/tests/conftest.py,sha256=Cr768nLMWUfIP6FMR7aFhUCY4uQ9Tz_bXOq0udpiDEE,411
 ingestify/tests/test_auto_ingest.py,sha256=coMOzJBTbeDwUSYDLnqmkaCXpG-6WQeKqf_nj9XFiA4,14502
-ingestify/tests/test_engine.py,sha256=x3_o6M3satos029Er84ptbzxRKoBw5KB0am2KJSQ16Q,15738
+ingestify/tests/test_engine.py,sha256=ooqy4EjDSBFo3aHIlnqjokj9r8PnNcYxzl0rhPc0fVo,16783
 ingestify/tests/test_events.py,sha256=A1f8H4HRyn52SWo3wV_MgSeb6IbT_lNi9wWAK8EGsK4,7806
 ingestify/tests/test_file_cache.py,sha256=Xbh_VLLDH-KQXE3MeujDeOjjYYbAnjGR6wsHwMInKco,3049
 ingestify/tests/test_pagination.py,sha256=uAKDMsM6fYSa4NcAlXDllu2y-8lnh0AclhPZ5MWJKn8,5539
 ingestify/tests/test_store_version.py,sha256=4czUG8LtaGxgjW4trw7BzYJA8blQp3-HM8w-7HjqFl0,2508
 ingestify/tests/test_table_prefix.py,sha256=6N42T6hfulqTlsUlrwhNmZ-TK-ZOt4U8Jx9NxKyLS4I,2844
-ingestify-0.9.1.dist-info/METADATA,sha256=Ov_CZkPYjC8mWH5ePR7MsEhlRrrIcCXtoDGRHLrToto,8263
-ingestify-0.9.1.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
-ingestify-0.9.1.dist-info/entry_points.txt,sha256=czYYXeX2ul4zdeB6bKlz3HaUF7zyVVcj9E_sRNDisI0,53
-ingestify-0.9.1.dist-info/top_level.txt,sha256=Lwnjgns4KequS7KiicXhh6mLUvcdfjzLyPI4qf_s4A0,10
-ingestify-0.9.1.dist-info/RECORD,,
+ingestify-0.9.2.dist-info/METADATA,sha256=wbLlX2aaVowBJRKKOvz817ipB48gZmMeE48EcJgMycs,8263
+ingestify-0.9.2.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+ingestify-0.9.2.dist-info/entry_points.txt,sha256=czYYXeX2ul4zdeB6bKlz3HaUF7zyVVcj9E_sRNDisI0,53
+ingestify-0.9.2.dist-info/top_level.txt,sha256=Lwnjgns4KequS7KiicXhh6mLUvcdfjzLyPI4qf_s4A0,10
+ingestify-0.9.2.dist-info/RECORD,,