ingestify 0.6.1__tar.gz → 0.6.2__tar.gz

{ingestify-0.6.1 → ingestify-0.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.6.1
+Version: 0.6.2
 Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl

{ingestify-0.6.1 → ingestify-0.6.2}/ingestify/__init__.py

@@ -8,4 +8,4 @@ if not __INGESTIFY_SETUP__:
     from .infra import retrieve_http
     from .source_base import Source, DatasetResource
 
-__version__ = "0.6.1"
+__version__ = "0.6.2"

{ingestify-0.6.1 → ingestify-0.6.2}/ingestify/application/dataset_store.py

@@ -11,6 +11,7 @@ from typing import Dict, List, Optional, Union, Callable, BinaryIO, Awaitable
 
 from ingestify.domain.models.dataset.dataset import DatasetState
 from ingestify.domain.models.dataset.events import RevisionAdded, MetadataUpdated
+from ingestify.domain.models.dataset.file import NotModifiedFile
 from ingestify.domain.models.dataset.file_collection import FileCollection
 from ingestify.domain.models.dataset.revision import RevisionSource
 from ingestify.domain.models.event import EventBus
@@ -140,8 +141,8 @@ class DatasetStore:
         current_revision = dataset.current_revision
 
         for file_id, file_ in modified_files.items():
-            if file_ is None:
-                # It's always allowed to pass None as file. This means it didn't change and must be ignored.
+            if isinstance(file_, NotModifiedFile):
+                # It's always allowed to pass NotModifiedFile as file. This means it didn't change and must be ignored.
                 continue
 
             current_file = (
@@ -210,9 +211,22 @@ class DatasetStore:
                 f"Added a new revision to {dataset.identifier} -> {', '.join([file.file_id for file in persisted_files_])}"
             )
         else:
-            logger.info(
-                f"Ignoring a new revision without changed files -> {dataset.identifier}"
-            )
+            if dataset.update_last_modified(files):
+                # For some Datasets the last modified doesn't make sense (for sources that don't provide it)
+                # Do we want to update last modified of a Dataset when the value is utcnow()?
+                # self.dataset_repository.save(bucket=self.bucket, dataset=dataset)
+                # TODO: dispatch some event?
+                # self.dispatch(DatasetLastModifiedChanged(dataset=dataset))
+                logger.info(
+                    f"Ignoring a new revision without changed files -> {dataset.identifier}, but "
+                    f"might need to update last modified to {dataset.last_modified_at} ?"
+                )
+
+            else:
+                logger.info(
+                    f"Ignoring a new revision without changed files -> {dataset.identifier}"
+                )
+
             revision = None
 
         return revision

{ingestify-0.6.1 → ingestify-0.6.2}/ingestify/domain/models/dataset/collection_metadata.py

@@ -11,4 +11,6 @@ class DatasetCollectionMetadata:
 
     # Use the last modified to only retrieve datasets that are changed
     last_modified: Optional[datetime]
-    row_count: int
+
+    # Not really used
+    row_count: Optional[int] = None

{ingestify-0.6.1 → ingestify-0.6.2}/ingestify/domain/models/dataset/dataset.py

@@ -1,6 +1,6 @@
 from datetime import datetime
 from enum import Enum
-from typing import List, Optional
+from typing import List, Optional, Dict
 from pydantic import Field, field_validator
 
 from ingestify.utils import utcnow
@@ -52,6 +52,22 @@ class Dataset(BaseModel):
         else:
             self.last_modified_at = revision.last_modified_at
 
+    def update_last_modified(self, files: Dict[str, DraftFile]):
+        """Update the last modified, even tho there was no new revision. Some Sources
+        may report a Dataset is changed, even when there are no changed files.
+        Update the last_modified to prevent hitting the same Source for updates
+        """
+        changed = False
+        for file in files.values():
+            if file.modified_at and (
+                self.last_modified_at is None
+                or file.modified_at > self.last_modified_at
+            ):
+                # Update, and continue looking for others
+                self.last_modified_at = file.modified_at
+                changed = True
+        return changed
+
     def update_metadata(self, name: str, metadata: dict, state: DatasetState) -> bool:
         changed = False
         if self.name != name:

{ingestify-0.6.1 → ingestify-0.6.2}/ingestify/domain/models/dataset/file.py

@@ -29,7 +29,7 @@ class DraftFile(BaseModel):
         modified_at: Optional[datetime] = None,
     ):
         # Pass-through for these types
-        if isinstance(file_, DraftFile) or file_ is None:
+        if isinstance(file_, (DraftFile, NotModifiedFile)):
             return file_
         elif isinstance(file_, str):
             stream = BytesIO(file_.encode("utf-8"))
@@ -102,6 +102,11 @@ class File(BaseModel):
         )
 
 
+class NotModifiedFile(BaseModel):
+    modified_at: datetime
+    reason: str
+
+
 class LoadedFile(BaseModel):
     # Unique key to identify this File within a Dataset
     file_id: str

{ingestify-0.6.1 → ingestify-0.6.2}/ingestify/domain/models/dataset/revision.py

@@ -44,13 +44,22 @@ class Revision(BaseModel):
     def modified_files_map(self) -> Dict[str, File]:
         return {file.file_id: file for file in self.modified_files}
 
-    def is_changed(self, files: Dict[str, datetime]) -> bool:
+    def is_changed(
+        self, files: Dict[str, datetime], dataset_last_modified_at: datetime
+    ) -> bool:
         modified_files_map = self.modified_files_map
         for file_id, last_modified in files.items():
             if file_id not in modified_files_map:
                 return True
 
             if modified_files_map[file_id].modified_at < last_modified:
-                return True
+                if dataset_last_modified_at < last_modified:
+                    # For StatsBomb we use last_modified of match for lineups, and events files.
+                    # When only match is updated, the lineups and events files won't be updated
+                    # as the content is not changed. Therefore, those modified_at is not updated,
+                    # and we try to update it over and over again.
+                    # This check prevents that; always take the LastModifiedAt of the Dataset
+                    # into account
+                    return True
 
         return False

{ingestify-0.6.1 → ingestify-0.6.2}/ingestify/domain/models/dataset/selector.py

@@ -1,3 +1,6 @@
+from datetime import datetime
+from typing import Optional
+
 from ingestify.domain.models.data_spec_version_collection import (
     DataSpecVersionCollection,
 )
@@ -28,10 +31,17 @@ class Selector(AttributeBag):
     def data_spec_versions(self):
         return self._data_spec_versions
 
+    @property
+    def last_modified(self) -> Optional[datetime]:
+        try:
+            return self._last_modified
+        except AttributeError:
+            return None
+
     @property
     def custom_attributes(self):
         return {
             k: v
             for k, v in self.items()
-            if k not in ("_matcher", "_data_spec_versions")
+            if k not in ("_matcher", "_data_spec_versions", "_last_modified")
         }

{ingestify-0.6.1 → ingestify-0.6.2}/ingestify/domain/models/fetch_policy.py

@@ -26,7 +26,9 @@ class FetchPolicy:
                 file.file_id: file.last_modified
                 for file in dataset_resource.files.values()
             }
-            if current_revision.is_changed(files_last_modified):
+            if current_revision.is_changed(
+                files_last_modified, dataset.last_modified_at
+            ):
                 return True
 
             # We don't set last_modified on Dataset level anymore, only on file level

{ingestify-0.6.1 → ingestify-0.6.2}/ingestify/domain/models/ingestion/ingestion_job.py

@@ -3,11 +3,12 @@ import json
 import logging
 import uuid
 from enum import Enum
-from typing import Optional, Iterator
+from typing import Optional, Iterator, Union
 
 from ingestify import retrieve_http
 from ingestify.application.dataset_store import DatasetStore
 from ingestify.domain import Selector, Identifier, TaskSet, Dataset, DraftFile, Task
+from ingestify.domain.models.dataset.file import NotModifiedFile
 from ingestify.domain.models.dataset.revision import RevisionSource, SourceType
 from ingestify.domain.models.ingestion.ingestion_job_summary import (
     IngestionJobSummary,
@@ -54,7 +55,7 @@ def to_batches(input_):
 
 def load_file(
     file_resource: FileResource, dataset: Optional[Dataset] = None
-) -> Optional[DraftFile]:
+) -> Union[DraftFile, NotModifiedFile]:
     current_file = None
     if dataset:
         current_file = dataset.current_revision.modified_files_map.get(
@@ -72,7 +73,10 @@ def load_file(
         )
         if current_file and current_file.tag == file.tag:
             # Nothing changed
-            return None
+            return NotModifiedFile(
+                modified_at=file_resource.last_modified,
+                reason="tag matched current_file",
+            )
         return file
     elif file_resource.url:
         http_options = {}
@@ -228,6 +232,19 @@ class IngestionJob:
             ).metadata
         logger.info(f"Done: {dataset_collection_metadata}")
 
+        if self.selector.last_modified and dataset_collection_metadata.last_modified:
+            # This check might fail when the data_spec_versions is changed;
+            # missing files are not detected
+            if self.selector.last_modified < dataset_collection_metadata.last_modified:
+                logger.info(
+                    f"Skipping find_datasets because selector last_modified "
+                    f"'{self.selector.last_modified}' < metadata last_modified "
+                    f"'{dataset_collection_metadata.last_modified}'"
+                )
+                ingestion_job_summary.set_skipped()
+                yield ingestion_job_summary
+                return
+
         # There are two different, but similar flows here:
         # 1. The discover_datasets returns a list, and the entire list can be processed at once
         # 2. The discover_datasets returns an iterator of batches, in this case we need to process each batch
@@ -252,8 +269,6 @@ class IngestionJob:
 
         logger.info("Starting tasks")
 
-        finish_task_timer = ingestion_job_summary.start_timing("tasks")
-
         while True:
             logger.info(f"Finding next batch of datasets for selector={self.selector}")
 
@@ -266,7 +281,6 @@ class IngestionJob:
             except Exception as e:
                 logger.exception("Failed to fetch next batch")
 
-                finish_task_timer()
                 ingestion_job_summary.set_exception(e)
                 yield ingestion_job_summary
                 return
@@ -294,54 +308,57 @@ class IngestionJob:
             skipped_tasks = 0
 
             task_set = TaskSet()
-            for dataset_resource in batch:
-                dataset_identifier = Identifier.create_from_selector(
-                    self.selector, **dataset_resource.dataset_resource_id
-                )
 
-                if dataset := dataset_collection.get(dataset_identifier):
-                    if self.ingestion_plan.fetch_policy.should_refetch(
-                        dataset, dataset_resource
-                    ):
-                        task_set.add(
-                            UpdateDatasetTask(
-                                dataset=dataset,  # Current dataset from the database
-                                dataset_resource=dataset_resource,  # Most recent dataset_resource
-                                store=store,
+            with ingestion_job_summary.record_timing("build_task_set"):
+                for dataset_resource in batch:
+                    dataset_identifier = Identifier.create_from_selector(
+                        self.selector, **dataset_resource.dataset_resource_id
+                    )
+
+                    if dataset := dataset_collection.get(dataset_identifier):
+                        if self.ingestion_plan.fetch_policy.should_refetch(
+                            dataset, dataset_resource
+                        ):
+                            task_set.add(
+                                UpdateDatasetTask(
+                                    dataset=dataset,  # Current dataset from the database
+                                    dataset_resource=dataset_resource,  # Most recent dataset_resource
+                                    store=store,
+                                )
                             )
-                        )
+                        else:
+                            skipped_tasks += 1
                     else:
-                        skipped_tasks += 1
-                else:
-                    if self.ingestion_plan.fetch_policy.should_fetch(dataset_resource):
-                        task_set.add(
-                            CreateDatasetTask(
-                                dataset_resource=dataset_resource,
-                                store=store,
+                        if self.ingestion_plan.fetch_policy.should_fetch(
+                            dataset_resource
+                        ):
+                            task_set.add(
+                                CreateDatasetTask(
+                                    dataset_resource=dataset_resource,
+                                    store=store,
+                                )
                             )
-                        )
-                    else:
-                        skipped_tasks += 1
-
-            if task_set:
-                logger.info(
-                    f"Discovered {len(dataset_identifiers)} datasets from {self.ingestion_plan.source.__class__.__name__} "
-                    f"using selector {self.selector} => {len(task_set)} tasks. {skipped_tasks} skipped."
-                )
-                logger.info(f"Running {len(task_set)} tasks")
-                ingestion_job_summary.add_task_summaries(
-                    task_executor.run(run_task, task_set)
-                )
-            else:
-                logger.info(
-                    f"Discovered {len(dataset_identifiers)} datasets from {self.ingestion_plan.source.__class__.__name__} "
-                    f"using selector {self.selector} => nothing to do"
-                )
-
-            ingestion_job_summary.increase_skipped_tasks(skipped_tasks)
+                        else:
+                            skipped_tasks += 1
+
+            with ingestion_job_summary.record_timing("tasks"):
+                if task_set:
+                    logger.info(
+                        f"Discovered {len(dataset_identifiers)} datasets from {self.ingestion_plan.source.__class__.__name__} "
+                        f"using selector {self.selector} => {len(task_set)} tasks. {skipped_tasks} skipped."
+                    )
+                    logger.info(f"Running {len(task_set)} tasks")
+                    ingestion_job_summary.add_task_summaries(
+                        task_executor.run(run_task, task_set)
+                    )
+                else:
+                    logger.info(
+                        f"Discovered {len(dataset_identifiers)} datasets from {self.ingestion_plan.source.__class__.__name__} "
+                        f"using selector {self.selector} => nothing to do"
+                    )
+                ingestion_job_summary.increase_skipped_tasks(skipped_tasks)
 
             if ingestion_job_summary.task_count() >= MAX_TASKS_PER_CHUNK:
-                finish_task_timer()
                 ingestion_job_summary.set_finished()
                 yield ingestion_job_summary
 
@@ -349,11 +366,7 @@ class IngestionJob:
                 is_first_chunk = False
                 ingestion_job_summary = IngestionJobSummary.new(ingestion_job=self)
 
-                # We will resume tasks, start timer right away
-                finish_task_timer = ingestion_job_summary.start_timing("tasks")
-
         if ingestion_job_summary.task_count() > 0 or is_first_chunk:
             # When there is interesting information to store, or there was no data at all, store it
-            finish_task_timer()
             ingestion_job_summary.set_finished()
             yield ingestion_job_summary

{ingestify-0.6.1 → ingestify-0.6.2}/ingestify/domain/models/ingestion/ingestion_job_summary.py

@@ -18,6 +18,7 @@ if TYPE_CHECKING:
 class IngestionJobState(str, Enum):
     RUNNING = "RUNNING"
     FINISHED = "FINISHED"
+    SKIPPED = "SKIPPED"
     FAILED = "FAILED"
 
 
@@ -104,6 +105,10 @@ class IngestionJobSummary(BaseModel, HasTiming):
         self.state = IngestionJobState.FAILED
         self._set_ended()
 
+    def set_skipped(self):
+        self.state = IngestionJobState.SKIPPED
+        self._set_ended()
+
     @property
     def duration(self) -> timedelta:
         return self.ended_at - self.started_at

{ingestify-0.6.1 → ingestify-0.6.2}/ingestify/infra/fetch/http.py

@@ -3,13 +3,14 @@ from datetime import datetime
 from email.utils import format_datetime, parsedate
 from hashlib import sha1
 from io import BytesIO
-from typing import Optional, Callable, Tuple
+from typing import Optional, Callable, Tuple, Union
 
 import requests
 from requests.adapters import HTTPAdapter
 from urllib3 import Retry
 
 from ingestify.domain.models import DraftFile, File
+from ingestify.domain.models.dataset.file import NotModifiedFile
 from ingestify.utils import utcnow
 
 _session = None
@@ -46,12 +47,15 @@ def retrieve_http(
     pager: Optional[Tuple[str, Callable[[str, dict], Optional[str]]]] = None,
     last_modified: Optional[datetime] = None,
     **kwargs,
-) -> Optional[DraftFile]:
+) -> Union[DraftFile, NotModifiedFile]:
     headers = headers or {}
     if current_file:
         if last_modified and current_file.modified_at >= last_modified:
             # Not changed
-            return None
+            return NotModifiedFile(
+                modified_at=last_modified,
+                reason=f"last-modified same as current file: {current_file.modified_at} >= {last_modified}",
+            )
         # else:
         #     print(f"{current_file.modified_at=} {last_modified=}")
         # headers["if-modified-since"] = (
@@ -73,12 +77,14 @@ def retrieve_http(
 
     response = get_session().get(url, headers=headers, **http_kwargs)
     if response.status_code == 404 and ignore_not_found:
-        return None
+        return NotModifiedFile(
+            modified_at=last_modified, reason="404 http code and ignore-not-found"
+        )
 
     response.raise_for_status()
     if response.status_code == 304:
         # Not modified
-        return None
+        return NotModifiedFile(modified_at=last_modified, reason="304 http code")
 
     if last_modified:
         # From metadata received from api in discover_datasets
@@ -120,7 +126,9 @@ def retrieve_http(
 
     if current_file and current_file.tag == tag:
         # Not changed. Don't keep it
-        return None
+        return NotModifiedFile(
+            modified_at=last_modified, reason="tag matched current_file"
+        )
 
     return DraftFile(
         created_at=utcnow(),

{ingestify-0.6.1 → ingestify-0.6.2}/ingestify/infra/store/dataset/sqlalchemy/repository.py

@@ -413,20 +413,25 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
             else:
                 datasets = []
 
-                metadata_result_query = apply_query_filter(
-                    self.session.query(
-                        func.max(dataset_table.c.last_modified_at).label(
-                            "last_modified_at"
-                        ),
-                        func.count().label("row_count"),
+                metadata_result_query = (
+                    apply_query_filter(
+                        self.session.query(dataset_table.c.last_modified_at)
                     )
+                    .order_by(dataset_table.c.last_modified_at.desc())
+                    .limit(1)
                 )
 
                 self._debug_query(metadata_result_query)
 
-                dataset_collection_metadata = DatasetCollectionMetadata(
-                    *metadata_result_query.first()
-                )
+                metadata_row = metadata_result_query.first()
+                if metadata_row:
+                    dataset_collection_metadata = DatasetCollectionMetadata(
+                        last_modified=metadata_row.last_modified_at
+                    )
+                else:
+                    dataset_collection_metadata = DatasetCollectionMetadata(
+                        last_modified=None
+                    )
 
         return DatasetCollection(dataset_collection_metadata, datasets)
 

{ingestify-0.6.1 → ingestify-0.6.2}/ingestify.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.6.1
+Version: 0.6.2
 Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl