ingestify 0.3.4__tar.gz → 0.4.0__tar.gz

{ingestify-0.3.4 → ingestify-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.3.4
+Version: 0.4.0
 Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl

{ingestify-0.3.4 → ingestify-0.4.0}/ingestify/__init__.py

@@ -8,4 +8,4 @@ if not __INGESTIFY_SETUP__:
     from .infra import retrieve_http
     from .source_base import Source, DatasetResource
 
-__version__ = "0.3.4"
+__version__ = "0.4.0"

{ingestify-0.3.4 → ingestify-0.4.0}/ingestify/application/dataset_store.py

@@ -270,6 +270,7 @@ class DatasetStore:
             metadata=metadata,
             created_at=now,
             updated_at=now,
+            last_modified_at=None,  # Not known at this moment
         )
         revision = self.add_revision(dataset, files, revision_source, description)
 

{ingestify-0.3.4 → ingestify-0.4.0}/ingestify/application/ingestion_engine.py

@@ -21,8 +21,13 @@ class IngestionEngine:
     def add_ingestion_plan(self, ingestion_plan: IngestionPlan):
         self.loader.add_ingestion_plan(ingestion_plan)
 
-    def load(self, dry_run: bool = False, provider: Optional[str] = None):
-        self.loader.collect_and_run(dry_run=dry_run, provider=provider)
+    def load(
+        self,
+        dry_run: bool = False,
+        provider: Optional[str] = None,
+        source: Optional[str] = None,
+    ):
+        self.loader.collect_and_run(dry_run=dry_run, provider=provider, source=source)
 
     def list_datasets(self, as_count: bool = False):
         """Consider moving this to DataStore"""

{ingestify-0.3.4 → ingestify-0.4.0}/ingestify/application/loader.py

@@ -29,7 +29,12 @@ class Loader:
     def add_ingestion_plan(self, ingestion_plan: IngestionPlan):
         self.ingestion_plans.append(ingestion_plan)
 
-    def collect_and_run(self, dry_run: bool = False, provider: Optional[str] = None):
+    def collect_and_run(
+        self,
+        dry_run: bool = False,
+        provider: Optional[str] = None,
+        source: Optional[str] = None,
+    ):
         # First collect all selectors, before discovering datasets
         selectors = {}
         for ingestion_plan in self.ingestion_plans:
@@ -42,6 +47,13 @@ class Loader:
                     )
                     continue
 
+            if source is not None:
+                if ingestion_plan.source.name != source:
+                    logger.info(
+                        f"Skipping {ingestion_plan} because source doesn't match '{source}'"
+                    )
+                    continue
+
             static_selectors = [
                 selector
                 for selector in ingestion_plan.selectors
@@ -60,6 +72,7 @@ class Loader:
 
                     # TODO: consider making this lazy and fetch once per Source instead of
                     #       once per IngestionPlan
+                    # TODO: Log exception when `discover_selectors` fails
                     all_selectors = ingestion_plan.source.discover_selectors(
                         ingestion_plan.dataset_type
                     )

{ingestify-0.3.4 → ingestify-0.4.0}/ingestify/cmdline.py

@@ -58,7 +58,14 @@ def cli():
     help="bucket",
     type=str,
 )
-@click.option("--debug", "debug", required=False, help="Debugging enabled", type=bool)
+@click.option(
+    "--debug",
+    "debug",
+    required=False,
+    help="Debugging enabled",
+    is_flag=True,
+    type=bool,
+)
 @click.option(
     "--dry-run",
     "dry_run",
@@ -74,11 +81,19 @@ def cli():
     help="Provider - only run tasks for a single provider",
     type=str,
 )
+@click.option(
+    "--source",
+    "source",
+    required=False,
+    help="Source - only run tasks for a single source",
+    type=str,
+)
 def run(
     config_file: str,
     bucket: Optional[str],
     dry_run: Optional[bool],
     provider: Optional[str],
+    source: Optional[str],
     debug: Optional[bool],
 ):
     try:
@@ -90,7 +105,10 @@ def run(
             logger.exception(f"Failed due a configuration error: {e}")
             sys.exit(1)
 
-    engine.load(dry_run=dry_run, provider=provider)
+    if debug:
+        logging.getLogger("root").setLevel(logging.DEBUG)
+
+    engine.load(dry_run=dry_run, provider=provider, source=source)
 
     logger.info("Done")
 

{ingestify-0.3.4 → ingestify-0.4.0}/ingestify/domain/models/dataset/collection_metadata.py

@@ -6,7 +6,8 @@ from typing import Optional
 @dataclass
 class DatasetCollectionMetadata:
     # This can be useful to figure out if a backfill is required
-    first_modified: Optional[datetime]
+    # TODO - Note: not stored at Dataset level and requires joined query to retrieve
+    # first_modified: Optional[datetime]
 
     # Use the last modified to only retrieve datasets that are changed
     last_modified: Optional[datetime]

{ingestify-0.3.4 → ingestify-0.4.0}/ingestify/domain/models/dataset/dataset.py

@@ -22,7 +22,10 @@ class Dataset(BaseModel):
     metadata: dict
     created_at: datetime
     updated_at: datetime
+
     revisions: List[Revision] = Field(default_factory=list)
+    # The last_modified_at is equal to the max modified_at of all files in all revisions
+    last_modified_at: Optional[datetime]
 
     @field_validator("identifier", mode="before")
     @classmethod
@@ -42,6 +45,13 @@ class Dataset(BaseModel):
         self.revisions.append(revision)
         self.updated_at = utcnow()
 
+        if self.last_modified_at:
+            self.last_modified_at = max(
+                self.last_modified_at, revision.last_modified_at
+            )
+        else:
+            self.last_modified_at = revision.last_modified_at
+
     def update_metadata(self, name: str, metadata: dict, state: DatasetState) -> bool:
         changed = False
         if self.name != name:

{ingestify-0.3.4 → ingestify-0.4.0}/ingestify/domain/models/dataset/revision.py

@@ -36,6 +36,10 @@ class Revision(BaseModel):
     is_squashed: bool = False
     state: RevisionState = RevisionState.PENDING_VALIDATION
 
+    @property
+    def last_modified_at(self):
+        return max(file.modified_at for file in self.modified_files)
+
     @property
     def modified_files_map(self) -> Dict[str, File]:
         return {file.file_id: file for file in self.modified_files}

{ingestify-0.3.4 → ingestify-0.4.0}/ingestify/domain/models/ingestion/ingestion_job.py

@@ -214,9 +214,6 @@ class IngestionJob:
         self, store: DatasetStore, task_executor: TaskExecutor
     ) -> Iterator[IngestionJobSummary]:
         is_first_chunk = True
-        ingestion_job_exception = (
-            None  # Indicate if there was an exception during the IngestionJob itself
-        )
         ingestion_job_summary = IngestionJobSummary.new(ingestion_job=self)
         # Process all items in batches. Yield a IngestionJobSummary per batch
 
@@ -224,6 +221,7 @@ class IngestionJob:
         with ingestion_job_summary.record_timing("get_dataset_collection"):
             dataset_collection_metadata = store.get_dataset_collection(
                 dataset_type=self.ingestion_plan.dataset_type,
+                provider=self.ingestion_plan.source.provider,
                 data_spec_versions=self.selector.data_spec_versions,
                 selector=self.selector,
                 metadata_only=True,
@@ -233,8 +231,8 @@ class IngestionJob:
         # There are two different, but similar flows here:
         # 1. The discover_datasets returns a list, and the entire list can be processed at once
         # 2. The discover_datasets returns an iterator of batches, in this case we need to process each batch
-        with ingestion_job_summary.record_timing("find_datasets"):
-            try:
+        try:
+            with ingestion_job_summary.record_timing("find_datasets"):
                 dataset_resources = self.ingestion_plan.source.find_datasets(
                     dataset_type=self.ingestion_plan.dataset_type,
                     data_spec_versions=self.selector.data_spec_versions,
@@ -244,12 +242,12 @@ class IngestionJob:
 
                 # We need to include the to_batches as that will start the generator
                 batches = to_batches(dataset_resources)
-            except Exception as e:
-                logger.exception("Failed to find datasets")
+        except Exception as e:
+            logger.exception("Failed to find datasets")
 
-                ingestion_job_summary.set_exception(e)
-                yield ingestion_job_summary
-                return
+            ingestion_job_summary.set_exception(e)
+            yield ingestion_job_summary
+            return
 
         finish_task_timer = ingestion_job_summary.start_timing("tasks")
 

{ingestify-0.3.4 → ingestify-0.4.0}/ingestify/domain/models/ingestion/ingestion_job_summary.py

@@ -9,7 +9,7 @@ from ingestify.domain import Selector, DataSpecVersionCollection
 from ingestify.domain.models.base import BaseModel
 from ingestify.domain.models.task.task_summary import TaskSummary, TaskState
 from ingestify.domain.models.timing import Timing
-from ingestify.utils import utcnow
+from ingestify.utils import utcnow, HasTiming
 
 if TYPE_CHECKING:
     from ingestify.domain.models.ingestion.ingestion_job import IngestionJob
@@ -25,7 +25,7 @@ def format_duration(duration: timedelta):
     return f"{duration.total_seconds():.2f}sec"
 
 
-class IngestionJobSummary(BaseModel):
+class IngestionJobSummary(BaseModel, HasTiming):
     ingestion_job_summary_id: str
     ingestion_job_id: str
 
@@ -39,7 +39,6 @@ class IngestionJobSummary(BaseModel):
     started_at: datetime = Field(default_factory=utcnow)
     ended_at: Optional[datetime] = None
     state: IngestionJobState = IngestionJobState.RUNNING
-    timings: List[Timing] = Field(default_factory=list)
     task_summaries: List[TaskSummary] = Field(default_factory=list)
 
     skipped_datasets: int = 0
@@ -60,22 +59,6 @@ class IngestionJobSummary(BaseModel):
         )
         return cls(**args)
 
-    @contextmanager
-    def record_timing(self, name: str):
-        start = utcnow()
-        try:
-            yield
-        finally:
-            self.timings.append(Timing(name=name, started_at=start, ended_at=utcnow()))
-
-    def start_timing(self, name):
-        start = utcnow()
-
-        def finish():
-            self.timings.append(Timing(name=name, started_at=start, ended_at=utcnow()))
-
-        return finish
-
     def add_task_summaries(self, task_summaries: List[TaskSummary]):
         self.task_summaries.extend(task_summaries)
 
@@ -101,6 +84,11 @@ class IngestionJobSummary(BaseModel):
         )
         self.ended_at = utcnow()
 
+        # Only keep failed tasks. Rest isn't interesting
+        self.task_summaries = [
+            task for task in self.task_summaries if task.state == TaskState.FAILED
+        ]
+
     def set_finished(self):
         self.state = IngestionJobState.FINISHED
         self._set_ended()
@@ -114,25 +102,27 @@ class IngestionJobSummary(BaseModel):
         return self.ended_at - self.started_at
 
     def output_report(self):
-        print(f"\nIngestionJobSummary {self.state} in {format_duration(self.duration)}")
-        print("--------------------")
-        print(f"  - IngestionPlan:")
-        print(f"        Source: {self.source_name}")
-        print(f"        Provider: {self.provider}")
-        print(f"        DatasetType: {self.dataset_type}")
-        print(f"  - Selector: {self.selector}")
-        print(f"  - Timings: ")
+        print(
+            f"\nIngestionJobSummary {self.state.value} in {format_duration(self.duration)}"
+        )
+        print("********************************")
+        print(f"*  - IngestionPlan:")
+        print(f"*        Source: {self.source_name}")
+        print(f"*        Provider: {self.provider}")
+        print(f"*        DatasetType: {self.dataset_type}")
+        print(f"*  - Selector: {self.selector}")
+        print(f"*  - Timings: ")
         for timing in self.timings:
-            print(f"    - {timing.name}: {format_duration(timing.duration)}")
+            print(f"*    - {timing.name}: {format_duration(timing.duration)}")
         print(
-            f"  - Tasks: {len(self.task_summaries)} - {(len(self.task_summaries) / self.duration.total_seconds()):.1f} tasks/sec"
+            f"*  - Tasks: {len(self.task_summaries)} - {(len(self.task_summaries) / self.duration.total_seconds()):.1f} tasks/sec"
         )
 
-        print(f"    - Failed tasks: {self.failed_tasks}")
-        print(f"    - Successful tasks: {self.successful_tasks}")
-        print(f"    - Successful ignored tasks: {self.ignored_successful_tasks}")
-        print(f"    - Skipped datasets: {self.skipped_datasets}")
-        print("--------------------")
+        print(f"*    - Failed tasks: {self.failed_tasks}")
+        print(f"*    - Successful tasks: {self.successful_tasks}")
+        print(f"*    - Successful ignored tasks: {self.ignored_successful_tasks}")
+        print(f"*    - Skipped datasets: {self.skipped_datasets}")
+        print("********************************")
 
     def __enter__(self):
         return self

{ingestify-0.3.4 → ingestify-0.4.0}/ingestify/domain/models/task/task_summary.py

@@ -10,8 +10,7 @@ from ingestify.domain.models.base import BaseModel
 from ingestify.domain.models.dataset.identifier import Identifier
 from ingestify.domain.models.timing import Timing
 from ingestify.exceptions import IngestifyError
-from ingestify.utils import utcnow
-
+from ingestify.utils import utcnow, HasTiming
 
 logger = logging.getLogger(__name__)
 
@@ -28,7 +27,7 @@ class Operation(str, Enum):
     UPDATE = "UPDATE"
 
 
-class TaskSummary(BaseModel):
+class TaskSummary(BaseModel, HasTiming):
     task_id: str
     started_at: datetime
     operation: Operation
@@ -38,7 +37,6 @@ class TaskSummary(BaseModel):
     bytes_retrieved: int = 0
     last_modified: Optional[datetime] = None
     state: TaskState = TaskState.RUNNING
-    timings: List[Timing] = Field(default_factory=list)
 
     @field_validator("dataset_identifier", mode="before")
     @classmethod
@@ -48,27 +46,8 @@ class TaskSummary(BaseModel):
         return value
 
     def record_load_file(self, fn, metadata: dict):
-        start = utcnow()
-        try:
-            result = None
+        with self.record_timing(f"Load of {metadata.get('file_id', 'file')}", metadata):
             return fn()
-        except Exception as e:
-            result = {
-                "type": type(e).__name__,
-                "message": str(e),
-                "traceback": traceback.format_exc(),
-            }
-            raise e
-        finally:
-            metadata = dict(result=result, **metadata)
-            self.timings.append(
-                Timing(
-                    name=f"Load of {metadata.get('file_id', 'file')}",
-                    started_at=start,
-                    ended_at=utcnow(),
-                    metadata=metadata,
-                )
-            )
 
     @classmethod
     @contextmanager

{ingestify-0.3.4 → ingestify-0.4.0}/ingestify/infra/serialization/__init__.py

@@ -1,12 +1,5 @@
-import json
-from datetime import datetime
-from typing import Type, Any, TypeVar
-
-from dataclass_factory import Schema, Factory, NameStyle
-from dataclass_factory.schema_helpers import type_checker
-
-from ingestify.domain import DatasetCreated, Identifier
-from ingestify.domain.models.dataset.events import MetadataUpdated, RevisionAdded
+from ingestify.domain import DatasetCreated
+from ingestify.domain.models.dataset.events import RevisionAdded
 from ingestify.domain.models.event import DomainEvent
 
 
@@ -18,10 +11,6 @@ event_types = {
 
 def deserialize(event_dict: dict) -> DomainEvent:
     event_cls = event_types[event_dict["event_type"]]
-    event_dict["dataset"]["identifier"] = Identifier(
-        **event_dict["dataset"]["identifier"]
-    )
-
     return event_cls.model_validate(event_dict)
 
 

{ingestify-0.3.4 → ingestify-0.4.0}/ingestify/infra/store/dataset/sqlalchemy/repository.py

@@ -320,10 +320,9 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
 
         metadata_result_row = apply_query_filter(
             self.session.query(
-                func.min(file_table.c.modified_at).label("first_modified_at"),
-                func.max(file_table.c.modified_at).label("last_modified_at"),
+                func.max(dataset_table.c.last_modified_at).label("last_modified_at"),
                 func.count().label("row_count"),
-            ).join(dataset_table, dataset_table.c.dataset_id == file_table.c.dataset_id)
+            )
         ).first()
         dataset_collection_metadata = DatasetCollectionMetadata(*metadata_result_row)
 

{ingestify-0.3.4 → ingestify-0.4.0}/ingestify/infra/store/dataset/sqlalchemy/tables.py

@@ -16,6 +16,8 @@ from sqlalchemy import (
     TypeDecorator,
 )
 
+from sqlalchemy.dialects.postgresql import JSONB
+
 from ingestify.domain import Identifier, DataSpecVersionCollection, Selector
 from ingestify.domain.models.dataset.dataset import DatasetState
 from ingestify.domain.models.ingestion.ingestion_job_summary import IngestionJobState
@@ -25,18 +27,18 @@ from ingestify.domain.models.timing import Timing
 from ingestify.domain.models.dataset.revision import RevisionState
 
 
-def JSONType(serializer=None, deserializer=None):
+def JSONType(serializer=None, deserializer=None, base_type=JSON):
     class _JsonType(TypeDecorator):
         cache_ok = True
-        impl = JSON
+        impl = base_type
 
         def process_bind_param(self, value, dialect):
-            if serializer is not None:
+            if serializer and value is not None:
                 return serializer(value)
             return value
 
         def process_result_value(self, value, dialect):
-            if deserializer is not None:
+            if deserializer and value is not None:
                 return deserializer(value)
             return value
 
@@ -152,14 +154,19 @@ dataset_table = Table(
     metadata,
     Column("bucket", String(255), default=None),
     Column("dataset_id", String(255), primary_key=True),
-    Column("provider", String(255)),
-    Column("dataset_type", String(255)),
+    Column("provider", String(255), index=True),
+    Column("dataset_type", String(255), index=True),
     Column("state", DatasetStateString),
     Column("name", String(255)),
-    Column("identifier", JSONType(deserializer=lambda item: Identifier(**item))),
+    Column(
+        "identifier",
+        # Use JSONB when available
+        JSON().with_variant(JSONB(), "postgresql"),
+    ),
     Column("metadata", JSON),
     Column("created_at", TZDateTime(6)),
     Column("updated_at", TZDateTime(6)),
+    Column("last_modified_at", TZDateTime(6)),
 )
 
 revision_table = Table(

{ingestify-0.3.4 → ingestify-0.4.0}/ingestify/utils.py

@@ -1,34 +1,23 @@
-import abc
-import asyncio
-import inspect
 import logging
 import os
 import time
 import re
+import traceback
+from contextlib import contextmanager
 from multiprocessing import get_context, cpu_count, get_all_start_methods
 
 from datetime import datetime, timezone
 from string import Template
-from typing import (
-    Dict,
-    Generic,
-    Type,
-    TypeVar,
-    Tuple,
-    Optional,
-    Any,
-    Callable,
-    Awaitable,
-    List,
-    Iterable,
-)
+from typing import Dict, Tuple, Optional, Any, List
 
 import cloudpickle
+from pydantic import Field
 from typing_extensions import Self
 
 
 from itertools import islice
 
+from ingestify.domain.models.timing import Timing
 
 logger = logging.getLogger(__name__)
 
@@ -221,3 +210,46 @@ def try_number(s: str):
             return float(s)
         except ValueError:
             return s
+
+
+class HasTiming:
+    """Mixin to give Pydantic models ability to time actions."""
+
+    timings: List[Timing] = Field(default_factory=list)
+
+    @contextmanager
+    def record_timing(
+        self, description: str, metadata: Optional[dict] = None
+    ) -> Timing:
+        if not metadata:
+            metadata = {}
+
+        start = utcnow()
+        try:
+            result = None
+            yield
+        except Exception as e:
+            result = {
+                "type": type(e).__name__,
+                "message": str(e),
+                "traceback": traceback.format_exc(),
+            }
+            raise e
+        finally:
+            metadata = dict(result=result, **metadata)
+            self.timings.append(
+                Timing(
+                    name=description,
+                    started_at=start,
+                    ended_at=utcnow(),
+                    metadata=metadata,
+                )
+            )
+
+    def start_timing(self, name):
+        start = utcnow()
+
+        def finish():
+            self.timings.append(Timing(name=name, started_at=start, ended_at=utcnow()))
+
+        return finish

{ingestify-0.3.4 → ingestify-0.4.0}/ingestify.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.3.4
+Version: 0.4.0
 Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl