ingestify 0.5.0__tar.gz → 0.6.0__tar.gz

{ingestify-0.5.0 → ingestify-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.5.0
+Version: 0.6.0
 Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl

{ingestify-0.5.0 → ingestify-0.6.0}/ingestify/__init__.py

@@ -8,4 +8,4 @@ if not __INGESTIFY_SETUP__:
     from .infra import retrieve_http
     from .source_base import Source, DatasetResource
 
-__version__ = "0.5.0"
+__version__ = "0.6.0"

{ingestify-0.5.0 → ingestify-0.6.0}/ingestify/application/loader.py

@@ -155,13 +155,19 @@ class Loader:
             
             IngestionJobSummary holds the summary for an IngestionPlan and a single Selector
         """
-        for ingestion_plan, selector in selectors.values():
+
+        ingestion_job_prefix = str(uuid.uuid1())
+        for ingestion_job_idx, (ingestion_plan, selector) in enumerate(
+            selectors.values()
+        ):
             logger.info(
                 f"Discovering datasets from {ingestion_plan.source.__class__.__name__} using selector {selector}"
             )
 
             ingestion_job = IngestionJob(
-                ingestion_job_id=str(uuid.uuid1()),
+                # Create a combined IngestionJobId.
+                # This allows us to group all IngestionJobs within the same run
+                ingestion_job_id=f"{ingestion_job_prefix}.{ingestion_job_idx}",
                 ingestion_plan=ingestion_plan,
                 selector=selector,
             )

{ingestify-0.5.0 → ingestify-0.6.0}/ingestify/domain/models/data_spec_version_collection.py

@@ -1,5 +1,4 @@
 import copy
-from dataclasses import dataclass
 from typing import Dict, Union, List, Set, Optional
 
 

{ingestify-0.5.0 → ingestify-0.6.0}/ingestify/domain/models/ingestion/ingestion_job.py

@@ -24,7 +24,7 @@ from ingestify.utils import TaskExecutor, chunker
 logger = logging.getLogger(__name__)
 
 
-DEFAULT_CHUNK_SIZE = 1_000
+DEFAULT_CHUNK_SIZE = 1000
 
 
 def run_task(task):
@@ -255,8 +255,10 @@ class IngestionJob:
         finish_task_timer = ingestion_job_summary.start_timing("tasks")
 
         while True:
+            logger.info(f"Finding next batch of datasets for selector={self.selector}")
             try:
-                batch = next(batches)
+                with ingestion_job_summary.record_timing("find_datasets"):
+                    batch = next(batches)
             except StopIteration:
                 break
             except Exception as e:

{ingestify-0.5.0 → ingestify-0.6.0}/ingestify/infra/store/dataset/sqlalchemy/repository.py

@@ -13,9 +13,12 @@ from sqlalchemy import (
     literal,
     select,
     and_,
-    Column,
-    or_,
     Dialect,
+    values,
+    CTE,
+    column as sqlalchemy_column,
+    Integer,
+    String,
 )
 from sqlalchemy.engine import make_url
 from sqlalchemy.exc import NoSuchModuleError
@@ -137,10 +140,6 @@ class SqlAlchemySessionProvider:
         return self.session()
 
 
-def in_(column: Column, values):
-    return or_(*[column == value for value in values])
-
-
 class SqlAlchemyDatasetRepository(DatasetRepository):
     def __init__(self, session_provider: SqlAlchemySessionProvider):
         self.session_provider = session_provider
@@ -178,6 +177,40 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
 
         connection.execute(stmt)
 
+    def _build_cte_sqlite(self, records, name: str) -> CTE:
+        """SQLite has a limit of 500 compound select statements. When we have more records,
+        create a nested CTE"""
+        if len(records) > 500:
+            return union_all(
+                select(self._build_cte_sqlite(records[:500], name + "1")),
+                select(self._build_cte_sqlite(records[500:], name + "2")),
+            ).cte(name)
+
+        return union_all(
+            *[
+                select(*(literal(value).label(key) for key, value in record.items()))
+                for record in records
+            ]
+        ).cte(name)
+
+    def _build_cte(self, records: list[dict], name: str) -> CTE:
+        """Build a CTE from a list of dictionaries."""
+
+        if self.dialect.name == "sqlite":
+            return self._build_cte_sqlite(records, name)
+
+        first_row = records[0]
+        columns = []
+        for key, value in first_row.items():
+            columns.append(
+                sqlalchemy_column(key, Integer if isinstance(value, int) else String)
+            )
+
+        # Prepare the data in tuples, in same order as columns
+        data = [tuple(record[column.name] for column in columns) for record in records]
+
+        return select(values(*columns, name=name).data(data)).cte(name)
+
     def _filter_query(
         self,
         query,
@@ -194,7 +227,17 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
                     # return an empty DatasetCollection
                     return DatasetCollection()
 
-                query = query.filter(in_(dataset_table.c.dataset_id, dataset_id))
+                dataset_ids_cte = self._build_cte(
+                    [{"dataset_id": dataset_id} for dataset_id in set(dataset_id)],
+                    "dataset_ids",
+                )
+
+                query = query.select_from(
+                    dataset_table.join(
+                        dataset_ids_cte,
+                        dataset_ids_cte.c.dataset_id == dataset_table.c.dataset_id,
+                    )
+                )
             else:
                 query = query.filter(dataset_table.c.dataset_id == dataset_id)
 
@@ -214,17 +257,9 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
             if not selectors:
                 raise ValueError("Selectors must contain at least one item")
 
-            attribute_sets = {
-                tuple(selector.filtered_attributes.items()) for selector in selectors
-            }
-
-            # Define a virtual table using a CTE for all attributes
-            attribute_cte = union_all(
-                *[
-                    select(*(literal(value).label(key) for key, value in attr_set))
-                    for attr_set in attribute_sets
-                ]
-            ).cte("attributes")
+            attribute_cte = self._build_cte(
+                [selector.filtered_attributes for selector in selectors], "attributes"
+            )
 
             keys = list(selectors[0].filtered_attributes.keys())
             first_selector = selectors[0].filtered_attributes
@@ -265,15 +300,28 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
         if not dataset_ids:
             return []
 
+        dataset_ids_cte = self._build_cte(
+            [{"dataset_id": dataset_id} for dataset_id in set(dataset_ids)],
+            "dataset_ids",
+        )
+
         dataset_rows = list(
-            self.session.query(dataset_table).filter(
-                in_(dataset_table.c.dataset_id, dataset_ids)
+            self.session.query(dataset_table).select_from(
+                dataset_table.join(
+                    dataset_ids_cte,
+                    dataset_ids_cte.c.dataset_id == dataset_table.c.dataset_id,
+                )
             )
         )
         revisions_per_dataset = {}
         rows = (
             self.session.query(revision_table)
-            .filter(in_(revision_table.c.dataset_id, dataset_ids))
+            .select_from(
+                revision_table.join(
+                    dataset_ids_cte,
+                    dataset_ids_cte.c.dataset_id == revision_table.c.dataset_id,
+                )
+            )
             .order_by(revision_table.c.dataset_id)
         )
 
@@ -285,7 +333,12 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
         files_per_revision = {}
         rows = (
             self.session.query(file_table)
-            .filter(in_(file_table.c.dataset_id, dataset_ids))
+            .select_from(
+                file_table.join(
+                    dataset_ids_cte,
+                    dataset_ids_cte.c.dataset_id == file_table.c.dataset_id,
+                )
+            )
             .order_by(file_table.c.dataset_id, file_table.c.revision_id)
         )
 

{ingestify-0.5.0 → ingestify-0.6.0}/ingestify.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.5.0
+Version: 0.6.0
 Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl