ingestify 0.5.1__tar.gz → 0.6.1__tar.gz

{ingestify-0.5.1 → ingestify-0.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.5.1
+Version: 0.6.1
 Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl

{ingestify-0.5.1 → ingestify-0.6.1}/ingestify/__init__.py

@@ -8,4 +8,4 @@ if not __INGESTIFY_SETUP__:
     from .infra import retrieve_http
     from .source_base import Source, DatasetResource
 
-__version__ = "0.5.1"
+__version__ = "0.6.1"

{ingestify-0.5.1 → ingestify-0.6.1}/ingestify/application/loader.py

@@ -155,13 +155,19 @@ class Loader:
             
             IngestionJobSummary holds the summary for an IngestionPlan and a single Selector
         """
-        for ingestion_plan, selector in selectors.values():
+
+        ingestion_job_prefix = str(uuid.uuid1())
+        for ingestion_job_idx, (ingestion_plan, selector) in enumerate(
+            selectors.values()
+        ):
             logger.info(
                 f"Discovering datasets from {ingestion_plan.source.__class__.__name__} using selector {selector}"
             )
 
             ingestion_job = IngestionJob(
-                ingestion_job_id=str(uuid.uuid1()),
+                # Create a combined IngestionJobId.
+                # This allows us to group all IngestionJobs within the same run
+                ingestion_job_id=f"{ingestion_job_prefix}.{ingestion_job_idx}",
                 ingestion_plan=ingestion_plan,
                 selector=selector,
             )

{ingestify-0.5.1 → ingestify-0.6.1}/ingestify/domain/models/data_spec_version_collection.py

@@ -1,5 +1,4 @@
 import copy
-from dataclasses import dataclass
 from typing import Dict, Union, List, Set, Optional
 
 

{ingestify-0.5.1 → ingestify-0.6.1}/ingestify/domain/models/ingestion/ingestion_job.py

@@ -24,9 +24,7 @@ from ingestify.utils import TaskExecutor, chunker
 logger = logging.getLogger(__name__)
 
 
-# Decrease batch size from 1_000 to 500. The sqlalchemy repository uses
-# a compound select, which breaks at more than 500 select statements
-DEFAULT_CHUNK_SIZE = 500
+DEFAULT_CHUNK_SIZE = 1000
 
 
 def run_task(task):
@@ -257,10 +255,14 @@ class IngestionJob:
         finish_task_timer = ingestion_job_summary.start_timing("tasks")
 
         while True:
+            logger.info(f"Finding next batch of datasets for selector={self.selector}")
+
             try:
-                batch = next(batches)
-            except StopIteration:
-                break
+                with ingestion_job_summary.record_timing("find_datasets"):
+                    try:
+                        batch = next(batches)
+                    except StopIteration:
+                        break
             except Exception as e:
                 logger.exception("Failed to fetch next batch")
 

{ingestify-0.5.1 → ingestify-0.6.1}/ingestify/infra/store/dataset/sqlalchemy/repository.py

@@ -13,9 +13,12 @@ from sqlalchemy import (
     literal,
     select,
     and_,
-    Column,
-    or_,
     Dialect,
+    values,
+    CTE,
+    column as sqlalchemy_column,
+    Integer,
+    String,
 )
 from sqlalchemy.engine import make_url
 from sqlalchemy.exc import NoSuchModuleError
@@ -174,6 +177,40 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
 
         connection.execute(stmt)
 
+    def _build_cte_sqlite(self, records, name: str) -> CTE:
+        """SQLite has a limit of 500 compound select statements. When we have more records,
+        create a nested CTE"""
+        if len(records) > 500:
+            return union_all(
+                select(self._build_cte_sqlite(records[:500], name + "1")),
+                select(self._build_cte_sqlite(records[500:], name + "2")),
+            ).cte(name)
+
+        return union_all(
+            *[
+                select(*(literal(value).label(key) for key, value in record.items()))
+                for record in records
+            ]
+        ).cte(name)
+
+    def _build_cte(self, records: list[dict], name: str) -> CTE:
+        """Build a CTE from a list of dictionaries."""
+
+        if self.dialect.name == "sqlite":
+            return self._build_cte_sqlite(records, name)
+
+        first_row = records[0]
+        columns = []
+        for key, value in first_row.items():
+            columns.append(
+                sqlalchemy_column(key, Integer if isinstance(value, int) else String)
+            )
+
+        # Prepare the data in tuples, in same order as columns
+        data = [tuple(record[column.name] for column in columns) for record in records]
+
+        return select(values(*columns, name=name).data(data)).cte(name)
+
     def _filter_query(
         self,
         query,
@@ -190,12 +227,10 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
                     # return an empty DatasetCollection
                     return DatasetCollection()
 
-                dataset_ids_cte = union_all(
-                    *[
-                        select(literal(dataset_id).label("dataset_id"))
-                        for dataset_id in set(dataset_id)
-                    ]
-                ).cte("dataset_ids")
+                dataset_ids_cte = self._build_cte(
+                    [{"dataset_id": dataset_id} for dataset_id in set(dataset_id)],
+                    "dataset_ids",
+                )
 
                 query = query.select_from(
                     dataset_table.join(
@@ -222,17 +257,9 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
             if not selectors:
                 raise ValueError("Selectors must contain at least one item")
 
-            attribute_sets = {
-                tuple(selector.filtered_attributes.items()) for selector in selectors
-            }
-
-            # Define a virtual table using a CTE for all attributes
-            attribute_cte = union_all(
-                *[
-                    select(*(literal(value).label(key) for key, value in attr_set))
-                    for attr_set in attribute_sets
-                ]
-            ).cte("attributes")
+            attribute_cte = self._build_cte(
+                [selector.filtered_attributes for selector in selectors], "attributes"
+            )
 
             keys = list(selectors[0].filtered_attributes.keys())
             first_selector = selectors[0].filtered_attributes
@@ -273,12 +300,10 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
         if not dataset_ids:
             return []
 
-        dataset_ids_cte = union_all(
-            *[
-                select(literal(dataset_id).label("dataset_id"))
-                for dataset_id in set(dataset_ids)
-            ]
-        ).cte("dataset_ids")
+        dataset_ids_cte = self._build_cte(
+            [{"dataset_id": dataset_id} for dataset_id in set(dataset_ids)],
+            "dataset_ids",
+        )
 
         dataset_rows = list(
             self.session.query(dataset_table).select_from(

{ingestify-0.5.1 → ingestify-0.6.1}/ingestify.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.5.1
+Version: 0.6.1
 Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl