ingestify 0.5.1__py3-none-any.whl → 0.6.0__py3-none-any.whl

ingestify/__init__.py

@@ -8,4 +8,4 @@ if not __INGESTIFY_SETUP__:
     from .infra import retrieve_http
     from .source_base import Source, DatasetResource
 
-__version__ = "0.5.1"
+__version__ = "0.6.0"

ingestify/application/loader.py

@@ -155,13 +155,19 @@ class Loader:
             
             IngestionJobSummary holds the summary for an IngestionPlan and a single Selector
         """
-        for ingestion_plan, selector in selectors.values():
+
+        ingestion_job_prefix = str(uuid.uuid1())
+        for ingestion_job_idx, (ingestion_plan, selector) in enumerate(
+            selectors.values()
+        ):
             logger.info(
                 f"Discovering datasets from {ingestion_plan.source.__class__.__name__} using selector {selector}"
             )
 
             ingestion_job = IngestionJob(
-                ingestion_job_id=str(uuid.uuid1()),
+                # Create a combined IngestionJobId.
+                # This allows us to group all IngestionJobs within the same run
+                ingestion_job_id=f"{ingestion_job_prefix}.{ingestion_job_idx}",
                 ingestion_plan=ingestion_plan,
                 selector=selector,
             )

ingestify/domain/models/data_spec_version_collection.py

@@ -1,5 +1,4 @@
 import copy
-from dataclasses import dataclass
 from typing import Dict, Union, List, Set, Optional
 
 

ingestify/domain/models/ingestion/ingestion_job.py

@@ -24,9 +24,7 @@ from ingestify.utils import TaskExecutor, chunker
 logger = logging.getLogger(__name__)
 
 
-# Decrease batch size from 1_000 to 500. The sqlalchemy repository uses
-# a compound select, which breaks at more than 500 select statements
-DEFAULT_CHUNK_SIZE = 500
+DEFAULT_CHUNK_SIZE = 1000
 
 
 def run_task(task):
@@ -257,8 +255,10 @@ class IngestionJob:
         finish_task_timer = ingestion_job_summary.start_timing("tasks")
 
         while True:
+            logger.info(f"Finding next batch of datasets for selector={self.selector}")
             try:
-                batch = next(batches)
+                with ingestion_job_summary.record_timing("find_datasets"):
+                    batch = next(batches)
             except StopIteration:
                 break
             except Exception as e:

ingestify/infra/store/dataset/sqlalchemy/repository.py

@@ -13,9 +13,12 @@ from sqlalchemy import (
     literal,
     select,
     and_,
-    Column,
-    or_,
     Dialect,
+    values,
+    CTE,
+    column as sqlalchemy_column,
+    Integer,
+    String,
 )
 from sqlalchemy.engine import make_url
 from sqlalchemy.exc import NoSuchModuleError
@@ -174,6 +177,40 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
 
         connection.execute(stmt)
 
+    def _build_cte_sqlite(self, records, name: str) -> CTE:
+        """SQLite has a limit of 500 compound select statements. When we have more records,
+        create a nested CTE"""
+        if len(records) > 500:
+            return union_all(
+                select(self._build_cte_sqlite(records[:500], name + "1")),
+                select(self._build_cte_sqlite(records[500:], name + "2")),
+            ).cte(name)
+
+        return union_all(
+            *[
+                select(*(literal(value).label(key) for key, value in record.items()))
+                for record in records
+            ]
+        ).cte(name)
+
+    def _build_cte(self, records: list[dict], name: str) -> CTE:
+        """Build a CTE from a list of dictionaries."""
+
+        if self.dialect.name == "sqlite":
+            return self._build_cte_sqlite(records, name)
+
+        first_row = records[0]
+        columns = []
+        for key, value in first_row.items():
+            columns.append(
+                sqlalchemy_column(key, Integer if isinstance(value, int) else String)
+            )
+
+        # Prepare the data in tuples, in same order as columns
+        data = [tuple(record[column.name] for column in columns) for record in records]
+
+        return select(values(*columns, name=name).data(data)).cte(name)
+
     def _filter_query(
         self,
         query,
@@ -190,12 +227,10 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
                     # return an empty DatasetCollection
                     return DatasetCollection()
 
-                dataset_ids_cte = union_all(
-                    *[
-                        select(literal(dataset_id).label("dataset_id"))
-                        for dataset_id in set(dataset_id)
-                    ]
-                ).cte("dataset_ids")
+                dataset_ids_cte = self._build_cte(
+                    [{"dataset_id": dataset_id} for dataset_id in set(dataset_id)],
+                    "dataset_ids",
+                )
 
                 query = query.select_from(
                     dataset_table.join(
@@ -222,17 +257,9 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
             if not selectors:
                 raise ValueError("Selectors must contain at least one item")
 
-            attribute_sets = {
-                tuple(selector.filtered_attributes.items()) for selector in selectors
-            }
-
-            # Define a virtual table using a CTE for all attributes
-            attribute_cte = union_all(
-                *[
-                    select(*(literal(value).label(key) for key, value in attr_set))
-                    for attr_set in attribute_sets
-                ]
-            ).cte("attributes")
+            attribute_cte = self._build_cte(
+                [selector.filtered_attributes for selector in selectors], "attributes"
+            )
 
             keys = list(selectors[0].filtered_attributes.keys())
             first_selector = selectors[0].filtered_attributes
@@ -273,12 +300,10 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
         if not dataset_ids:
             return []
 
-        dataset_ids_cte = union_all(
-            *[
-                select(literal(dataset_id).label("dataset_id"))
-                for dataset_id in set(dataset_ids)
-            ]
-        ).cte("dataset_ids")
+        dataset_ids_cte = self._build_cte(
+            [{"dataset_id": dataset_id} for dataset_id in set(dataset_ids)],
+            "dataset_ids",
+        )
 
         dataset_rows = list(
             self.session.query(dataset_table).select_from(

{ingestify-0.5.1.dist-info → ingestify-0.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.5.1
+Version: 0.6.0
 Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl

{ingestify-0.5.1.dist-info → ingestify-0.6.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-ingestify/__init__.py,sha256=Un08YQgoC1u_2dbvOdtOD59OKsUL78ekru-86GA3zpA,301
+ingestify/__init__.py,sha256=lkRolxtOzdwCaj9gGdmYjUXstsVpm3SKvP_lEMwKJ7U,301
 ingestify/cmdline.py,sha256=JcveX6e4i6mJtIllhTuruwbqxyoKIITIWE8kB6byvJU,7721
 ingestify/exceptions.py,sha256=izRzaLQmMy-4P8ZqGqVZyf4k6LFYOYqwYLuRaUH8BJw,187
 ingestify/main.py,sha256=yYKA-4WAk04RdBCGmatsCKiPFQzpyufoG4VzHiWkVtU,8979
@@ -8,12 +8,12 @@ ingestify/utils.py,sha256=EMdG3ZP3bX9DHxHvBLdkYLC3vcEVym7dmpIXQTikI3I,7281
 ingestify/application/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/application/dataset_store.py,sha256=JkAb1W0HaUgOwbohKntM4ttyrFXQ7df1uZSu2rbZllY,11680
 ingestify/application/ingestion_engine.py,sha256=4SAmPZDm3e2QA5jZvMrb6xz1eDDshKoSZDWH3TCe4Bo,2372
-ingestify/application/loader.py,sha256=Lg3qPLaeKOFGheeqqfVeCBEF3cn61oZThgYYHoqfOvQ,7694
+ingestify/application/loader.py,sha256=OvlBBmCiQS3KTs5G7kBbxcP80WTfFxJZ-CXGsJJGH8M,7958
 ingestify/application/secrets_manager.py,sha256=5qCbPfUvRGP1Xbq6xPkMfpgYl8uPzF_0NbiKeRY5zxU,1757
 ingestify/domain/__init__.py,sha256=M7_fVTJjQUx53P4UQUPhowRKPKsIIjx4JYZL1yjHKsM,46
 ingestify/domain/models/__init__.py,sha256=cjQmdSDFA-saXjdF1mLPNWILFHIFgdj20J_fC5FmFsI,770
 ingestify/domain/models/base.py,sha256=4gKbREajxJHlS-VwKoosNtHVupZ4eDLKMqnJ4ib0aS8,184
-ingestify/domain/models/data_spec_version_collection.py,sha256=CAXlO4W2AOOWAPdPAuymqBHnJpiYtkr2z7fYFJ3HSCk,1372
+ingestify/domain/models/data_spec_version_collection.py,sha256=x5BvBnVI9QAfqhjCrUK19HKAiujdU1m8PkbQZwDheFU,1338
 ingestify/domain/models/fetch_policy.py,sha256=d7K1TzliNJXxqaqzqEOQWLhvgIvmmqhUQEliXvSUcTs,1405
 ingestify/domain/models/sink.py,sha256=OBVfFMpB7puJmHg4q2KYx4qgoAnlmX8xKWYnPi8a9pc,178
 ingestify/domain/models/source.py,sha256=sB3aqr2LfjIbtw7ODJpHnPj3RUeo7gYmTU7MXvfaYg4,973
@@ -39,7 +39,7 @@ ingestify/domain/models/event/event_bus.py,sha256=iseourbCwdUg-ODM5bM_u6cageJmce
 ingestify/domain/models/event/publisher.py,sha256=TOAawYYiPQCLR2Gm17LumMEzeapMDYcAYeklLFmwqAY,620
 ingestify/domain/models/event/subscriber.py,sha256=tP1ZFSvpJWKUITnATYekRxJzepz85UY7egBTMiP-dwg,1039
 ingestify/domain/models/ingestion/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ingestify/domain/models/ingestion/ingestion_job.py,sha256=2Tibe1fKZU84LO_qHniO33ChTcJy3K0YLkVro8CjJPs,13573
+ingestify/domain/models/ingestion/ingestion_job.py,sha256=uv0nM-5T_xBUUgb8e6A5Km0oG4GNU3CRp9x0GVW9TUU,13600
 ingestify/domain/models/ingestion/ingestion_job_summary.py,sha256=ZEoL8kZfDM_NUYXD4_7Xpmtz6WduN50UcJBgNOxOxrE,4669
 ingestify/domain/models/ingestion/ingestion_plan.py,sha256=KAvITBMQt3zmMFokESQJyp3rMuz1Hxr6msfZK1_faZM,648
 ingestify/domain/models/resources/__init__.py,sha256=ZuY9DPRfwk-aLB3Lj6DYP_NqMkcQfcYjZp4VejTtcbU,46
@@ -64,7 +64,7 @@ ingestify/infra/source/wyscout.py,sha256=DxCzdkzYpVRHTfV9GpF8pe3FzwIk-WHYUlea6nO
 ingestify/infra/store/__init__.py,sha256=3dA6NWfB6FS5SFdQiSlJ0ZghBfnUAUuGIP5Vr4rkCqk,43
 ingestify/infra/store/dataset/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/infra/store/dataset/sqlalchemy/__init__.py,sha256=Z5JHWGO_hwT6rO-ecMOOAmOKjFFJi449KZvJTQgt6vQ,52
-ingestify/infra/store/dataset/sqlalchemy/repository.py,sha256=hn4x5tglqxO4EdyiAt_4wnDXmmWU87twHtBmVBRHVSY,18309
+ingestify/infra/store/dataset/sqlalchemy/repository.py,sha256=28m_eC7BhcRQLWyc66NwmFmaviV52V4n9D_X9YXJX7w,19239
 ingestify/infra/store/dataset/sqlalchemy/tables.py,sha256=kALM32jbdeZ4Wn9gON-w2WSb5tH1lIWaBFgn5i29qTk,10635
 ingestify/infra/store/file/__init__.py,sha256=DuEekZa2pmDuRCFiulbgoGotN0wGv3OrRXSvokY0PhY,104
 ingestify/infra/store/file/dummy_file_repository.py,sha256=azUq9c43Mz9-GWk9j0E97BaqyUKu-ZMrcuaIednLq5E,723
@@ -80,8 +80,8 @@ ingestify/static/templates/wyscout/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRk
 ingestify/static/templates/wyscout/config.yaml.jinja2,sha256=0zQXuvJVwd0oL2OJsPMZ8sOvRbdfRbieSGLQ44ezmYc,379
 ingestify/static/templates/wyscout/query.py,sha256=wjAOMoKvhX-BzCRqEm1SJp6YAcF8Fsq7ddrOaOpAeOk,364
 ingestify/static/templates/wyscout/database/README.md,sha256=7IuzjKo7Pqkx5wkmOETRZDljVOslqfA3ALuHMONq5dg,32
-ingestify-0.5.1.dist-info/METADATA,sha256=TOqbUz13KxM8v8kR-owtafHvYEXwj5ruaDP_bigoIbI,18854
-ingestify-0.5.1.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-ingestify-0.5.1.dist-info/entry_points.txt,sha256=czYYXeX2ul4zdeB6bKlz3HaUF7zyVVcj9E_sRNDisI0,53
-ingestify-0.5.1.dist-info/top_level.txt,sha256=Lwnjgns4KequS7KiicXhh6mLUvcdfjzLyPI4qf_s4A0,10
-ingestify-0.5.1.dist-info/RECORD,,
+ingestify-0.6.0.dist-info/METADATA,sha256=aT5qaahlo57g3hlk_MUudKrheOefMKmnFUiwF0bMYHU,18854
+ingestify-0.6.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+ingestify-0.6.0.dist-info/entry_points.txt,sha256=czYYXeX2ul4zdeB6bKlz3HaUF7zyVVcj9E_sRNDisI0,53
+ingestify-0.6.0.dist-info/top_level.txt,sha256=Lwnjgns4KequS7KiicXhh6mLUvcdfjzLyPI4qf_s4A0,10
+ingestify-0.6.0.dist-info/RECORD,,