ingestify 0.5.0__tar.gz → 0.5.1__tar.gz

{ingestify-0.5.0 → ingestify-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.5.0
+Version: 0.5.1
 Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl

{ingestify-0.5.0 → ingestify-0.5.1}/ingestify/__init__.py

@@ -8,4 +8,4 @@ if not __INGESTIFY_SETUP__:
     from .infra import retrieve_http
     from .source_base import Source, DatasetResource
 
-__version__ = "0.5.0"
+__version__ = "0.5.1"

{ingestify-0.5.0 → ingestify-0.5.1}/ingestify/domain/models/ingestion/ingestion_job.py

@@ -24,7 +24,9 @@ from ingestify.utils import TaskExecutor, chunker
 logger = logging.getLogger(__name__)
 
 
-DEFAULT_CHUNK_SIZE = 1_000
+# Decrease batch size from 1_000 to 500. The sqlalchemy repository uses
+# a compound select, which breaks at more than 500 select statements
+DEFAULT_CHUNK_SIZE = 500
 
 
 def run_task(task):

{ingestify-0.5.0 → ingestify-0.5.1}/ingestify/infra/store/dataset/sqlalchemy/repository.py

@@ -137,10 +137,6 @@ class SqlAlchemySessionProvider:
         return self.session()
 
 
-def in_(column: Column, values):
-    return or_(*[column == value for value in values])
-
-
 class SqlAlchemyDatasetRepository(DatasetRepository):
     def __init__(self, session_provider: SqlAlchemySessionProvider):
         self.session_provider = session_provider
@@ -194,7 +190,19 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
                     # return an empty DatasetCollection
                     return DatasetCollection()
 
-                query = query.filter(in_(dataset_table.c.dataset_id, dataset_id))
+                dataset_ids_cte = union_all(
+                    *[
+                        select(literal(dataset_id).label("dataset_id"))
+                        for dataset_id in set(dataset_id)
+                    ]
+                ).cte("dataset_ids")
+
+                query = query.select_from(
+                    dataset_table.join(
+                        dataset_ids_cte,
+                        dataset_ids_cte.c.dataset_id == dataset_table.c.dataset_id,
+                    )
+                )
             else:
                 query = query.filter(dataset_table.c.dataset_id == dataset_id)
 
@@ -265,15 +273,30 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
         if not dataset_ids:
             return []
 
+        dataset_ids_cte = union_all(
+            *[
+                select(literal(dataset_id).label("dataset_id"))
+                for dataset_id in set(dataset_ids)
+            ]
+        ).cte("dataset_ids")
+
         dataset_rows = list(
-            self.session.query(dataset_table).filter(
-                in_(dataset_table.c.dataset_id, dataset_ids)
+            self.session.query(dataset_table).select_from(
+                dataset_table.join(
+                    dataset_ids_cte,
+                    dataset_ids_cte.c.dataset_id == dataset_table.c.dataset_id,
+                )
             )
         )
         revisions_per_dataset = {}
         rows = (
             self.session.query(revision_table)
-            .filter(in_(revision_table.c.dataset_id, dataset_ids))
+            .select_from(
+                revision_table.join(
+                    dataset_ids_cte,
+                    dataset_ids_cte.c.dataset_id == revision_table.c.dataset_id,
+                )
+            )
             .order_by(revision_table.c.dataset_id)
         )
 
@@ -285,7 +308,12 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
         files_per_revision = {}
         rows = (
             self.session.query(file_table)
-            .filter(in_(file_table.c.dataset_id, dataset_ids))
+            .select_from(
+                file_table.join(
+                    dataset_ids_cte,
+                    dataset_ids_cte.c.dataset_id == file_table.c.dataset_id,
+                )
+            )
             .order_by(file_table.c.dataset_id, file_table.c.revision_id)
         )
 

{ingestify-0.5.0 → ingestify-0.5.1}/ingestify.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.5.0
+Version: 0.5.1
 Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl