ingestify 0.4.2__tar.gz → 0.5.0__tar.gz

{ingestify-0.4.2 → ingestify-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.4.2
+Version: 0.5.0
 Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl

{ingestify-0.4.2 → ingestify-0.5.0}/ingestify/__init__.py

@@ -8,4 +8,4 @@ if not __INGESTIFY_SETUP__:
     from .infra import retrieve_http
     from .source_base import Source, DatasetResource
 
-__version__ = "0.4.2"
+__version__ = "0.5.0"

{ingestify-0.4.2 → ingestify-0.5.0}/ingestify/domain/models/ingestion/ingestion_job.py

@@ -287,7 +287,7 @@ class IngestionJob:
                     selector=dataset_identifiers,
                 )
 
-            skipped_datasets = 0
+            skipped_tasks = 0
 
             task_set = TaskSet()
             for dataset_resource in batch:
@@ -307,7 +307,7 @@ class IngestionJob:
                             )
                         )
                     else:
-                        skipped_datasets += 1
+                        skipped_tasks += 1
                 else:
                     if self.ingestion_plan.fetch_policy.should_fetch(dataset_resource):
                         task_set.add(
@@ -317,12 +317,12 @@ class IngestionJob:
                             )
                         )
                     else:
-                        skipped_datasets += 1
+                        skipped_tasks += 1
 
             if task_set:
                 logger.info(
                     f"Discovered {len(dataset_identifiers)} datasets from {self.ingestion_plan.source.__class__.__name__} "
-                    f"using selector {self.selector} => {len(task_set)} tasks. {skipped_datasets} skipped."
+                    f"using selector {self.selector} => {len(task_set)} tasks. {skipped_tasks} skipped."
                 )
                 logger.info(f"Running {len(task_set)} tasks")
                 ingestion_job_summary.add_task_summaries(
@@ -334,7 +334,7 @@ class IngestionJob:
                     f"using selector {self.selector} => nothing to do"
                 )
 
-            ingestion_job_summary.increase_skipped_datasets(skipped_datasets)
+            ingestion_job_summary.increase_skipped_tasks(skipped_tasks)
 
             if ingestion_job_summary.task_count() >= MAX_TASKS_PER_CHUNK:
                 finish_task_timer()

{ingestify-0.4.2 → ingestify-0.5.0}/ingestify/domain/models/ingestion/ingestion_job_summary.py

@@ -41,7 +41,8 @@ class IngestionJobSummary(BaseModel, HasTiming):
     state: IngestionJobState = IngestionJobState.RUNNING
     task_summaries: List[TaskSummary] = Field(default_factory=list)
 
-    skipped_datasets: int = 0
+    total_tasks: int = 0
+    skipped_tasks: int = 0
     failed_tasks: int = 0
     successful_tasks: int = 0
     ignored_successful_tasks: int = 0
@@ -62,11 +63,11 @@ class IngestionJobSummary(BaseModel, HasTiming):
     def add_task_summaries(self, task_summaries: List[TaskSummary]):
         self.task_summaries.extend(task_summaries)
 
-    def increase_skipped_datasets(self, skipped_datasets: int):
-        self.skipped_datasets += skipped_datasets
+    def increase_skipped_tasks(self, skipped_tasks: int):
+        self.skipped_tasks += skipped_tasks
 
     def task_count(self):
-        return len(self.task_summaries) + self.skipped_datasets
+        return len(self.task_summaries) + self.skipped_tasks
 
     def _set_ended(self):
         self.failed_tasks = len(
@@ -82,6 +83,12 @@ class IngestionJobSummary(BaseModel, HasTiming):
                 if task.state == TaskState.FINISHED_IGNORED
             ]
         )
+        self.total_tasks = (
+            self.failed_tasks
+            + self.successful_tasks
+            + self.ignored_successful_tasks
+            + self.skipped_tasks
+        )
         self.ended_at = utcnow()
 
         # Only keep failed tasks. Rest isn't interesting
@@ -115,13 +122,13 @@ class IngestionJobSummary(BaseModel, HasTiming):
         for timing in self.timings:
             print(f"   - {timing.name}: {format_duration(timing.duration)}")
         print(
-            f" - Tasks: {len(self.task_summaries)} - {(len(self.task_summaries) / self.duration.total_seconds()):.1f} tasks/sec"
+            f" - Tasks: {self.total_tasks} - {(self.total_tasks / self.duration.total_seconds()):.1f} tasks/sec"
         )
 
         print(f"   - Failed tasks: {self.failed_tasks}")
         print(f"   - Successful tasks: {self.successful_tasks}")
         print(f"   - Successful ignored tasks: {self.ignored_successful_tasks}")
-        print(f"   - Skipped datasets: {self.skipped_datasets}")
+        print(f"   - Skipped datasets: {self.skipped_tasks}")
         print("********************************")
 
     def __enter__(self):

{ingestify-0.4.2 → ingestify-0.5.0}/ingestify/infra/store/dataset/sqlalchemy/repository.py

@@ -19,7 +19,7 @@ from sqlalchemy import (
 )
 from sqlalchemy.engine import make_url
 from sqlalchemy.exc import NoSuchModuleError
-from sqlalchemy.orm import Session, Query
+from sqlalchemy.orm import Session, Query, sessionmaker, scoped_session
 
 from ingestify.domain import File, Revision
 from ingestify.domain.models import (
@@ -34,6 +34,7 @@ from ingestify.domain.models.dataset.collection_metadata import (
 from ingestify.domain.models.ingestion.ingestion_job_summary import IngestionJobSummary
 from ingestify.domain.models.task.task_summary import TaskSummary
 from ingestify.exceptions import IngestifyError
+from ingestify.utils import get_concurrency
 
 from .tables import (
     metadata,
@@ -96,17 +97,15 @@ class SqlAlchemySessionProvider:
             self.url,
             # Use the default isolation level, don't need SERIALIZABLE
             # isolation_level="SERIALIZABLE",
+            pool_size=get_concurrency(),  # Maximum number of connections in the pool
+            max_overflow=5,
+            pool_recycle=1800,
+            pool_pre_ping=True,
         )
         self.dialect = self.engine.dialect
-        self.session = Session(bind=self.engine)
 
-    def __init__(self, url: str):
-        url = self.fix_url(url)
-
-        self.url = url
-        self._init_engine()
-
-        metadata.create_all(self.engine)
+        session_factory = sessionmaker(bind=self.engine)
+        self.session = scoped_session(session_factory)
 
     def __getstate__(self):
         return {"url": self.url}
@@ -115,6 +114,14 @@ class SqlAlchemySessionProvider:
         self.url = state["url"]
         self._init_engine()
 
+    def __init__(self, url: str):
+        url = self.fix_url(url)
+
+        self.url = url
+        self._init_engine()
+
+        metadata.create_all(self.engine)
+
     def __del__(self):
         self.close()
 
@@ -123,12 +130,11 @@ class SqlAlchemySessionProvider:
         self._init_engine()
 
     def close(self):
-        if hasattr(self, "session"):
-            self.session.close()
+        if hasattr(self, "engine"):
             self.engine.dispose()
 
     def get(self):
-        return self.session
+        return self.session()
 
 
 def in_(column: Column, values):

{ingestify-0.4.2 → ingestify-0.5.0}/ingestify/infra/store/dataset/sqlalchemy/tables.py

@@ -247,9 +247,10 @@ ingestion_job_summary_table = Table(
     Column("ended_at", TZDateTime(6)),
     # Some task counters
     Column("state", IngestionJobStateString),
+    Column("total_tasks", Integer),
     Column("successful_tasks", Integer),
     Column("ignored_successful_tasks", Integer),
-    Column("skipped_datasets", Integer),
+    Column("skipped_tasks", Integer),
     Column("failed_tasks", Integer),
     Column(
         "timings",

{ingestify-0.4.2 → ingestify-0.5.0}/ingestify/infra/store/file/s3_file_repository.py

@@ -2,9 +2,11 @@ from pathlib import Path
 from typing import BinaryIO
 
 import boto3 as boto3
+import botocore.config
 
 from ingestify.domain import Dataset
 from ingestify.domain.models import FileRepository
+from ingestify.utils import get_concurrency
 
 
 class S3FileRepository(FileRepository):
@@ -13,7 +15,10 @@ class S3FileRepository(FileRepository):
     @property
     def s3(self):
         if not self._s3:
-            self._s3 = boto3.resource("s3")
+            client_config = botocore.config.Config(
+                max_pool_connections=get_concurrency(),
+            )
+            self._s3 = boto3.resource("s3", config=client_config)
         return self._s3
 
     def __getstate__(self):

{ingestify-0.4.2 → ingestify-0.5.0}/ingestify/utils.py

@@ -3,6 +3,7 @@ import os
 import time
 import re
 import traceback
+from concurrent.futures import ThreadPoolExecutor
 from contextlib import contextmanager
 from multiprocessing import get_context, cpu_count, get_all_start_methods
 
@@ -137,59 +138,65 @@ def map_in_pool(func, iterable, processes=0):
         )
 
 
-class SyncPool:
+class SyncExecutor:
     def map(self, func, iterable):
         return [func(item) for item in iterable]
 
-    def join(self):
-        return True
+    def __enter__(self):
+        return self
 
-    def close(self):
-        return True
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
 
 
-class DummyPool:
+class DummyExecutor:
     def map(self, func, iterable):
         logger.info(f"DummyPool: not running {len(list(iterable))} tasks")
         return None
 
-    def join(self):
-        return True
+    def __enter__(self):
+        return self
 
-    def close(self):
-        return True
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
 
 
 class TaskExecutor:
     def __init__(self, processes=0, dry_run: bool = False):
         if dry_run:
-            pool = DummyPool()
+            executor = DummyExecutor()
         elif os.environ.get("INGESTIFY_RUN_EAGER") == "true":
-            pool = SyncPool()
+            executor = SyncExecutor()
         else:
             if not processes:
-                processes = int(os.environ.get("INGESTIFY_CONCURRENCY", "0"))
+                processes = get_concurrency()
+
+            # if "fork" in get_all_start_methods():
+            #     ctx = get_context("fork")
+            # else:
+            #     ctx = get_context("spawn")
 
-            if "fork" in get_all_start_methods():
-                ctx = get_context("fork")
-            else:
-                ctx = get_context("spawn")
+            # pool = ctx.Pool(processes or cpu_count())
 
-            pool = ctx.Pool(processes or cpu_count())
-        self.pool = pool
+            executor = ThreadPoolExecutor(max_workers=processes)
+
+        self.executor = executor
 
     def __enter__(self):
+        self.executor.__enter__()
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        self.join()
+        self.executor.__exit__(exc_type, exc_val, exc_tb)
 
     def run(self, func, iterable):
-        wrapped_fn = cloudpickle.dumps(func)
+        # If multiprocessing
+        # wrapped_fn = cloudpickle.dumps(func)
+        # res = self.executor.map(
+        #     cloud_unpack_and_call, ((wrapped_fn, item) for item in iterable)
+        # )
         start_time = time.time()
-        res = self.pool.map(
-            cloud_unpack_and_call, ((wrapped_fn, item) for item in iterable)
-        )
+        res = list(self.executor.map(func, iterable))
         if res:
             took = time.time() - start_time
             logger.info(
@@ -197,10 +204,6 @@ class TaskExecutor:
             )
         return res
 
-    def join(self):
-        self.pool.close()
-        self.pool.join()
-
 
 def try_number(s: str):
     try:
@@ -253,3 +256,10 @@ class HasTiming:
             self.timings.append(Timing(name=name, started_at=start, ended_at=utcnow()))
 
         return finish
+
+
+def get_concurrency():
+    concurrency = int(os.environ.get("INGESTIFY_CONCURRENCY", "0"))
+    if not concurrency:
+        concurrency = min(32, (os.cpu_count() or 1) + 4)
+    return concurrency

{ingestify-0.4.2 → ingestify-0.5.0}/ingestify.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.4.2
+Version: 0.5.0
 Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl