ingestify 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

ingestify/__init__.py

@@ -8,4 +8,4 @@ if not __INGESTIFY_SETUP__:
     from .infra import retrieve_http
     from .source_base import Source, DatasetResource
 
-__version__ = "0.1.0"
+__version__ = "0.1.2"

ingestify/application/loader.py

@@ -84,9 +84,15 @@ def load_file(
             file_data_serialization_format=file_resource.data_serialization_format
             or "txt",
             **http_options,
+            **file_resource.loader_kwargs,
         )
     else:
-        return file_resource.file_loader(file_resource, current_file)
+        return file_resource.file_loader(
+            file_resource,
+            current_file,
+            # TODO: check how to fix this with typehints
+            **file_resource.loader_kwargs,
+        )
 
 
 class UpdateDatasetTask(Task):

ingestify/domain/models/resources/dataset_resource.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import datetime
 from typing import Optional, Callable, TYPE_CHECKING
 
@@ -29,6 +29,7 @@ class FileResource:
     file_loader: Optional[
         Callable[["FileResource", Optional["File"]], Optional["DraftFile"]]
     ] = None
+    loader_kwargs: dict = field(default_factory=dict)
 
     def __post_init__(self):
         if self.json_content is None and not self.url and not self.file_loader:
@@ -75,6 +76,7 @@ class DatasetResource:
                 Optional["DraftFile"],
             ]
         ] = None,
+        loader_kwargs: Optional[dict] = None,
     ):
         file_id = f"{data_feed_key}__{data_spec_version}"
         if file_id in self.files:
@@ -91,6 +93,7 @@ class DatasetResource:
             http_options=http_options,
             data_serialization_format=data_serialization_format,
             file_loader=file_loader,
+            loader_kwargs=loader_kwargs or {},
         )
 
         self.files[file_id] = file_resource

ingestify/infra/fetch/http.py

@@ -6,10 +6,38 @@ from io import BytesIO
 from typing import Optional, Callable, Tuple
 
 import requests
+from requests.adapters import HTTPAdapter
+from urllib3 import Retry
 
 from ingestify.domain.models import DraftFile, File
 from ingestify.utils import utcnow
 
+_session = None
+
+
+def get_session():
+    """Initialize the session when it's needed. This will make sure it's initialized
+    within the correct context, and we don't get issues when the session is created
+    in process #1 and used in process #2
+    """
+    global _session
+
+    if not _session:
+        retry_strategy = Retry(
+            total=4,  # Maximum number of retries
+            backoff_factor=2,  # Exponential backoff factor (e.g., 2 means 1, 2, 4, 8 seconds, ...)
+            status_forcelist=[429, 500, 502, 503, 504],  # HTTP status codes to retry on
+        )
+
+        adapter = HTTPAdapter(max_retries=retry_strategy)
+
+        # Create a new session object
+        _session = requests.Session()
+        _session.mount("http://", adapter)
+        _session.mount("https://", adapter)
+
+    return _session
+
 
 def retrieve_http(
     url,
@@ -41,7 +69,7 @@ def retrieve_http(
         else:
             raise Exception(f"Don't know how to use {key}")
 
-    response = requests.get(url, headers=headers, **http_kwargs)
+    response = get_session().get(url, headers=headers, **http_kwargs)
     response.raise_for_status()
     if response.status_code == 304:
         # Not modified

{ingestify-0.1.0.dist-info → ingestify-0.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.1.0
+Version: 0.1.2
 Summary: Standardizing soccer tracking- and event data
 Author: Koen Vossen
 Author-email: info@koenvossen.nl

{ingestify-0.1.0.dist-info → ingestify-0.1.2.dist-info}/RECORD

@@ -1,4 +1,4 @@
-ingestify/__init__.py,sha256=DdhKleT3RggJUMj5Auq_ImGLmKm-3HHs5Yerx_VsH_w,301
+ingestify/__init__.py,sha256=Jq8NUvWIQpVwqUiHza92T9gPyuEZtcg7sKNJkyG6oCE,301
 ingestify/cmdline.py,sha256=gLy79Cq3OnEyoEcI6koWIEbCwvgFZ1E8n3UU1sKS8FM,7143
 ingestify/exceptions.py,sha256=wMMuajl4AkQRfW60TLN7btJmQaH8-lUczXyW_2g9kOU,143
 ingestify/main.py,sha256=YjrAOiGzwurtoDyIf981DSJHHA6IT5q09k3QNzTKCC8,6814
@@ -8,7 +8,7 @@ ingestify/utils.py,sha256=eEHwulqNEb2YTRDrCMVxr6mWZYI6KOcNCAIWFTi74u0,8029
 ingestify/application/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/application/dataset_store.py,sha256=NAW-XSvp118Lr2hXZd3qtuQr6VkPdWCLksIwd5MSs30,11489
 ingestify/application/ingestion_engine.py,sha256=GYIhb8a9ePkEcNOBPdfu-YawiD7eRZMRlxCA-6g9DRA,2249
-ingestify/application/loader.py,sha256=d7iXmdHN_yhDkEc2MMZ_6BLMEdRz9ChpBMy4yCWvxQo,13317
+ingestify/application/loader.py,sha256=DSdSNFf7WynGsMCoK3iQGiMKkO76fZ_KIOBDEMZK3zU,13495
 ingestify/application/secrets_manager.py,sha256=5qCbPfUvRGP1Xbq6xPkMfpgYl8uPzF_0NbiKeRY5zxU,1757
 ingestify/domain/__init__.py,sha256=M7_fVTJjQUx53P4UQUPhowRKPKsIIjx4JYZL1yjHKsM,46
 ingestify/domain/models/__init__.py,sha256=xHVQZP57ZQYUKwAtbccnDKX89_yTOvBKAtn4XDVbEbY,930
@@ -37,7 +37,7 @@ ingestify/domain/models/event/event_bus.py,sha256=iseourbCwdUg-ODM5bM_u6cageJmce
 ingestify/domain/models/event/publisher.py,sha256=TOAawYYiPQCLR2Gm17LumMEzeapMDYcAYeklLFmwqAY,620
 ingestify/domain/models/event/subscriber.py,sha256=tP1ZFSvpJWKUITnATYekRxJzepz85UY7egBTMiP-dwg,1039
 ingestify/domain/models/resources/__init__.py,sha256=ZuY9DPRfwk-aLB3Lj6DYP_NqMkcQfcYjZp4VejTtcbU,46
-ingestify/domain/models/resources/dataset_resource.py,sha256=g0tu9QZQEdAGR-dRXQPL3ddcbEEGI__pvkDJGoscUTE,3027
+ingestify/domain/models/resources/dataset_resource.py,sha256=HH5wMqzoWvcL84GzNa7QL3YsciI757FG4iZu9DbXn_k,3181
 ingestify/domain/models/task/__init__.py,sha256=BdlyIPvE07Xax_IzLgO9DUw0wsz9OZutxnxdDNyRlys,79
 ingestify/domain/models/task/set.py,sha256=04txDYgS5rotXofD9TqChKdW0VZIYshrkfPIpXtlhW4,430
 ingestify/domain/models/task/task.py,sha256=R6tEZub-N_Wjl4VjwlPySdFb3L9D7nH4St2CcDzFoKA,107
@@ -46,7 +46,7 @@ ingestify/domain/services/transformers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JC
 ingestify/domain/services/transformers/kloppy_to_pandas.py,sha256=NcN6nTBGVn9gz-_hWZJTMcduS1Gg7EM4X95Cqxi1QIM,809
 ingestify/infra/__init__.py,sha256=V0hpLzPVTcOHRVh0gguF6FT30YIgEOUd5v87xUHkfZ4,88
 ingestify/infra/fetch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ingestify/infra/fetch/http.py,sha256=gm7x0dACp3sTY1FMlbv8zRoQLZuZgtXmBg3HbhQ0syI,3086
+ingestify/infra/fetch/http.py,sha256=4CcEkwtNzYkPspNIaQIfcthA5yLow0x_M9xpEsoucWw,3982
 ingestify/infra/serialization/__init__.py,sha256=LwfmRoO4qykZkJZXxVPSKpwoVIkg9qzXa7Egut9JjL4,1772
 ingestify/infra/sink/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/infra/sink/postgresql.py,sha256=SxuM3LntfYcpCriUpqJhMvgAf0s9cohXf6WkxSEDYDY,1816
@@ -72,8 +72,8 @@ ingestify/static/templates/wyscout/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRk
 ingestify/static/templates/wyscout/config.yaml.jinja2,sha256=0zQXuvJVwd0oL2OJsPMZ8sOvRbdfRbieSGLQ44ezmYc,379
 ingestify/static/templates/wyscout/query.py,sha256=wjAOMoKvhX-BzCRqEm1SJp6YAcF8Fsq7ddrOaOpAeOk,364
 ingestify/static/templates/wyscout/database/README.md,sha256=7IuzjKo7Pqkx5wkmOETRZDljVOslqfA3ALuHMONq5dg,32
-ingestify-0.1.0.dist-info/METADATA,sha256=ryin_4RwMcyvqa4l6nESqysjlutcffHHWwoBvcwU784,18822
-ingestify-0.1.0.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-ingestify-0.1.0.dist-info/entry_points.txt,sha256=czYYXeX2ul4zdeB6bKlz3HaUF7zyVVcj9E_sRNDisI0,53
-ingestify-0.1.0.dist-info/top_level.txt,sha256=Lwnjgns4KequS7KiicXhh6mLUvcdfjzLyPI4qf_s4A0,10
-ingestify-0.1.0.dist-info/RECORD,,
+ingestify-0.1.2.dist-info/METADATA,sha256=66WW34koisv72Sqvwn9gpYI0Le1wuwEW_nZorjYc0MY,18822
+ingestify-0.1.2.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+ingestify-0.1.2.dist-info/entry_points.txt,sha256=czYYXeX2ul4zdeB6bKlz3HaUF7zyVVcj9E_sRNDisI0,53
+ingestify-0.1.2.dist-info/top_level.txt,sha256=Lwnjgns4KequS7KiicXhh6mLUvcdfjzLyPI4qf_s4A0,10
+ingestify-0.1.2.dist-info/RECORD,,