ingestify 0.1.0__tar.gz → 0.1.2__tar.gz

{ingestify-0.1.0 → ingestify-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.1.0
+Version: 0.1.2
 Summary: Standardizing soccer tracking- and event data
 Author: Koen Vossen
 Author-email: info@koenvossen.nl

{ingestify-0.1.0 → ingestify-0.1.2}/ingestify/__init__.py

@@ -8,4 +8,4 @@ if not __INGESTIFY_SETUP__:
     from .infra import retrieve_http
     from .source_base import Source, DatasetResource
 
-__version__ = "0.1.0"
+__version__ = "0.1.2"

{ingestify-0.1.0 → ingestify-0.1.2}/ingestify/application/loader.py

@@ -84,9 +84,15 @@ def load_file(
             file_data_serialization_format=file_resource.data_serialization_format
             or "txt",
             **http_options,
+            **file_resource.loader_kwargs,
         )
     else:
-        return file_resource.file_loader(file_resource, current_file)
+        return file_resource.file_loader(
+            file_resource,
+            current_file,
+            # TODO: check how to fix this with typehints
+            **file_resource.loader_kwargs,
+        )
 
 
 class UpdateDatasetTask(Task):

{ingestify-0.1.0 → ingestify-0.1.2}/ingestify/domain/models/resources/dataset_resource.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import datetime
 from typing import Optional, Callable, TYPE_CHECKING
 
@@ -29,6 +29,7 @@ class FileResource:
     file_loader: Optional[
         Callable[["FileResource", Optional["File"]], Optional["DraftFile"]]
     ] = None
+    loader_kwargs: dict = field(default_factory=dict)
 
     def __post_init__(self):
         if self.json_content is None and not self.url and not self.file_loader:
@@ -75,6 +76,7 @@ class DatasetResource:
                 Optional["DraftFile"],
             ]
         ] = None,
+        loader_kwargs: Optional[dict] = None,
     ):
         file_id = f"{data_feed_key}__{data_spec_version}"
         if file_id in self.files:
@@ -91,6 +93,7 @@ class DatasetResource:
             http_options=http_options,
             data_serialization_format=data_serialization_format,
             file_loader=file_loader,
+            loader_kwargs=loader_kwargs or {},
         )
 
         self.files[file_id] = file_resource

{ingestify-0.1.0 → ingestify-0.1.2}/ingestify/infra/fetch/http.py

@@ -6,10 +6,38 @@ from io import BytesIO
 from typing import Optional, Callable, Tuple
 
 import requests
+from requests.adapters import HTTPAdapter
+from urllib3 import Retry
 
 from ingestify.domain.models import DraftFile, File
 from ingestify.utils import utcnow
 
+_session = None
+
+
+def get_session():
+    """Initialize the session when it's needed. This will make sure it's initialized
+    within the correct context, and we don't get issues when the session is created
+    in process #1 and used in process #2
+    """
+    global _session
+
+    if not _session:
+        retry_strategy = Retry(
+            total=4,  # Maximum number of retries
+            backoff_factor=2,  # Exponential backoff factor (e.g., 2 means 1, 2, 4, 8 seconds, ...)
+            status_forcelist=[429, 500, 502, 503, 504],  # HTTP status codes to retry on
+        )
+
+        adapter = HTTPAdapter(max_retries=retry_strategy)
+
+        # Create a new session object
+        _session = requests.Session()
+        _session.mount("http://", adapter)
+        _session.mount("https://", adapter)
+
+    return _session
+
 
 def retrieve_http(
     url,
@@ -41,7 +69,7 @@ def retrieve_http(
         else:
             raise Exception(f"Don't know how to use {key}")
 
-    response = requests.get(url, headers=headers, **http_kwargs)
+    response = get_session().get(url, headers=headers, **http_kwargs)
     response.raise_for_status()
     if response.status_code == 304:
         # Not modified

{ingestify-0.1.0 → ingestify-0.1.2}/ingestify.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.1.0
+Version: 0.1.2
 Summary: Standardizing soccer tracking- and event data
 Author: Koen Vossen
 Author-email: info@koenvossen.nl