PyPI - ingestify - Versions diffs - 0.1.1__tar.gz → 0.1.3__tar.gz - Mend

@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import datetime
 from typing import Optional, Callable, TYPE_CHECKING
@@ -29,6 +29,7 @@ class FileResource:
     file_loader: Optional[
         Callable[["FileResource", Optional["File"]], Optional["DraftFile"]]
     ] = None
+    loader_kwargs: dict = field(default_factory=dict)
     def __post_init__(self):
         if self.json_content is None and not self.url and not self.file_loader:
@@ -75,6 +76,7 @@ class DatasetResource:
                 Optional["DraftFile"],
             ]
         ] = None,
+        loader_kwargs: Optional[dict] = None,
     ):
         file_id = f"{data_feed_key}__{data_spec_version}"
         if file_id in self.files:
@@ -91,6 +93,7 @@ class DatasetResource:
             http_options=http_options,
             data_serialization_format=data_serialization_format,
             file_loader=file_loader,
+            loader_kwargs=loader_kwargs or {},
         )
         self.files[file_id] = file_resource

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.1.1
+Version: 0.1.3
 Summary: Standardizing soccer tracking- and event data
 Author: Koen Vossen
 Author-email: info@koenvossen.nl

@@ -8,4 +8,4 @@ if not __INGESTIFY_SETUP__:
     from .infra import retrieve_http
     from .source_base import Source, DatasetResource
-__version__ = "0.1.1"
+__version__ = "0.1.3"

@@ -84,9 +84,15 @@ def load_file(
             file_data_serialization_format=file_resource.data_serialization_format
             or "txt",
             **http_options,
+            **file_resource.loader_kwargs,
         )
     else:
-        return file_resource.file_loader(file_resource, current_file)
+        return file_resource.file_loader(
+            file_resource,
+            current_file,
+            # TODO: check how to fix this with typehints
+            **file_resource.loader_kwargs,
+        )
 class UpdateDatasetTask(Task):

@@ -12,6 +12,7 @@ from ingestify.exceptions import ConfigurationError
 from ingestify.main import get_engine
 from ingestify import __version__
+from ingestify.utils import try_number
 logger = logging.getLogger(__name__)
 #
@@ -174,7 +175,7 @@ def delete_dataset(
     if "=" in dataset_id:
         selector = {
             # TODO: this `int` will might break stuff. Issue here is the int != str
-            _[0]: int(_[1])
+            _[0]: try_number(_[1])
             for _ in [_.split("=") for _ in dataset_id.split("/")]
         }
     else:

@@ -274,3 +274,13 @@ class TaskExecutor:
     def join(self):
         self.pool.close()
         self.pool.join()
+def try_number(s: str):
+    try:
+        return int(s)
+    except ValueError:
+        try:
+            return float(s)
+        except ValueError:
+            return s

ingestify 0.1.1__tar.gz → 0.1.3__tar.gz