ingestify 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

ingestify/__init__.py

@@ -8,4 +8,4 @@ if not __INGESTIFY_SETUP__:
     from .infra import retrieve_http
     from .source_base import Source, DatasetResource
 
-__version__ = "0.1.1"
+__version__ = "0.1.3"

ingestify/application/loader.py

@@ -84,9 +84,15 @@ def load_file(
             file_data_serialization_format=file_resource.data_serialization_format
             or "txt",
             **http_options,
+            **file_resource.loader_kwargs,
         )
     else:
-        return file_resource.file_loader(file_resource, current_file)
+        return file_resource.file_loader(
+            file_resource,
+            current_file,
+            # TODO: check how to fix this with typehints
+            **file_resource.loader_kwargs,
+        )
 
 
 class UpdateDatasetTask(Task):

ingestify/cmdline.py

@@ -12,6 +12,7 @@ from ingestify.exceptions import ConfigurationError
 from ingestify.main import get_engine
 
 from ingestify import __version__
+from ingestify.utils import try_number
 
 logger = logging.getLogger(__name__)
 #
@@ -174,7 +175,7 @@ def delete_dataset(
     if "=" in dataset_id:
         selector = {
             # TODO: this `int` will might break stuff. Issue here is the int != str
-            _[0]: int(_[1])
+            _[0]: try_number(_[1])
             for _ in [_.split("=") for _ in dataset_id.split("/")]
         }
     else:

ingestify/domain/models/resources/dataset_resource.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import datetime
 from typing import Optional, Callable, TYPE_CHECKING
 
@@ -29,6 +29,7 @@ class FileResource:
     file_loader: Optional[
         Callable[["FileResource", Optional["File"]], Optional["DraftFile"]]
     ] = None
+    loader_kwargs: dict = field(default_factory=dict)
 
     def __post_init__(self):
         if self.json_content is None and not self.url and not self.file_loader:
@@ -75,6 +76,7 @@ class DatasetResource:
                 Optional["DraftFile"],
             ]
         ] = None,
+        loader_kwargs: Optional[dict] = None,
     ):
         file_id = f"{data_feed_key}__{data_spec_version}"
         if file_id in self.files:
@@ -91,6 +93,7 @@ class DatasetResource:
             http_options=http_options,
             data_serialization_format=data_serialization_format,
             file_loader=file_loader,
+            loader_kwargs=loader_kwargs or {},
         )
 
         self.files[file_id] = file_resource

ingestify/utils.py

@@ -274,3 +274,13 @@ class TaskExecutor:
     def join(self):
         self.pool.close()
         self.pool.join()
+
+
+def try_number(s: str):
+    try:
+        return int(s)
+    except ValueError:
+        try:
+            return float(s)
+        except ValueError:
+            return s

{ingestify-0.1.1.dist-info → ingestify-0.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.1.1
+Version: 0.1.3
 Summary: Standardizing soccer tracking- and event data
 Author: Koen Vossen
 Author-email: info@koenvossen.nl

{ingestify-0.1.1.dist-info → ingestify-0.1.3.dist-info}/RECORD

@@ -1,14 +1,14 @@
-ingestify/__init__.py,sha256=aikpoo_eBRu5RSYV4NGLYO89gnOWPVM6TGriRmeqZLg,301
-ingestify/cmdline.py,sha256=gLy79Cq3OnEyoEcI6koWIEbCwvgFZ1E8n3UU1sKS8FM,7143
+ingestify/__init__.py,sha256=7SmMqBObZfacvSUG-7g0fFyt6ZlamLxBNgU5ReQ36lk,301
+ingestify/cmdline.py,sha256=bIuyPgGEw4wIglNzpG9zp7TsJozsP8NSVsCe4eAyWUg,7189
 ingestify/exceptions.py,sha256=wMMuajl4AkQRfW60TLN7btJmQaH8-lUczXyW_2g9kOU,143
 ingestify/main.py,sha256=YjrAOiGzwurtoDyIf981DSJHHA6IT5q09k3QNzTKCC8,6814
 ingestify/server.py,sha256=OVrf_XtpAQIn88MzqQzShXgsA9_jbnqYvD8YPBjn3cs,2413
 ingestify/source_base.py,sha256=GXAFCoT11Zov9M2v-fqQr9gFCXbtVfEIEH32V7r2oE8,382
-ingestify/utils.py,sha256=eEHwulqNEb2YTRDrCMVxr6mWZYI6KOcNCAIWFTi74u0,8029
+ingestify/utils.py,sha256=WcbG2mEb-oLF_sA-2JPRbx8nD55HqASrHQRz7Zd0Ejg,8198
 ingestify/application/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/application/dataset_store.py,sha256=NAW-XSvp118Lr2hXZd3qtuQr6VkPdWCLksIwd5MSs30,11489
 ingestify/application/ingestion_engine.py,sha256=GYIhb8a9ePkEcNOBPdfu-YawiD7eRZMRlxCA-6g9DRA,2249
-ingestify/application/loader.py,sha256=d7iXmdHN_yhDkEc2MMZ_6BLMEdRz9ChpBMy4yCWvxQo,13317
+ingestify/application/loader.py,sha256=DSdSNFf7WynGsMCoK3iQGiMKkO76fZ_KIOBDEMZK3zU,13495
 ingestify/application/secrets_manager.py,sha256=5qCbPfUvRGP1Xbq6xPkMfpgYl8uPzF_0NbiKeRY5zxU,1757
 ingestify/domain/__init__.py,sha256=M7_fVTJjQUx53P4UQUPhowRKPKsIIjx4JYZL1yjHKsM,46
 ingestify/domain/models/__init__.py,sha256=xHVQZP57ZQYUKwAtbccnDKX89_yTOvBKAtn4XDVbEbY,930
@@ -37,7 +37,7 @@ ingestify/domain/models/event/event_bus.py,sha256=iseourbCwdUg-ODM5bM_u6cageJmce
 ingestify/domain/models/event/publisher.py,sha256=TOAawYYiPQCLR2Gm17LumMEzeapMDYcAYeklLFmwqAY,620
 ingestify/domain/models/event/subscriber.py,sha256=tP1ZFSvpJWKUITnATYekRxJzepz85UY7egBTMiP-dwg,1039
 ingestify/domain/models/resources/__init__.py,sha256=ZuY9DPRfwk-aLB3Lj6DYP_NqMkcQfcYjZp4VejTtcbU,46
-ingestify/domain/models/resources/dataset_resource.py,sha256=g0tu9QZQEdAGR-dRXQPL3ddcbEEGI__pvkDJGoscUTE,3027
+ingestify/domain/models/resources/dataset_resource.py,sha256=HH5wMqzoWvcL84GzNa7QL3YsciI757FG4iZu9DbXn_k,3181
 ingestify/domain/models/task/__init__.py,sha256=BdlyIPvE07Xax_IzLgO9DUw0wsz9OZutxnxdDNyRlys,79
 ingestify/domain/models/task/set.py,sha256=04txDYgS5rotXofD9TqChKdW0VZIYshrkfPIpXtlhW4,430
 ingestify/domain/models/task/task.py,sha256=R6tEZub-N_Wjl4VjwlPySdFb3L9D7nH4St2CcDzFoKA,107
@@ -72,8 +72,8 @@ ingestify/static/templates/wyscout/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRk
 ingestify/static/templates/wyscout/config.yaml.jinja2,sha256=0zQXuvJVwd0oL2OJsPMZ8sOvRbdfRbieSGLQ44ezmYc,379
 ingestify/static/templates/wyscout/query.py,sha256=wjAOMoKvhX-BzCRqEm1SJp6YAcF8Fsq7ddrOaOpAeOk,364
 ingestify/static/templates/wyscout/database/README.md,sha256=7IuzjKo7Pqkx5wkmOETRZDljVOslqfA3ALuHMONq5dg,32
-ingestify-0.1.1.dist-info/METADATA,sha256=H8ASxletYwv0etkk4E85L3OecC7Kg3zsTqdoS3NUCGM,18822
-ingestify-0.1.1.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-ingestify-0.1.1.dist-info/entry_points.txt,sha256=czYYXeX2ul4zdeB6bKlz3HaUF7zyVVcj9E_sRNDisI0,53
-ingestify-0.1.1.dist-info/top_level.txt,sha256=Lwnjgns4KequS7KiicXhh6mLUvcdfjzLyPI4qf_s4A0,10
-ingestify-0.1.1.dist-info/RECORD,,
+ingestify-0.1.3.dist-info/METADATA,sha256=N5OO5RAYulhFCXCqp4Hi2gEcEB9dnf6W7n59ZOGRUQ8,18822
+ingestify-0.1.3.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+ingestify-0.1.3.dist-info/entry_points.txt,sha256=czYYXeX2ul4zdeB6bKlz3HaUF7zyVVcj9E_sRNDisI0,53
+ingestify-0.1.3.dist-info/top_level.txt,sha256=Lwnjgns4KequS7KiicXhh6mLUvcdfjzLyPI4qf_s4A0,10
+ingestify-0.1.3.dist-info/RECORD,,