data-prep-toolkit 1.1.7.dev5__py3-none-any.whl → 1.1.8.dev0__py3-none-any.whl

{data_prep_toolkit-1.1.7.dev5.dist-info → data_prep_toolkit-1.1.8.dev0.dist-info}/METADATA

@@ -1,14 +1,15 @@
 Metadata-Version: 2.4
 Name: data_prep_toolkit
-Version: 1.1.7.dev5
+Version: 1.1.8.dev0
 Summary: Data Preparation Toolkit Library for Ray and Python
 Author-email: Maroun Touma <touma@us.ibm.com>
 License: Apache-2.0
 Keywords: data,data preprocessing,data preparation,llm,generative,ai,fine-tuning,llmapps
-Requires-Python: <3.13,>=3.10
+Requires-Python: <3.15,>=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: numpy<2.0.0
-Requires-Dist: pyarrow<=17.0.0
+Requires-Dist: pyarrow<=17.0.0; python_version <= "3.12"
+Requires-Dist: pyarrow; python_version > "3.12"
 Requires-Dist: boto3>=1.41
 Requires-Dist: mmh3
 Requires-Dist: psutil
@@ -29,7 +30,8 @@ Requires-Dist: fasttext-wheel; extra == "dev"
 Requires-Dist: huggingface-hub<1.0.0,>=0.21.4; extra == "dev"
 Requires-Dist: transformers; extra == "dev"
 Provides-Extra: ray
-Requires-Dist: ray[default]==2.36.1; extra == "ray"
+Requires-Dist: ray[default]==2.36.1; python_version < "3.13" and extra == "ray"
+Requires-Dist: ray[default]==2.45.0; python_version >= "3.13" and extra == "ray"
 Requires-Dist: fastapi>=0.110.2; extra == "ray"
 Requires-Dist: pillow>=10.3.0; extra == "ray"
 Provides-Extra: spark

{data_prep_toolkit-1.1.7.dev5.dist-info → data_prep_toolkit-1.1.8.dev0.dist-info}/RECORD

@@ -50,10 +50,10 @@ data_processing/utils/cli_utils.py,sha256=A3LPVbr7-8JJwMycYYI_x27pW-Hqs7OAw1hZpt
 data_processing/utils/config.py,sha256=tKVqP4R-ge7dg7DDLDUywknJyv8RDDrFynZLQV-VQp8,1900
 data_processing/utils/log.py,sha256=zF5pYcBPndwxSZpBZCAdngc3hA8UB2rJd3LHl-OgYMA,7054
 data_processing/utils/model_loader.py,sha256=zVTspZ8PsgUjpM38_IilbSbh2gVjgPFVak4Nn6VInhA,3448
-data_processing/utils/model_loader_registry.py,sha256=LbPLXANhQ0aasC54hYL4JaHHdnQxLU1yZ67WBylfOpg,2592
+data_processing/utils/model_loader_registry.py,sha256=BTQu8x686bWHy_fC5mE9n-JdgJyEQcIMBzdjteo2C5Y,3696
 data_processing/utils/multilock.py,sha256=59BlwY2JUcthkIChmyoy5BveC8wseL31cngMs-A008o,5881
 data_processing/utils/params_utils.py,sha256=T6Dd6I1HpAtRiMmRnSjqVzfRncXg2QK1nkEbZ7khuhw,6706
-data_processing/utils/pipinstaller.py,sha256=OeDNaRYBRRzNeqbIhAlvVxIQ2PkfmzaC510EphhRvHw,2557
+data_processing/utils/pipinstaller.py,sha256=M4ry1Y5bqa43u5aUO8qxxsnY7jne_AQqtKhI4p-8RDI,3113
 data_processing/utils/transform_configuration.json,sha256=6YBw0Hk2mokY6JBn1kR6L9AkV_yivbFrpSoHecAJp9o,4562
 data_processing/utils/transform_configurator.py,sha256=7jlirBcL_c3FwvOSeMvTEcYGBEUOmA8EqmGAUZwp8VU,3639
 data_processing/utils/transform_utils.py,sha256=0DlW3T--m1QJnifRLgujaAJhOvjONmNpTwq0av1sJng,11486
@@ -82,7 +82,7 @@ data_processing_spark/runtime/spark/transform_runtime.py,sha256=feIB6Hr-Omy1C9LJ
 data_processing_spark/test_support/transform/__init__.py,sha256=0b40r5bqo4gLPEL_nAdF36zOfowu7hf-EVbeHtdtL-4,910
 data_processing_spark/test_support/transform/noop_folder_transform.py,sha256=EJCW5XPrAjVIb2lOfq44wV8Hetwo7Pj7ZRLvZBLtTPA,2225
 data_processing_spark/test_support/transform/noop_transform.py,sha256=wBg2MVx4W6ARiuADt_4LeNoj5puniXLAcMAqOzVhueE,1732
-data_prep_toolkit-1.1.7.dev5.dist-info/METADATA,sha256=2wJMZ2Ehij5X7Kn4jXWGrvQuAc1BiP2CGnc6nTZTVdo,2455
-data_prep_toolkit-1.1.7.dev5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-data_prep_toolkit-1.1.7.dev5.dist-info/top_level.txt,sha256=XGMDmY55_pe5KeRWvO0un9a640e2v99tzbBBtjNybPM,58
-data_prep_toolkit-1.1.7.dev5.dist-info/RECORD,,
+data_prep_toolkit-1.1.8.dev0.dist-info/METADATA,sha256=upRDhZXr2bd9GNeokYmQWGsPGr5YA3CmkiIYuN0oXVw,2638
+data_prep_toolkit-1.1.8.dev0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+data_prep_toolkit-1.1.8.dev0.dist-info/top_level.txt,sha256=XGMDmY55_pe5KeRWvO0un9a640e2v99tzbBBtjNybPM,58
+data_prep_toolkit-1.1.8.dev0.dist-info/RECORD,,

{data_prep_toolkit-1.1.7.dev5.dist-info → data_prep_toolkit-1.1.8.dev0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (82.0.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

data_processing/utils/model_loader_registry.py

@@ -74,3 +74,39 @@ def load_fasttext_model(model_path: str, token: str = None, **kwargs):
 
     model = fasttext.load_model(model_path)
     return model
+
+
+@register_model_loader("yolo")
+def load_yolo_model(model_path: str, token: str = None, **kwargs):
+    from ultralytics import YOLO
+    from huggingface_hub import hf_hub_download
+
+    if os.path.isfile(model_path):
+        model_path = model_path
+
+    elif os.path.isdir(model_path):
+        found = False
+        for root, _, files in os.walk(model_path):
+            for f in files:
+                if f.endswith(".pt"):
+                    model_path = os.path.join(root, f)
+                    found = True
+                    break
+        if not found:
+            raise FileNotFoundError(f"No .pt file found in : {model_path}")
+
+    else:
+        # assume hugging face repo and download .pt
+        filename = kwargs.get("model_filename", "model.pt")
+        subfolder = kwargs.get("subfolder", None)
+        revision = kwargs.get("revision", None)
+        model_path = hf_hub_download(
+            repo_id=model_path,
+            subfolder=subfolder,
+            revision=revision,
+            filename=filename,
+            token=token,
+        )
+
+    model = YOLO(model_path)
+    return model

data_processing/utils/pipinstaller.py

@@ -18,7 +18,7 @@ warnings.filterwarnings("ignore")
 
 import subprocess
 import sys
-
+import shutil
 
 class PipInstaller:
     """
@@ -36,14 +36,27 @@ class PipInstaller:
         """
         # build pip package string
         package = f"git+{project}#subdirectory={subdirectory}&egg={name}"
-        try:
-            # Do pip install (in sub process)
-            subprocess.check_call([sys.executable, "-m", "pip", "install", package])
-            return True
-        except Exception as e:
-            # process exception
-            print(f"Exception installing package {name}: {e}")
-            return False
+
+        uv_path = shutil.which("uv")
+        if uv_path:
+            try:
+                # Use 'uv pip install'.
+                # Note: uv requires --system to install into the system Python
+                # instead of a virtual environment.
+                subprocess.check_call([uv_path, "pip", "install", "--system", package])
+                return True
+            except Exception as e:
+                print(f"uv installation failed for {package}: {e}")
+                return False
+        else:
+            try:
+                # Do uv pip install (in sub process)
+                subprocess.check_call([sys.executable, "-m", "pip", "install", package])
+                return True
+            except Exception as e:
+                # process exception
+                print(f"Exception installing package {name}: {e}")
+                return False
 
     @staticmethod
     def validate(name: str) -> bool: