upgini 1.2.85a3857.dev1__py3-none-any.whl → 1.2.86a1__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.85a3857.dev1"
+__version__ = "1.2.86a1"

upgini/autofe/feature.py

@@ -8,6 +8,7 @@ from pandas._typing import DtypeObj
 
 from upgini.autofe.all_operators import find_op
 from upgini.autofe.operator import Operator, PandasOperator
+from upgini.autofe.utils import pydantic_dump_method, pydantic_parse_method
 
 
 class Column:
@@ -80,9 +81,9 @@ class Feature:
         self.alias = alias
 
     def set_op_params(self, params: Optional[Dict[str, str]]) -> "Feature":
-        obj_dict = self.op.dict().copy()
+        obj_dict = pydantic_dump_method(self.op)().copy()
         obj_dict.update(params or {})
-        self.op = self.op.__class__.parse_obj(obj_dict)
+        self.op = pydantic_parse_method(self.op.__class__)(obj_dict)
         self.op.set_params(params)
 
         for child in self.children:

upgini/autofe/utils.py

@@ -81,3 +81,31 @@ def pydantic_validator(field_name: str, *fields, mode: str = "before", **kwargs)
             return wrapper
 
         return decorator
+
+
+def pydantic_json_method(obj):
+    if get_pydantic_version() >= 2:
+        return obj.model_dump_json
+    else:
+        return obj.json
+
+
+def pydantic_parse_method(cls):
+    if get_pydantic_version() >= 2:
+        return cls.model_validate
+    else:
+        return cls.parse_obj
+
+
+def pydantic_dump_method(obj):
+    if get_pydantic_version() >= 2:
+        return obj.model_dump
+    else:
+        return obj.dict
+
+
+def pydantic_copy_method(obj):
+    if get_pydantic_version() >= 2:
+        return obj.model_copy
+    else:
+        return obj.copy

upgini/features_enricher.py

@@ -3,6 +3,7 @@ import datetime
 import gc
 import hashlib
 import itertools
+import json
 import logging
 import numbers
 import os
@@ -59,6 +60,7 @@ from upgini.metadata import (
     CVType,
     FeaturesMetadataV2,
     FileColumnMeaningType,
+    FileColumnMetadata,
     ModelTaskType,
     RuntimeParameters,
     SearchKey,
@@ -2152,7 +2154,7 @@ class FeaturesEnricher(TransformerMixin):
             trace_id = trace_id or uuid.uuid4()
             return search_task.get_progress(trace_id)
 
-    def get_transactional_transform_api(self, only_online_sources=False):
+    def display_transactional_transform_api(self, only_online_sources=False):
         if self.api_key is None:
             raise ValidationError(self.bundle.get("transactional_transform_unregistered"))
         if self._search_task is None:
@@ -2178,20 +2180,36 @@ class FeaturesEnricher(TransformerMixin):
             return "test_value"
 
         file_metadata = self._search_task.get_file_metadata(str(uuid.uuid4()))
+
+        def get_column_meta(column_name: str) -> FileColumnMetadata:
+            for c in file_metadata.columns:
+                if c.name == column_name:
+                    return c
+
         search_keys = file_metadata.search_types()
         if SearchKey.IPV6_ADDRESS in search_keys:
             search_keys.pop(SearchKey.IPV6_ADDRESS, None)
-        original_names = {c.name: c.originalName for c in file_metadata.columns}
-        keys = (
-            "{"
-            + ", ".join(
-                [
-                    f'"{key.name}": {{"name": "{original_names.get(name, name)}", "value": "{key_example(key)}"}}'
-                    for key, name in search_keys.items()
-                ]
-            )
-            + "}"
-        )
+
+        search_keys_with_values = dict()
+        for sk_type, sk_name in search_keys.items():
+            if sk_type == SearchKey.IPV6_ADDRESS:
+                continue
+
+            sk_meta = get_column_meta(sk_name)
+            if sk_meta is None:
+                search_keys_with_values[sk_type.name] = [{"name": sk_name, "value": key_example(sk_type)}]
+            else:
+                if sk_meta.isUnnest:
+                    search_keys_with_values[sk_type.name] = [
+                        {"name": name, "value": key_example(sk_type)} for name in sk_meta.unnestKeyNames
+                    ]
+                else:
+                    search_keys_with_values[sk_type.name] = [{
+                        "name": sk_meta.originalName,
+                        "value": key_example(sk_type),
+                    }]
+
+        keys_section = json.dumps(search_keys_with_values)
         features_for_transform = self._search_task.get_features_for_transform()
         if features_for_transform:
             original_features_for_transform = [
@@ -2212,7 +2230,7 @@ class FeaturesEnricher(TransformerMixin):
 curl 'https://search.upgini.com/online/api/http_inference_trigger?search_id={search_id}' \\
     -H 'Authorization: {self.api_key}' \\
     -H 'Content-Type: application/json' \\
-    -d '{{"search_keys": {keys}{features_section}, "only_online_sources": {str(only_online_sources).lower()}}}'
+    -d '{{"search_keys": {keys_section}{features_section}, "only_online_sources": {str(only_online_sources).lower()}}}'
 
 {Format.BOLD}Python{Format.END}:
 
@@ -2221,13 +2239,12 @@ import requests
 response = requests.post(
     url='https://search.upgini.com/online/api/http_inference_trigger?search_id={search_id}',
     headers={{'Authorization': '{self.api_key}'}},
-    json={{"search_keys": {keys}{features_section}, "only_online_sources": {only_online_sources}}}
+    json={{"search_keys": {keys_section}{features_section}, "only_online_sources": {only_online_sources}}}
 )
 if response.status_code == 200:
     print(response.json())
 """
-
-        return api_example
+        print(api_example)
 
     def _get_copy_of_runtime_parameters(self) -> RuntimeParameters:
         return RuntimeParameters(properties=self.runtime_parameters.properties.copy())
@@ -2288,7 +2305,7 @@ if response.status_code == 200:
                 msg = self.bundle.get("online_api_features_transform").format(online_api_features)
                 self.logger.warning(msg)
                 print(msg)
-                print(self.get_transactional_transform_api(only_online_sources=True))
+                self.display_transactional_transform_api(only_online_sources=True)
 
             if not metrics_calculation:
                 transform_usage = self.rest_client.get_current_transform_usage(trace_id)

upgini/http.py

@@ -24,6 +24,12 @@ from pythonjsonlogger import json as jsonlogger
 from requests.exceptions import RequestException
 
 from upgini.__about__ import __version__
+from upgini.autofe.utils import (
+    pydantic_copy_method,
+    pydantic_dump_method,
+    pydantic_json_method,
+    pydantic_parse_method,
+)
 from upgini.errors import (
     HttpError,
     UnauthorizedError,
@@ -459,19 +465,19 @@ class _RestClient:
                 content = file.read()
                 md5_hash.update(content)
                 digest = md5_hash.hexdigest()
-                metadata_with_md5 = metadata.model_copy(update={"checksumMD5": digest})
+                metadata_with_md5 = pydantic_copy_method(metadata)(update={"checksumMD5": digest})
 
             # digest_sha256 = hashlib.sha256(
             #     pd.util.hash_pandas_object(pd.read_parquet(file_path, engine="fastparquet")).values
             # ).hexdigest()
             digest_sha256 = self.compute_file_digest(file_path)
-            metadata_with_md5 = metadata_with_md5.model_copy(update={"digest": digest_sha256})
+            metadata_with_md5 = pydantic_copy_method(metadata_with_md5)(update={"digest": digest_sha256})
 
             with open(file_path, "rb") as file:
                 files = {
                     "metadata": (
                         "metadata.json",
-                        metadata_with_md5.model_dump_json(exclude_none=True).encode(),
+                        pydantic_json_method(metadata_with_md5)(exclude_none=True).encode(),
                         "application/json",
                     ),
                     "tracking": (
@@ -481,7 +487,7 @@ class _RestClient:
                     ),
                     "metrics": (
                         "metrics.json",
-                        metrics.model_dump_json(exclude_none=True).encode(),
+                        pydantic_json_method(metrics)(exclude_none=True).encode(),
                         "application/json",
                     ),
                     "file": (metadata_with_md5.name, file, "application/octet-stream"),
@@ -489,7 +495,7 @@ class _RestClient:
                 if search_customization is not None:
                     files["customization"] = (
                         "customization.json",
-                        search_customization.model_dump_json(exclude_none=True).encode(),
+                        pydantic_json_method(search_customization)(exclude_none=True).encode(),
                         "application/json",
                     )
                 additional_headers = {self.SEARCH_KEYS_HEADER_NAME: ",".join(self.search_keys_meaning_types(metadata))}
@@ -504,7 +510,7 @@ class _RestClient:
     def check_uploaded_file_v2(self, trace_id: str, file_upload_id: str, metadata: FileMetadata) -> bool:
         api_path = self.CHECK_UPLOADED_FILE_URL_FMT_V2.format(file_upload_id)
         response = self._with_unauth_retry(
-            lambda: self._send_post_req(api_path, trace_id, metadata.model_dump_json(exclude_none=True))
+            lambda: self._send_post_req(api_path, trace_id, pydantic_json_method(metadata)(exclude_none=True))
         )
         return bool(response)
 
@@ -518,11 +524,15 @@ class _RestClient:
     ) -> SearchTaskResponse:
         api_path = self.INITIAL_SEARCH_WITHOUT_UPLOAD_URI_FMT_V2.format(file_upload_id)
         files = {
-            "metadata": ("metadata.json", metadata.model_dump_json(exclude_none=True).encode(), "application/json"),
-            "metrics": ("metrics.json", metrics.model_dump_json(exclude_none=True).encode(), "application/json"),
+            "metadata": (
+                "metadata.json",
+                pydantic_json_method(metadata)(exclude_none=True).encode(),
+                "application/json",
+            ),
+            "metrics": ("metrics.json", pydantic_json_method(metrics)(exclude_none=True).encode(), "application/json"),
         }
         if search_customization is not None:
-            files["customization"] = search_customization.model_dump_json(exclude_none=True).encode()
+            files["customization"] = pydantic_json_method(search_customization)(exclude_none=True).encode()
         additional_headers = {self.SEARCH_KEYS_HEADER_NAME: ",".join(self.search_keys_meaning_types(metadata))}
         response = self._with_unauth_retry(
             lambda: self._send_post_file_req_v2(
@@ -548,19 +558,19 @@ class _RestClient:
                 content = file.read()
                 md5_hash.update(content)
                 digest = md5_hash.hexdigest()
-                metadata_with_md5 = metadata.model_copy(update={"checksumMD5": digest})
+                metadata_with_md5 = pydantic_copy_method(metadata)(update={"checksumMD5": digest})
 
             # digest_sha256 = hashlib.sha256(
             #     pd.util.hash_pandas_object(pd.read_parquet(file_path, engine="fastparquet")).values
             # ).hexdigest()
             digest_sha256 = self.compute_file_digest(file_path)
-            metadata_with_md5 = metadata_with_md5.model_copy(update={"digest": digest_sha256})
+            metadata_with_md5 = pydantic_copy_method(metadata_with_md5)(update={"digest": digest_sha256})
 
             with open(file_path, "rb") as file:
                 files = {
                     "metadata": (
                         "metadata.json",
-                        metadata_with_md5.model_dump_json(exclude_none=True).encode(),
+                        pydantic_json_method(metadata_with_md5)(exclude_none=True).encode(),
                         "application/json",
                     ),
                     "tracking": (
@@ -570,7 +580,7 @@ class _RestClient:
                     ),
                     "metrics": (
                         "metrics.json",
-                        metrics.model_dump_json(exclude_none=True).encode(),
+                        pydantic_json_method(metrics)(exclude_none=True).encode(),
                         "application/json",
                     ),
                     "file": (metadata_with_md5.name, file, "application/octet-stream"),
@@ -578,7 +588,7 @@ class _RestClient:
                 if search_customization is not None:
                     files["customization"] = (
                         "customization.json",
-                        search_customization.model_dump_json(exclude_none=True).encode(),
+                        pydantic_json_method(search_customization)(exclude_none=True).encode(),
                         "application/json",
                     )
 
@@ -602,11 +612,15 @@ class _RestClient:
     ) -> SearchTaskResponse:
         api_path = self.VALIDATION_SEARCH_WITHOUT_UPLOAD_URI_FMT_V2.format(file_upload_id, initial_search_task_id)
         files = {
-            "metadata": ("metadata.json", metadata.model_dump_json(exclude_none=True).encode(), "application/json"),
-            "metrics": ("metrics.json", metrics.model_dump_json(exclude_none=True).encode(), "application/json"),
+            "metadata": (
+                "metadata.json",
+                pydantic_json_method(metadata)(exclude_none=True).encode(),
+                "application/json",
+            ),
+            "metrics": ("metrics.json", pydantic_json_method(metrics)(exclude_none=True).encode(), "application/json"),
         }
         if search_customization is not None:
-            files["customization"] = search_customization.model_dump_json(exclude_none=True).encode()
+            files["customization"] = pydantic_json_method(search_customization)(exclude_none=True).encode()
         additional_headers = {self.SEARCH_KEYS_HEADER_NAME: ",".join(self.search_keys_meaning_types(metadata))}
         response = self._with_unauth_retry(
             lambda: self._send_post_file_req_v2(
@@ -670,7 +684,7 @@ class _RestClient:
                     "file": (metadata.name, file, "application/octet-stream"),
                     "metadata": (
                         "metadata.json",
-                        metadata.model_dump_json(exclude_none=True).encode(),
+                        pydantic_json_method(metadata)(exclude_none=True).encode(),
                         "application/json",
                     ),
                 }
@@ -682,12 +696,12 @@ class _RestClient:
     def get_search_file_metadata(self, search_task_id: str, trace_id: str) -> FileMetadata:
         api_path = self.SEARCH_FILE_METADATA_URI_FMT_V2.format(search_task_id)
         response = self._with_unauth_retry(lambda: self._send_get_req(api_path, trace_id))
-        return FileMetadata.model_validate(response)
+        return pydantic_parse_method(FileMetadata)(response)
 
     def get_provider_search_metadata_v3(self, provider_search_task_id: str, trace_id: str) -> ProviderTaskMetadataV2:
         api_path = self.SEARCH_TASK_METADATA_FMT_V3.format(provider_search_task_id)
         response = self._with_unauth_retry(lambda: self._send_get_req(api_path, trace_id))
-        return ProviderTaskMetadataV2.model_validate(response)
+        return pydantic_parse_method(ProviderTaskMetadataV2)(response)
 
     def get_current_transform_usage(self, trace_id) -> TransformUsage:
         track_metrics = get_track_metrics(self.client_ip, self.client_visitorid)
@@ -706,7 +720,7 @@ class _RestClient:
                 lambda: self._send_post_req(
                     api_path,
                     trace_id=None,
-                    json_data=log_event.dict(exclude_none=True),
+                    json_data=pydantic_dump_method(log_event)(exclude_none=True),
                     content_type="application/json",
                     result_format="text",
                     silent=True,
@@ -723,7 +737,7 @@ class _RestClient:
         try:
             requests.post(
                 url=urljoin(_RestClient.PROD_BACKEND_URL, api_path),
-                json=log_event.model_dump(exclude_none=True),
+                json=pydantic_dump_method(log_event)(exclude_none=True),
                 headers=_RestClient._get_base_headers(content_type="application/json"),
             )
         except Exception:

upgini/metrics.py

@@ -391,7 +391,9 @@ class EstimatorWrapper:
                     self.converted_to_int.append(c)
                     self.cat_features.remove(c)
                 elif is_float_dtype(x[c]) or (x[c].dtype == "category" and is_float_dtype(x[c].cat.categories)):
-                    self.logger.info(f"Convert float cat feature {c} to string")
+                    self.logger.info(
+                        f"Convert float cat feature {c} to string"
+                    )
                     x[c] = x[c].astype(str)
                     self.converted_to_str.append(c)
                 elif x[c].dtype not in ["category", "int64"]:
@@ -692,15 +694,7 @@ class CatBoostWrapper(EstimatorWrapper):
                     x[c] = x[c].fillna(np.nan)
                 elif x[c].dtype != "category":
                     x[c] = x[c].fillna("NA")
-            if isinstance(self.cv, TimeSeriesSplit) or isinstance(self.cv, BlockedTimeSeriesSplit):
-                self.logger.info("Using time-aware encoder for CatBoost")
-                encoder = CatBoostEncoder(random_state=DEFAULT_RANDOM_STATE, cols=self.cat_features, return_df=True)
-                encoded = encoder.fit_transform(x[self.cat_features].astype("object"), y)
-                x[self.cat_features] = encoded
-                self.cat_encoder = encoder
-            else:
-                self.cat_encoder = None
-                params["cat_features"] = self.cat_features
+            params["cat_features"] = self.cat_features
 
         return x, y, groups, params
 
@@ -744,16 +738,7 @@ class CatBoostWrapper(EstimatorWrapper):
                     x[c] = x[c].fillna(np.nan)
                 elif x[c].dtype != "category":
                     x[c] = x[c].fillna("NA")
-            if (
-                isinstance(self.cv, TimeSeriesSplit)
-                or isinstance(self.cv, BlockedTimeSeriesSplit)
-                and self.cat_encoder is not None
-            ):
-                self.logger.info("Using time-aware encoder for CatBoost")
-                encoded = self.cat_encoder.transform(x[self.cat_features].astype("object"), y)
-                x[self.cat_features] = encoded
-            else:
-                params["cat_features"] = self.cat_features
+            params["cat_features"] = self.cat_features
 
         return x, y, params
 

{upgini-1.2.85a3857.dev1.dist-info → upgini-1.2.86a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.85a3857.dev1
+Version: 1.2.86a1
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.2.85a3857.dev1.dist-info → upgini-1.2.86a1.dist-info}/RECORD

@@ -1,12 +1,12 @@
-upgini/__about__.py,sha256=fECI7PUZQG8IW2eHjUqgqHVtT40sMjfMgzLhuxKuQFA,33
+upgini/__about__.py,sha256=ecLPy8e5nmkHwHQrZPVIZD4Et6AoZCfYC38tO5V5-UI,25
 upgini/__init__.py,sha256=LXSfTNU0HnlOkE69VCxkgIKDhWP-JFo_eBQ71OxTr5Y,261
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
 upgini/dataset.py,sha256=fRtqSkXNONLnPe6cCL967GMt349FTIpXzy_u8LUKncw,35354
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=2ryADtOVEEebuUBhimusvnBzGxUkdTaqpEh2F1PqHSs,212719
-upgini/http.py,sha256=AfaJ3c8z_tK2hZFEehNybDKE0mp1tYcyAP_l0_p8bLQ,43933
+upgini/features_enricher.py,sha256=G0qbRPdlWe9p6cwYF3khP99-0kgAO8N0A2sfQxSLgmM,213446
+upgini/http.py,sha256=6Qcepv0tDC72mBBJxYHnA2xqw6QwFaKrXN8o4vju8Es,44372
 upgini/metadata.py,sha256=zt_9k0iQbWXuiRZcel4ORNPdQKt6Ou69ucZD_E1Q46o,12341
-upgini/metrics.py,sha256=zRrRpNqjSTubsyKPi0_jbHjE8QO_YqyHWtt1B5MfVH8,44086
+upgini/metrics.py,sha256=3cip0_L6-OFew74KsRwzxJDU6UFq05h2v7IsyHLcMRc,43164
 upgini/search_task.py,sha256=Q5HjBpLIB3OCxAD1zNv5yQ3ZNJx696WCK_-H35_y7Rs,17912
 upgini/spinner.py,sha256=4iMd-eIe_BnkqFEMIliULTbj6rNI2HkN_VJ4qYe0cUc,1118
 upgini/version_validator.py,sha256=DvbaAvuYFoJqYt0fitpsk6Xcv-H1BYDJYHUMxaKSH_Y,1509
@@ -16,11 +16,11 @@ upgini/autofe/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/autofe/all_operators.py,sha256=rdjF5eaE4bC6Q4eu_el5Z7ekYt8DjOFermz2bePPbUc,333
 upgini/autofe/binary.py,sha256=oOEECc4nRzZN2tYaiqx8F2XHnfWpk1bVvb7ZkZJ0lO8,7709
 upgini/autofe/date.py,sha256=MM1S-6imNSzCDOhbNnmsc_bwSqUWBcS8vWAdHF8j1kY,11134
-upgini/autofe/feature.py,sha256=MjBbviB5Jy90EuWlnhgGgBptn3GomJv0xNjhjZN0P5I,15329
+upgini/autofe/feature.py,sha256=aivkeVkJYzaT65Ug8dnd6__vr9QwcXxXQiNummjFwkE,15435
 upgini/autofe/groupby.py,sha256=IYmQV9uoCdRcpkeWZj_kI3ObzoNCNx3ff3h8sTL01tk,3603
 upgini/autofe/operator.py,sha256=EOffJw6vKXpEh5yymqb1RFNJPxGxmnHdFRo9dB5SCFo,4969
 upgini/autofe/unary.py,sha256=Sx11IoHRh5nwyALzjgG9GQOrVNIs8NZ1JzunAJuN66A,5731
-upgini/autofe/utils.py,sha256=fK1am2_tQj3fL2vDslblye8lmyfWgGIUOX1beYVBz4k,2420
+upgini/autofe/utils.py,sha256=dYrtyAM8Vcc_R8u4dNo54IsGrHKagTHDJTKhGho0bRg,2967
 upgini/autofe/vector.py,sha256=jHs0nNTOaHspYUlxW7fjQepk4cvr_JDQ65L1OCiVsds,1360
 upgini/autofe/timeseries/__init__.py,sha256=PGwwDAMwvkXl3el12tXVEmZUgDUvlmIPlXtROm6bD18,738
 upgini/autofe/timeseries/base.py,sha256=rWJqRuFAzTZEsUdWG5s1Vhif9zzRRmalASXvarufRxI,3610
@@ -70,7 +70,7 @@ upgini/utils/target_utils.py,sha256=LRN840dzx78-wg7ftdxAkp2c1eu8-JDvkACiRThm4HE,
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/ts_utils.py,sha256=26vhC0pN7vLXK6R09EEkMK3Lwb9IVPH7LRdqFIQ3kPs,1383
 upgini/utils/warning_counter.py,sha256=-GRY8EUggEBKODPSuXAkHn9KnEQwAORC0mmz_tim-PM,254
-upgini-1.2.85a3857.dev1.dist-info/METADATA,sha256=XycmCsMeqC_7hsO0YzR0E8b4eGnIcD-MBzuFvB4T24s,49172
-upgini-1.2.85a3857.dev1.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
-upgini-1.2.85a3857.dev1.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.85a3857.dev1.dist-info/RECORD,,
+upgini-1.2.86a1.dist-info/METADATA,sha256=NFKCl8fGXfDQAKjDwAKG-Q6xEpvQ9mRld6WvqhUSZcU,49164
+upgini-1.2.86a1.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
+upgini-1.2.86a1.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.86a1.dist-info/RECORD,,