upgini 1.2.89a1__py3-none-any.whl → 1.2.90__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.89a1"
+__version__ = "1.2.90"

upgini/http.py

@@ -252,6 +252,7 @@ class _RestClient:
 
     # V2
     CHECK_UPLOADED_FILE_URL_FMT_V2 = SERVICE_ROOT_V2 + "search/check-file?fileUploadId={0}"
+    IS_FILE_UPLOADED_URL_FMT_V2 = SERVICE_ROOT_V2 + "search/files/exists?digest={0}"
     INITIAL_SEARCH_URI_FMT_V2 = SERVICE_ROOT_V2 + "search/initial"
     INITIAL_SEARCH_WITHOUT_UPLOAD_URI_FMT_V2 = SERVICE_ROOT_V2 + "search/initial-without-upload?fileUploadId={0}"
     VALIDATION_SEARCH_URI_FMT_V2 = SERVICE_ROOT_V2 + "search/validation?initialSearchTaskId={0}"
@@ -272,6 +273,7 @@ class _RestClient:
     SEARCH_FILE_METADATA_URI_FMT_V2 = SERVICE_ROOT_V2 + "search/{0}/metadata"
     SEARCH_TASK_METADATA_FMT_V3 = SERVICE_ROOT_V2 + "search/metadata-v2/{0}"
     SEARCH_DUMP_INPUT_FMT_V2 = SERVICE_ROOT_V2 + "search/dump-input"
+    SEARCH_DUMP_INPUT_FILE_FMT = SERVICE_ROOT_V2 + "search/dump-input-file"
     TRANSFORM_USAGE_FMT = SERVICE_ROOT_V2 + "user/transform-usage"
 
     UPLOAD_USER_ADS_URI = SERVICE_ROOT + "ads/upload"
@@ -410,32 +412,29 @@ class _RestClient:
         eval_x_path: Optional[str] = None,
         eval_y_path: Optional[str] = None,
     ):
-        api_path = self.SEARCH_DUMP_INPUT_FMT_V2
-        files = {}
-        with open(x_path, "rb") as x_file:
-            files["x"] = ("x.pickle", x_file, "application/octet-stream")
-            if y_path:
-                with open(y_path, "rb") as y_file:
-                    files["y"] = ("y.pickle", y_file, "application/octet-stream")
-                    if eval_x_path and eval_y_path:
-                        with open(eval_x_path, "rb") as eval_x_file, open(eval_y_path, "rb") as eval_y_file:
-                            files["eval_x"] = ("eval_x.pickle", eval_x_file, "application/octet-stream")
-                            files["eval_y"] = ("eval_y.pickle", eval_y_file, "application/octet-stream")
-                            self._with_unauth_retry(
-                                lambda: self._send_post_file_req_v2(
-                                    api_path, files, trace_id=trace_id, need_json_response=False
-                                )
-                            )
-                    else:
-                        self._with_unauth_retry(
-                            lambda: self._send_post_file_req_v2(
-                                api_path, files, trace_id=trace_id, need_json_response=False
-                            )
-                        )
+        api_path = self.SEARCH_DUMP_INPUT_FILE_FMT
+
+        def upload_with_check(path: str, file_name: str):
+            digest_sha256 = self.compute_file_digest(path)
+            if self.is_file_uploaded(trace_id, digest_sha256):
+                # print(f"File {path} was already uploaded with digest {digest_sha256}, skipping")
+                return
             else:
-                self._with_unauth_retry(
-                    lambda: self._send_post_file_req_v2(api_path, files, trace_id=trace_id, need_json_response=False)
-                )
+                with open(path, "rb") as file:
+                    files = {"file": (file_name, file, "application/octet-stream")}
+                    self._with_unauth_retry(
+                        lambda: self._send_post_file_req_v2(
+                            api_path, files, trace_id=trace_id, need_json_response=False
+                        )
+                    )
+
+        upload_with_check(x_path, "x.parquet")
+        if y_path:
+            upload_with_check(y_path, "y.parquet")
+        if eval_x_path:
+            upload_with_check(eval_x_path, "eval_x.parquet")
+        if eval_y_path:
+            upload_with_check(eval_y_path, "eval_y.parquet")
 
     @staticmethod
     def compute_file_digest(filepath: str, algorithm="sha256", chunk_size=4096) -> str:
@@ -514,6 +513,11 @@ class _RestClient:
         )
         return bool(response)
 
+    def is_file_uploaded(self, trace_id: str, digest: str) -> bool:
+        api_path = self.IS_FILE_UPLOADED_URL_FMT_V2.format(digest)
+        response = self._with_unauth_retry(lambda: self._send_get_req(api_path, trace_id))
+        return bool(response)
+
     def initial_search_without_upload_v2(
         self,
         trace_id: str,

upgini/utils/target_utils.py

@@ -416,6 +416,7 @@ def calculate_psi(expected: pd.Series, actual: pd.Series) -> Union[float, Except
         test_distribution = actual.value_counts(bins=bins, normalize=True).sort_index().values
 
         # Calculate the PSI
-        return np.sum((train_distribution - test_distribution) * np.log(train_distribution / test_distribution))
+        ratio = np.where(test_distribution > 0, train_distribution / test_distribution, 1)
+        return np.sum((train_distribution - test_distribution) * np.log(ratio))
     except Exception as e:
         return e

{upgini-1.2.89a1.dist-info → upgini-1.2.90.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.89a1
+Version: 1.2.90
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.2.89a1.dist-info → upgini-1.2.90.dist-info}/RECORD

@@ -1,10 +1,10 @@
-upgini/__about__.py,sha256=zOnCJXwGq7WXQ73_SrUBkMqGfV00s4WlXFUPNyejNQ8,25
+upgini/__about__.py,sha256=GHc4XyRcf-LRcunv2-fpap4slj_PhG6QeOQqttDwIno,23
 upgini/__init__.py,sha256=LXSfTNU0HnlOkE69VCxkgIKDhWP-JFo_eBQ71OxTr5Y,261
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
 upgini/dataset.py,sha256=fRtqSkXNONLnPe6cCL967GMt349FTIpXzy_u8LUKncw,35354
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
 upgini/features_enricher.py,sha256=rieH8wjC1c_q2LYZoju8KZyshokNzFpwVtrCtG88w3s,215940
-upgini/http.py,sha256=6Qcepv0tDC72mBBJxYHnA2xqw6QwFaKrXN8o4vju8Es,44372
+upgini/http.py,sha256=4i7fQwrwU3WzDUOWzrgR-4C8eJwj_5dBwRAR-UjUtlc,44345
 upgini/metadata.py,sha256=zt_9k0iQbWXuiRZcel4ORNPdQKt6Ou69ucZD_E1Q46o,12341
 upgini/metrics.py,sha256=zIOaiyfQLedU9Fk4877drnlWh-KiImSkZpPeiq6Xr1E,45295
 upgini/search_task.py,sha256=Q5HjBpLIB3OCxAD1zNv5yQ3ZNJx696WCK_-H35_y7Rs,17912
@@ -66,11 +66,11 @@ upgini/utils/postal_code_utils.py,sha256=5M0sUqH2DAr33kARWCTXR-ACyzWbjDq_-0mmEml
 upgini/utils/progress_bar.py,sha256=N-Sfdah2Hg8lXP_fV9EfUTXz_PyRt4lo9fAHoUDOoLc,1550
 upgini/utils/sklearn_ext.py,sha256=jLJWAKkqQinV15Z4y1ZnsN3c-fKFwXTsprs00COnyVU,49315
 upgini/utils/sort.py,sha256=8uuHs2nfSMVnz8GgvbOmgMB1PgEIZP1uhmeRFxcwnYw,7039
-upgini/utils/target_utils.py,sha256=LRN840dzx78-wg7ftdxAkp2c1eu8-JDvkACiRThm4HE,16832
+upgini/utils/target_utils.py,sha256=mVZ8wrkBb-tzEnVZwZw0m-Y0Sojb5t-wIsACRH05nIw,16890
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/ts_utils.py,sha256=26vhC0pN7vLXK6R09EEkMK3Lwb9IVPH7LRdqFIQ3kPs,1383
 upgini/utils/warning_counter.py,sha256=-GRY8EUggEBKODPSuXAkHn9KnEQwAORC0mmz_tim-PM,254
-upgini-1.2.89a1.dist-info/METADATA,sha256=d9XvUcHoqSr2RzIpqLR42x1bffkKnr7PyT6iB6kZGYQ,49164
-upgini-1.2.89a1.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-upgini-1.2.89a1.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.89a1.dist-info/RECORD,,
+upgini-1.2.90.dist-info/METADATA,sha256=QWKn1q4NNZEH8k41xW03uvPmUSjwb-2uFH_Asecnr44,49162
+upgini-1.2.90.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+upgini-1.2.90.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.90.dist-info/RECORD,,