upgini 1.2.122a4__py3-none-any.whl → 1.2.146a4__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.122a4"
+__version__ = "1.2.146a4"

upgini/autofe/binary.py

@@ -1,5 +1,6 @@
 import abc
 from typing import Optional
+
 import Levenshtein
 import numpy as np
 import pandas as pd
@@ -201,7 +202,7 @@ class JaroWinklerSim1(StringSim):
     has_symmetry_importance: bool = True
 
     def _prepare_value(self, value: Optional[str]) -> Optional[str]:
-        return value
+        return value if value is not None and len(value) > 0 else None
 
     def _similarity(self, left: str, right: str) -> float:
         return jarowinkler_similarity(left, right)
@@ -216,7 +217,7 @@ class JaroWinklerSim2(StringSim):
     has_symmetry_importance: bool = True
 
     def _prepare_value(self, value: Optional[str]) -> Optional[str]:
-        return value[::-1] if value is not None else None
+        return value[::-1] if value is not None and len(value) > 0 else None
 
     def _similarity(self, left: str, right: str) -> float:
         return jarowinkler_similarity(left, right)
@@ -231,7 +232,7 @@ class LevenshteinSim(StringSim):
     has_symmetry_importance: bool = True
 
     def _prepare_value(self, value: Optional[str]) -> Optional[str]:
-        return value
+        return value if value is not None and len(value) > 0 else None
 
     def _similarity(self, left: str, right: str) -> float:
         return 1 - Levenshtein.distance(left, right) / max(len(left), len(right))

upgini/data_source/data_source_publisher.py

@@ -123,17 +123,9 @@ class DataSourcePublisher:
                     set(search_keys.values()) == {SearchKey.IP_RANGE_FROM, SearchKey.IP_RANGE_TO}
                     or set(search_keys.values()) == {SearchKey.IPV6_RANGE_FROM, SearchKey.IPV6_RANGE_TO}
                     or set(search_keys.values()) == {SearchKey.MSISDN_RANGE_FROM, SearchKey.MSISDN_RANGE_TO}
+                    or snapshot_frequency_days is not None or join_date_abs_limit_days is not None
                 ) and sort_column is None:
                     raise ValidationError("Sort column is required for passed search keys")
-                if (
-                    set(search_keys.values()) == {SearchKey.PHONE, SearchKey.DATE}
-                    and snapshot_frequency_days is None
-                    and join_date_abs_limit_days is None
-                ):
-                    raise ValidationError(
-                        "With MSISDN and DATE keys one of the snapshot_frequency_days or"
-                        " join_date_abs_limit_days parameters is required"
-                    )
                 if (
                     set(search_keys.values()) == {SearchKey.PHONE, SearchKey.DATE}
                     or set(search_keys.values()) == {SearchKey.HEM, SearchKey.DATE}

upgini/dataset.py

@@ -7,6 +7,8 @@ from typing import Any, Callable, Dict, List, Optional, Tuple
 
 import numpy as np
 import pandas as pd
+import pyarrow as pa
+import pyarrow.parquet as pq
 from pandas.api.types import (
     is_float_dtype,
     is_integer_dtype,
@@ -18,6 +20,7 @@ from pandas.api.types import (
 from upgini.errors import ValidationError
 from upgini.http import ProgressStage, SearchProgress, _RestClient
 from upgini.metadata import (
+    CURRENT_DATE_COL,
     ENTITY_SYSTEM_RECORD_ID,
     EVAL_SET_INDEX,
     SYSTEM_RECORD_ID,
@@ -33,11 +36,13 @@ from upgini.metadata import (
     NumericInterval,
     RuntimeParameters,
     SearchCustomization,
+    SearchKey,
 )
 from upgini.resource_bundle import ResourceBundle, get_custom_bundle
 from upgini.search_task import SearchTask
 from upgini.utils.config import SampleConfig
 from upgini.utils.email_utils import EmailSearchKeyConverter
+from upgini.utils.hash_utils import file_hash
 from upgini.utils.sample_utils import SampleColumns, sample
 
 try:
@@ -50,7 +55,7 @@ except Exception:
 
 class Dataset:
     MIN_ROWS_COUNT = 100
-    MAX_ROWS = 200_000
+    MAX_ROWS = 3_000_000
     MIN_SUPPORTED_DATE_TS = 946684800000  # 2000-01-01
     MAX_FEATURES_COUNT = 3500
     MAX_UPLOADING_FILE_SIZE = 268435456  # 256 Mb
@@ -71,6 +76,8 @@ class Dataset:
         date_column: Optional[str] = None,
         id_columns: Optional[List[str]] = None,
         is_imbalanced: bool = False,
+        dropped_columns: Optional[List[str]] = None,
+        autodetected_search_keys: Optional[Dict[str, SearchKey]] = None,
         random_state: Optional[int] = None,
         sample_config: Optional[SampleConfig] = None,
         rest_client: Optional[_RestClient] = None,
@@ -118,6 +125,8 @@ class Dataset:
         self.is_imbalanced: bool = False
         self.id_columns = id_columns
         self.is_imbalanced = is_imbalanced
+        self.dropped_columns = dropped_columns
+        self.autodetected_search_keys = autodetected_search_keys
         self.date_column = date_column
         if logger is not None:
             self.logger = logger
@@ -151,7 +160,9 @@ class Dataset:
     def etalon_def_checked(self) -> Dict[str, str]:
         if self.etalon_def is None:
             self.etalon_def = {
-                v.value: k for k, v in self.meaning_types_checked.items() if v != FileColumnMeaningType.FEATURE
+                v.value: k
+                for k, v in self.meaning_types_checked.items()
+                if v not in [FileColumnMeaningType.FEATURE, FileColumnMeaningType.DATE_FEATURE]
             }
 
         return self.etalon_def
@@ -283,6 +294,7 @@ class Dataset:
             for key in search_group
             if key in self.columns_renaming
             and not self.columns_renaming.get(key).endswith(EmailSearchKeyConverter.ONE_DOMAIN_SUFFIX)
+            and not self.columns_renaming.get(key) == CURRENT_DATE_COL
         }
         ipv4_column = self.etalon_def_checked.get(FileColumnMeaningType.IP_ADDRESS.value)
         if (
@@ -292,10 +304,11 @@ class Dataset:
         ):
             keys_to_validate.remove(ipv4_column)
 
-        mandatory_columns = [target]
+        mandatory_columns = {target} if target is not None else {}
         columns_to_validate = mandatory_columns.copy()
-        columns_to_validate.extend(keys_to_validate)
-        columns_to_validate = set([i for i in columns_to_validate if i is not None])
+        columns_to_validate.update(keys_to_validate)
+        if len(columns_to_validate) == 0:
+            return
 
         nrows = len(self.data)
         validation_stats = {}
@@ -358,7 +371,10 @@ class Dataset:
                 self.data["valid_keys"] = self.data["valid_keys"] + self.data[f"{col}_is_valid"]
             self.data.drop(columns=f"{col}_is_valid", inplace=True)
 
-        self.data["is_valid"] = self.data["valid_keys"] > 0
+        if len(keys_to_validate) > 0:
+            self.data["is_valid"] = self.data["valid_keys"] > 0
+        else:
+            self.data["is_valid"] = True
         self.data["is_valid"] = self.data["is_valid"] & self.data["valid_mandatory"]
         self.data.drop(columns=["valid_keys", "valid_mandatory"], inplace=True)
 
@@ -464,6 +480,37 @@ class Dataset:
 
             columns.append(column_meta)
 
+        current_date = int(pd.Timestamp(pd.Timestamp.now().date(), tz="UTC").timestamp() * 1000)
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            if (
+                self.date_column is not None
+                and self.data[self.date_column].nunique() == 1
+                and self.data[self.date_column].iloc[0] == current_date
+            ):
+                df_without_fake_date = self.data.drop(columns=[self.date_column])
+            else:
+                df_without_fake_date = self.data
+            parquet_file_path = f"{tmp_dir}/{self.dataset_name}.parquet"
+
+            # calculate deterministic digest for any environment
+
+            table = pa.Table.from_pandas(df_without_fake_date, preserve_index=False)
+            table = table.replace_schema_metadata({})  # remove all metadata
+            pq.write_table(
+                table,
+                parquet_file_path,
+                compression=None,  # any compression will make it non-deterministic
+                data_page_size=0,  # optional, to remove page layout variations
+                use_deprecated_int96_timestamps=False,  # fix timestamp format
+                write_statistics=False,  # remove statistics to make it deterministic
+            )
+
+            deterministic_digest = file_hash(parquet_file_path)
+
+        autodetected_search_keys = (
+            {k: v.name for k, v in self.autodetected_search_keys.items()} if self.autodetected_search_keys else None
+        )
+
         return FileMetadata(
             name=self.dataset_name,
             description=self.description,
@@ -473,6 +520,9 @@ class Dataset:
             hierarchicalGroupKeys=self.hierarchical_group_keys,
             hierarchicalSubgroupKeys=self.hierarchical_subgroup_keys,
             taskType=self.task_type,
+            droppedColumns=self.dropped_columns,
+            autodetectedSearchKeys=autodetected_search_keys,
+            deterministicDigest=deterministic_digest,
         )
 
     @staticmethod