upgini 1.1.296__py3-none-any.whl → 1.1.296a3511.dev1__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.1.296"
+__version__ = "1.1.296a3511.dev1"

upgini/autofe/all_operands.py

@@ -1,8 +1,22 @@
 from typing import Dict
 
-from upgini.autofe.binary import Add, Divide, Max, Min, Multiply, Sim, Subtract
+from upgini.autofe.binary import (
+    Add,
+    Combine,
+    CombineThenFreq,
+    Distance,
+    Divide,
+    JaroWinklerSim1,
+    JaroWinklerSim2,
+    LevenshteinSim,
+    Max,
+    Min,
+    Multiply,
+    Sim,
+    Subtract,
+)
 from upgini.autofe.date import DateDiff, DateDiffType2, DateListDiff, DateListDiffBounded, DatePercentile
-from upgini.autofe.groupby import GroupByThenAgg, GroupByThenRank
+from upgini.autofe.groupby import GroupByThenAgg, GroupByThenFreq, GroupByThenNUnique, GroupByThenRank
 from upgini.autofe.operand import Operand
 from upgini.autofe.unary import Abs, Floor, Freq, Log, Residual, Norm, Sigmoid, Sqrt, Square
 from upgini.autofe.vector import Mean, Sum
@@ -32,10 +46,10 @@ ALL_OPERANDS: Dict[str, Operand] = {
         GroupByThenAgg(name="GroupByThenMedian", agg="median"),
         GroupByThenAgg(name="GroupByThenStd", output_type="float", agg="std"),
         GroupByThenRank(),
-        Operand(name="Combine", has_symmetry_importance=True, output_type="object", is_categorical=True),
-        Operand(name="CombineThenFreq", has_symmetry_importance=True, output_type="float"),
-        Operand(name="GroupByThenNUnique", output_type="int", is_vectorizable=True, is_grouping=True),
-        Operand(name="GroupByThenFreq", output_type="float", is_grouping=True),
+        Combine(),
+        CombineThenFreq(),
+        GroupByThenNUnique(),
+        GroupByThenFreq(),
         Sim(),
         DateDiff(),
         DateDiffType2(),
@@ -51,6 +65,10 @@ ALL_OPERANDS: Dict[str, Operand] = {
         DateListDiffBounded(diff_unit="Y", aggregation="count", lower_bound=60),
         DatePercentile(),
         Norm(),
+        JaroWinklerSim1(),
+        JaroWinklerSim2(),
+        LevenshteinSim(),
+        Distance(),
     ]
 }
 

upgini/autofe/binary.py

@@ -1,7 +1,11 @@
+import abc
+from typing import Optional
+import Levenshtein
 import numpy as np
 import pandas as pd
 from numpy import dot
 from numpy.linalg import norm
+from jarowinkler import jarowinkler_similarity
 
 from upgini.autofe.operand import PandasOperand, VectorizableMixin
 
@@ -130,8 +134,8 @@ class CombineThenFreq(PandasOperand):
         self._loc(temp, value_counts)
 
 
-class Sim(PandasOperand):
-    name = "sim"
+class Distance(PandasOperand):
+    name = "dist"
     is_binary = True
     output_type = "float"
     is_symmetrical = True
@@ -139,3 +143,78 @@ class Sim(PandasOperand):
 
     def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
         return dot(left, right) / (norm(left) * norm(right))
+
+
+class Sim(Distance):
+    name = "sim"
+    is_binary = True
+    output_type = "float"
+    is_symmetrical = True
+    has_symmetry_importance = True
+
+
+class StringSim(PandasOperand, abc.ABC):
+    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
+        sims = []
+        for i in left.index:
+            left_i = self._prepare_value(left.get(i))
+            right_i = self._prepare_value(right.get(i))
+            if left_i is not None and right_i is not None:
+                sims.append(self._similarity(left_i, right_i))
+            else:
+                sims.append(None)
+
+        return pd.Series(sims, index=left.index)
+
+    @abc.abstractmethod
+    def _prepare_value(self, value: Optional[str]) -> Optional[str]:
+        pass
+
+    @abc.abstractmethod
+    def _similarity(self, left: str, right: str) -> float:
+        pass
+
+
+class JaroWinklerSim1(StringSim):
+    name = "sim_jw1"
+    is_binary = True
+    input_type = "string"
+    output_type = "float"
+    is_symmetrical = True
+    has_symmetry_importance = True
+
+    def _prepare_value(self, value: Optional[str]) -> Optional[str]:
+        return value
+
+    def _similarity(self, left: str, right: str) -> float:
+        return jarowinkler_similarity(left, right)
+
+
+class JaroWinklerSim2(StringSim):
+    name = "sim_jw2"
+    is_binary = True
+    input_type = "string"
+    output_type = "float"
+    is_symmetrical = True
+    has_symmetry_importance = True
+
+    def _prepare_value(self, value: Optional[str]) -> Optional[str]:
+        return value[::-1] if value is not None else None
+
+    def _similarity(self, left: str, right: str) -> float:
+        return jarowinkler_similarity(left, right)
+
+
+class LevenshteinSim(StringSim):
+    name = "sim_lv"
+    is_binary = True
+    input_type = "string"
+    output_type = "float"
+    is_symmetrical = True
+    has_symmetry_importance = True
+
+    def _prepare_value(self, value: Optional[str]) -> Optional[str]:
+        return value
+
+    def _similarity(self, left: str, right: str) -> float:
+        return 1 - Levenshtein.distance(left, right) / max(len(left), len(right))

upgini/data_source/data_source_publisher.py

@@ -59,35 +59,9 @@ class DataSourcePublisher:
         features_for_embeddings: Optional[List[str]] = DEFAULT_GENERATE_EMBEDDINGS,
         data_table_id_to_replace: Optional[str] = None,
         keep_features: Optional[List[str]] = None,
-        date_features: Optional[List[str]] = None,
-        date_vector_features: Optional[List[str]] = None,
         _force_generation=False,
         _silent=False,
     ) -> str:
-        """Register new ADS
-
-        Parameters
-        ----------
-        data_table_uri - str - table name in format {project_id}.{datasource_name}.{table_name}
-
-        search_keys - dict with column names as keys and SearchKey as value
-
-        update_frequency - str - (Monthly, Weekly, Daily, Annually, Quarterly)
-
-        exclude_from_autofe_generation - optional list of features that should be excluded from AutoFE
-
-        secondary_search_keys - optional dict of secondary search keys
-
-        sort_column - optional str - name of unique column that could be used for sort
-
-        date_format - optional str - format of date if it is present in search keys
-
-        ...
-
-        data_table_id_to_replace - optional str - id of registered ADS that should be replaced by new table
-
-        keep_features - optional list - features that should not be removed from ADS (even if they are personal)
-        """
         trace_id = str(uuid.uuid4())
 
         with MDC(trace_id=trace_id):
@@ -150,14 +124,6 @@ class DataSourcePublisher:
                     request["excludeFromGeneration"] = exclude_from_autofe_generation
                 if keep_features is not None:
                     request["keepFeatures"] = keep_features
-                if date_features is not None:
-                    if date_format is None:
-                        raise ValidationError("date_format should be presented if you use date features")
-                    request["dateFeatures"] = date_features
-                if date_vector_features is not None:
-                    if date_format is None:
-                        raise ValidationError("date_format should be presented if you use date vector features")
-                    request["dateVectorFeatures"] = date_vector_features
                 self.logger.info(f"Start registering data table {request}")
 
                 task_id = self._rest_client.register_ads(request, trace_id)
@@ -215,9 +181,6 @@ class DataSourcePublisher:
                 msg = f"Data table successfully registered with id: {data_table_id}"
                 self.logger.info(msg)
                 print(msg)
-                if "warnings" in status_response and status_response["warnings"]:
-                    self.logger.warning(status_response["warnings"])
-                    print(status_response["warnings"])
                 return data_table_id
             except KeyboardInterrupt:
                 if task_id is not None:

upgini/features_enricher.py

@@ -2870,7 +2870,7 @@ class FeaturesEnricher(TransformerMixin):
                 self.logger.info(f"Dates interval is ({min_date}, {max_date})")
 
         except Exception:
-            self.logger.warning("Failed to log debug information", exc_info=True)
+            self.logger.exception("Failed to log debug information")
 
     def __handle_index_search_keys(self, df: pd.DataFrame, search_keys: Dict[str, SearchKey]) -> pd.DataFrame:
         index_names = df.index.names if df.index.names != [None] else [DEFAULT_INDEX]

{upgini-1.1.296.dist-info → upgini-1.1.296a3511.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.1.296
+Version: 1.1.296a3511.dev1
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.1.296.dist-info → upgini-1.1.296a3511.dev1.dist-info}/RECORD

@@ -1,9 +1,9 @@
-upgini/__about__.py,sha256=VYB3k4Dn9TdkyE_1qiCFNhgHJH5heaX4R3m8VNi495g,24
+upgini/__about__.py,sha256=cimRDJH6AT3BICa6gKJosTY9mQiinR-sub8yCsdaiuY,34
 upgini/__init__.py,sha256=ObEtjFkIssl83qeKNMLpIQygfwK8TzztwiI43YTsAP0,353
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
 upgini/dataset.py,sha256=7TLVVhGtjgx_9yaiaIUK3kZSe_R9wg5dY0d4F5qCGM4,45636
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=HQFLw3VyEsZfAt4xFnIYOnp3fzQSHAsyHzIm0gTJpOI,177543
+upgini/features_enricher.py,sha256=rLy6BwhL94VkRCp8W4RxJ0lBb7sZqTOsz-bUVI8nXZU,177530
 upgini/http.py,sha256=bp6jWl422Icy3AhHMdCcJv5NjExE45gSMmzMTPJjPuk,42600
 upgini/lazy_import.py,sha256=EwoM0msNGbSmWBhGbrLDny1DSnOlvTxCjmMKPxYlDms,610
 upgini/metadata.py,sha256=qDAIO7NLSSQp_XiXCv3U4XJTLO0KH3YuQ8lvCLYPqzs,9781
@@ -14,8 +14,8 @@ upgini/version_validator.py,sha256=ddSKUK_-eGJB3NgrqOMoWJU-OxQ253WsNLp8aqJkaIM,1
 upgini/ads_management/__init__.py,sha256=qzyisOToVRP-tquAJD1PblZhNtMrOB8FiyF9JvfkvgE,50
 upgini/ads_management/ads_manager.py,sha256=igVbN2jz80Umb2BUJixmJVj-zx8unoKpecVo-R-nGdw,2648
 upgini/autofe/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-upgini/autofe/all_operands.py,sha256=cpwUfhZWF9QBfrUyJ0xZ72iGYyt1eXIZQ46FB-7ZDI4,2421
-upgini/autofe/binary.py,sha256=8FXPJxN7fnC5wphO0Dp1tQCa0lFMSDGQGvBMkSIVAcE,4155
+upgini/autofe/all_operands.py,sha256=ZS9A7u1gV7Bt8tHFfCiNx9u4q85I1ny4NIL-keyfWHY,2423
+upgini/autofe/binary.py,sha256=P46AVcFRiMKYucK3_N34QDqVMbvv2OuyWAREx0bUNwg,6367
 upgini/autofe/date.py,sha256=qzk0NT332Q0vR1eRwTuNiMSrGE3ulh6Ic3QLBZqSdvw,7284
 upgini/autofe/feature.py,sha256=_V9B74B3ue7eAYXSOt9JKhVC9klkAKks22MwnBRye_w,12487
 upgini/autofe/groupby.py,sha256=4WjDzQxqpZxB79Ih4ihMMI5GDxaFqiH6ZelfV82ClT4,3091
@@ -23,7 +23,7 @@ upgini/autofe/operand.py,sha256=MKEsl3zxpWzRDpTkE0sNJxTu62U20sWOvEKhPjUWS6s,2915
 upgini/autofe/unary.py,sha256=ZWjLd-CUkNt_PpM8YuWLLipW1v_RdBlsl4JxXIVo9aM,3652
 upgini/autofe/vector.py,sha256=dLxfAstJs-gw_OQ1xxoxcM6pVzORlV0HVzdzt7cLXVQ,606
 upgini/data_source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-upgini/data_source/data_source_publisher.py,sha256=1cQZrK630VztwGGDp41ec9gqIeUtkefaqSSQEitVWiM,19581
+upgini/data_source/data_source_publisher.py,sha256=6paupnciqagACnSzjGSkA2a5i-c9ETvZheLqBYOJemk,17810
 upgini/mdc/__init__.py,sha256=aM08nIWFc2gWdWUa3_IuEnNND0cQPkBGnYpRMnfFN8k,1019
 upgini/mdc/context.py,sha256=3u1B-jXt7tXEvNcV3qmR9SDCseudnY7KYsLclBdwVLk,1405
 upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -57,7 +57,7 @@ upgini/utils/sklearn_ext.py,sha256=13jQS_k7v0aUtudXV6nGUEWjttPQzAW9AFYL5wgEz9k,4
 upgini/utils/target_utils.py,sha256=Y96_PJ5cC-WsEbeqg20v9uqywDQobLoTb-xoP7S3o4E,7807
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/warning_counter.py,sha256=dIWBB4dI5XRRJZudvIlqlIYKEiwLLPcXarsZuYRt338,227
-upgini-1.1.296.dist-info/METADATA,sha256=4eJQUEcmU8LjaLKmvkZ4OPJBat7dPkBI0GwnNKpBZcc,48117
-upgini-1.1.296.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-upgini-1.1.296.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.1.296.dist-info/RECORD,,
+upgini-1.1.296a3511.dev1.dist-info/METADATA,sha256=_tPLQLuHI0_j5lGzkGhMSftXbfpLi8aCTRpjT4fd8QQ,48127
+upgini-1.1.296a3511.dev1.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
+upgini-1.1.296a3511.dev1.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.1.296a3511.dev1.dist-info/RECORD,,

{upgini-1.1.296.dist-info → upgini-1.1.296a3511.dev1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.25.0
+Generator: hatchling 1.24.2
 Root-Is-Purelib: true
 Tag: py3-none-any