upgini 1.1.274a4__py3-none-any.whl → 1.1.280.dev0__py3-none-any.whl

upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.1.280.dev0"

upgini/ads.py

@@ -5,7 +5,7 @@ from typing import Dict, Optional
 
 import numpy as np
 import pandas as pd
-from pandas.api.types import is_string_dtype
+from pandas.api.types import is_object_dtype, is_string_dtype
 
 from upgini import SearchKey
 from upgini.http import get_rest_client
@@ -34,7 +34,11 @@ def upload_user_ads(name: str, df: pd.DataFrame, search_keys: Dict[str, SearchKe
             if df[column_name].notnull().sum() < min_valid_rows_count:
                 raise ValueError(bundle.get("ads_upload_to_many_empty_rows"))
             meaning_type = search_keys[column_name].value
-            if meaning_type == FileColumnMeaningType.MSISDN and not is_string_dtype(df[column_name]):
+            if (
+                meaning_type == FileColumnMeaningType.MSISDN
+                and not is_string_dtype(df[column_name])
+                and not is_object_dtype(df[column_name])
+            ):
                 df[column_name] = df[column_name].values.astype(np.int64).astype("string")  # type: ignore
         else:
             meaning_type = FileColumnMeaningType.FEATURE

upgini/ads_management/ads_manager.py

@@ -1,9 +1,11 @@
 import time
-from typing import Dict, Optional
 import uuid
+from typing import Dict, Optional
+
+import pandas as pd
+
 from upgini.http import get_rest_client
 from upgini.spinner import Spinner
-import pandas as pd
 
 
 class AdsManager:

upgini/autofe/all_operands.py

@@ -1,9 +1,10 @@
 from typing import Dict
+
+from upgini.autofe.binary import Add, Divide, Max, Min, Multiply, Sim, Subtract
 from upgini.autofe.date import DateDiff, DateDiffType2, DateListDiff, DateListDiffBounded
 from upgini.autofe.groupby import GroupByThenAgg, GroupByThenRank
 from upgini.autofe.operand import Operand
-from upgini.autofe.unary import Abs, Log, Residual, Sqrt, Square, Sigmoid, Floor, Freq
-from upgini.autofe.binary import Min, Max, Add, Subtract, Multiply, Divide, Sim
+from upgini.autofe.unary import Abs, Floor, Freq, Log, Residual, Sigmoid, Sqrt, Square
 from upgini.autofe.vector import Mean, Sum
 
 ALL_OPERANDS: Dict[str, Operand] = {

upgini/autofe/binary.py

@@ -1,9 +1,10 @@
-from upgini.autofe.operand import PandasOperand, VectorizableMixin
 import numpy as np
 import pandas as pd
 from numpy import dot
 from numpy.linalg import norm
 
+from upgini.autofe.operand import PandasOperand, VectorizableMixin
+
 
 class Min(PandasOperand):
     name = "min"

upgini/autofe/date.py

@@ -1,6 +1,8 @@
 from typing import Any, Optional, Union
+
 import numpy as np
 import pandas as pd
+from pandas.core.arrays.timedeltas import TimedeltaArray
 from pydantic import BaseModel
 
 from upgini.autofe.operand import PandasOperand
@@ -73,8 +75,13 @@ class DateListDiff(PandasOperand, DateDiffMixin):
 
         return pd.Series(left - right.values).apply(lambda x: self._agg(self._diff(x)))
 
-    def _diff(self, x):
-        x = x / np.timedelta64(1, self.diff_unit)
+    def _diff(self, x: TimedeltaArray):
+        if self.diff_unit == "Y":
+            x = (x / 365 / 24 / 60 / 60 / 10**9).astype(int)
+        elif self.diff_unit == "M":
+            raise Exception("Unsupported difference unit: Month")
+        else:
+            x = x / np.timedelta64(1, self.diff_unit)
         return x[x > 0]
 
     def _agg(self, x):

upgini/autofe/feature.py

@@ -215,7 +215,7 @@ class Feature:
             return Column(string)
 
         def is_trivial_char(c: str) -> bool:
-            return not (c in "()+-*/,")
+            return c not in "()+-*/,"
 
         def find_prev(string: str) -> int:
             if string[-1] != ")":

upgini/autofe/groupby.py

@@ -1,7 +1,9 @@
-from upgini.autofe.operand import PandasOperand, VectorizableMixin
 from typing import Optional
+
 import pandas as pd
 
+from upgini.autofe.operand import PandasOperand, VectorizableMixin
+
 
 class GroupByThenAgg(PandasOperand, VectorizableMixin):
     agg: Optional[str]

upgini/autofe/operand.py

@@ -1,8 +1,9 @@
-from pydantic import BaseModel
-from typing import Dict, List, Optional, Tuple, Union
 import abc
-import pandas as pd
+from typing import Dict, List, Optional, Tuple, Union
+
 import numpy as np
+import pandas as pd
+from pydantic import BaseModel
 
 
 class Operand(BaseModel):

upgini/autofe/unary.py

@@ -1,7 +1,8 @@
-from upgini.autofe.operand import PandasOperand, VectorizableMixin
 import numpy as np
 import pandas as pd
 
+from upgini.autofe.operand import PandasOperand, VectorizableMixin
+
 
 class Abs(PandasOperand, VectorizableMixin):
     name = "abs"

upgini/autofe/vector.py

@@ -1,5 +1,7 @@
 from typing import List
+
 import pandas as pd
+
 from upgini.autofe.operand import PandasOperand, VectorizableMixin
 
 

upgini/dataset.py

@@ -15,6 +15,7 @@ from pandas.api.types import (
     is_float_dtype,
     is_integer_dtype,
     is_numeric_dtype,
+    is_object_dtype,
     is_period_dtype,
     is_string_dtype,
 )
@@ -94,7 +95,7 @@ class Dataset:  # (pd.DataFrame):
                 data = pd.read_csv(path, **kwargs)
             else:
                 # try different separators: , ; \t ...
-                with open(path, mode="r") as csvfile:
+                with open(path) as csvfile:
                     sep = csv.Sniffer().sniff(csvfile.read(2048)).delimiter
                 kwargs["sep"] = sep
                 data = pd.read_csv(path, **kwargs)
@@ -219,7 +220,7 @@ class Dataset:  # (pd.DataFrame):
         """Check that string values less than maximum characters for LLM"""
         # self.logger.info("Validate too long string values")
         for col in self.data.columns:
-            if is_string_dtype(self.data[col]):
+            if is_string_dtype(self.data[col]) or is_object_dtype(self.data[col]):
                 max_length: int = self.data[col].astype("str").str.len().max()
                 if max_length > self.MAX_STRING_FEATURE_LENGTH:
                     self.data[col] = self.data[col].astype("str").str.slice(stop=self.MAX_STRING_FEATURE_LENGTH)
@@ -250,7 +251,7 @@ class Dataset:  # (pd.DataFrame):
     @staticmethod
     def _ip_to_int(ip: Optional[_BaseAddress]) -> Optional[int]:
         try:
-            if isinstance(ip, IPv4Address) or isinstance(ip, IPv6Address):
+            if isinstance(ip, (IPv4Address, IPv6Address)):
                 return int(ip)
         except Exception:
             pass
@@ -258,7 +259,7 @@ class Dataset:  # (pd.DataFrame):
     @staticmethod
     def _ip_to_int_str(ip: Optional[_BaseAddress]) -> Optional[str]:
         try:
-            if isinstance(ip, IPv4Address) or isinstance(ip, IPv6Address):
+            if isinstance(ip, (IPv4Address, IPv6Address)):
                 return str(int(ip))
         except Exception:
             pass
@@ -350,7 +351,7 @@ class Dataset:  # (pd.DataFrame):
         if postal_code is not None and postal_code in self.data.columns:
             # self.logger.info("Normalize postal code")
 
-            if is_string_dtype(self.data[postal_code]):
+            if is_string_dtype(self.data[postal_code]) or is_object_dtype(self.data[postal_code]):
                 try:
                     self.data[postal_code] = self.data[postal_code].astype("float64").astype("Int64").astype("string")
                 except Exception:
@@ -821,7 +822,7 @@ class Dataset:  # (pd.DataFrame):
             return DataType.INT
         elif is_float_dtype(pandas_data_type):
             return DataType.DECIMAL
-        elif is_string_dtype(pandas_data_type):
+        elif is_string_dtype(pandas_data_type) or is_object_dtype(pandas_data_type):
             return DataType.STRING
         else:
             msg = self.bundle.get("dataset_invalid_column_type").format(column_name, pandas_data_type)

upgini/errors.py

@@ -16,7 +16,7 @@ class UnauthorizedError(HttpError):
     """Unauthorized error from REST API."""
 
     def __init__(self, message, status_code):
-        message = "Unauthorized, please check your authorization token ({})".format(message)
+        message = f"Unauthorized, please check your authorization token ({message})"
         super(UnauthorizedError, self).__init__(message, status_code)
 
 

upgini/features_enricher.py

@@ -21,6 +21,7 @@ from pandas.api.types import (
     is_bool,
     is_datetime64_any_dtype,
     is_numeric_dtype,
+    is_object_dtype,
     is_period_dtype,
     is_string_dtype,
 )
@@ -424,6 +425,9 @@ class FeaturesEnricher(TransformerMixin):
 
             self.__validate_search_keys(self.search_keys, self.search_id)
 
+            # Validate client estimator params
+            self._get_client_cat_features(estimator, X, self.search_keys)
+
             try:
                 self.X = X
                 self.y = y
@@ -817,6 +821,7 @@ class FeaturesEnricher(TransformerMixin):
         trace_id = trace_id or str(uuid.uuid4())
         start_time = time.time()
         with MDC(trace_id=trace_id):
+            self.logger.info("Start calculate metrics")
             if len(args) > 0:
                 msg = f"WARNING: Unsupported positional arguments for calculate_metrics: {args}"
                 self.logger.warning(msg)
@@ -868,22 +873,9 @@ class FeaturesEnricher(TransformerMixin):
                     self.__display_support_link(msg)
                     return None
 
-                cat_features = None
-                search_keys_for_metrics = []
-                if (
-                    estimator is not None
-                    and hasattr(estimator, "get_param")
-                    and estimator.get_param("cat_features") is not None
-                ):
-                    cat_features = estimator.get_param("cat_features")
-                    if len(cat_features) > 0 and isinstance(cat_features[0], int):
-                        cat_features = [effective_X.columns[i] for i in cat_features]
-                        for cat_feature in cat_features:
-                            if cat_feature in self.search_keys:
-                                if self.search_keys[cat_feature] in [SearchKey.COUNTRY, SearchKey.POSTAL_CODE]:
-                                    search_keys_for_metrics.append(cat_feature)
-                                else:
-                                    raise ValidationError(self.bundle.get("cat_feature_search_key").format(cat_feature))
+                cat_features, search_keys_for_metrics = self._get_client_cat_features(
+                    estimator, effective_X, self.search_keys
+                )
 
                 prepared_data = self._prepare_data_for_metrics(
                     trace_id=trace_id,
@@ -898,6 +890,7 @@ class FeaturesEnricher(TransformerMixin):
                     search_keys_for_metrics=search_keys_for_metrics,
                     progress_bar=progress_bar,
                     progress_callback=progress_callback,
+                    cat_features=cat_features,
                 )
                 if prepared_data is None:
                     return None
@@ -1273,6 +1266,29 @@ class FeaturesEnricher(TransformerMixin):
 
         return _cv, groups
 
+    def _get_client_cat_features(
+        self, estimator: Optional[Any], X: pd.DataFrame, search_keys: Dict[str, SearchKey]
+    ) -> Optional[List[str]]:
+        cat_features = None
+        search_keys_for_metrics = []
+        if (
+            estimator is not None
+            and hasattr(estimator, "get_param")
+            and estimator.get_param("cat_features") is not None
+        ):
+            cat_features = estimator.get_param("cat_features")
+            if len(cat_features) > 0:
+                if all([isinstance(f, int) for f in cat_features]):
+                    cat_features = [X.columns[i] for i in cat_features]
+                self.logger.info(f"Collected categorical features {cat_features} from user estimator")
+                for cat_feature in cat_features:
+                    if cat_feature in search_keys:
+                        if search_keys[cat_feature] in [SearchKey.COUNTRY, SearchKey.POSTAL_CODE]:
+                            search_keys_for_metrics.append(cat_feature)
+                        else:
+                            raise ValidationError(self.bundle.get("cat_feature_search_key").format(cat_feature))
+        return cat_features, search_keys_for_metrics
+
     def _prepare_data_for_metrics(
         self,
         trace_id: str,
@@ -1287,6 +1303,7 @@ class FeaturesEnricher(TransformerMixin):
         search_keys_for_metrics: Optional[List[str]] = None,
         progress_bar: Optional[ProgressBar] = None,
         progress_callback: Optional[Callable[[SearchProgress], Any]] = None,
+        cat_features: Optional[List[str]] = None,
     ):
         is_input_same_as_fit, X, y, eval_set = self._is_input_same_as_fit(X, y, eval_set)
         is_demo_dataset = hash_input(X, y, eval_set) in DEMO_DATASET_HASHES
@@ -1316,6 +1333,9 @@ class FeaturesEnricher(TransformerMixin):
         excluding_search_keys = list(search_keys.keys())
         if search_keys_for_metrics is not None and len(search_keys_for_metrics) > 0:
             excluding_search_keys = [sk for sk in excluding_search_keys if sk not in search_keys_for_metrics]
+        meta = self._search_task.get_all_features_metadata_v2()
+        zero_importance_client_features = [m for m in meta if m.source == "etalon" and m.shap_value == 0.0]
+
         client_features = [
             c
             for c in X_sampled.columns.to_list()
@@ -1324,6 +1344,7 @@ class FeaturesEnricher(TransformerMixin):
                 excluding_search_keys
                 + list(self.fit_dropped_features)
                 + [DateTimeSearchKeyConverter.DATETIME_COL, SYSTEM_RECORD_ID]
+                + zero_importance_client_features
             )
         ]
 
@@ -1344,9 +1365,8 @@ class FeaturesEnricher(TransformerMixin):
 
         # Detect and drop high cardinality columns in train
         columns_with_high_cardinality = FeaturesValidator.find_high_cardinality(fitting_X)
-        columns_with_high_cardinality = [
-            c for c in columns_with_high_cardinality if c not in (self.generate_features or [])
-        ]
+        non_excluding_columns = (self.generate_features or []) + (cat_features or [])
+        columns_with_high_cardinality = [c for c in columns_with_high_cardinality if c not in non_excluding_columns]
         if len(columns_with_high_cardinality) > 0:
             self.logger.warning(
                 f"High cardinality columns {columns_with_high_cardinality} will be dropped for metrics calculation"
@@ -2532,7 +2552,7 @@ class FeaturesEnricher(TransformerMixin):
             validated_X = X.copy()
         elif isinstance(X, pd.Series):
             validated_X = X.to_frame()
-        elif isinstance(X, np.ndarray) or isinstance(X, list):
+        elif isinstance(X, (list, np.ndarray)):
             validated_X = pd.DataFrame(X)
             renaming = {c: str(c) for c in validated_X.columns}
             validated_X = validated_X.rename(columns=renaming)
@@ -2621,7 +2641,7 @@ class FeaturesEnricher(TransformerMixin):
             validated_eval_X = eval_X.copy()
         elif isinstance(eval_X, pd.Series):
             validated_eval_X = eval_X.to_frame()
-        elif isinstance(eval_X, np.ndarray) or isinstance(eval_X, list):
+        elif isinstance(eval_X, (list, np.ndarray)):
             validated_eval_X = pd.DataFrame(eval_X)
             renaming = {c: str(c) for c in validated_eval_X.columns}
             validated_eval_X = validated_eval_X.rename(columns=renaming)
@@ -2803,7 +2823,7 @@ class FeaturesEnricher(TransformerMixin):
             )
 
             def sample(df):
-                if isinstance(df, pd.Series) or isinstance(df, pd.DataFrame):
+                if isinstance(df, (pd.DataFrame, pd.Series)):
                     return df.head(10)
                 else:
                     return df[:10]
@@ -2829,8 +2849,10 @@ class FeaturesEnricher(TransformerMixin):
             maybe_date_col = self._get_date_column(self.search_keys)
             if X is not None and maybe_date_col is not None and maybe_date_col in X.columns:
                 # TODO cast date column to single dtype
-                min_date = X[maybe_date_col].min()
-                max_date = X[maybe_date_col].max()
+                date_converter = DateTimeSearchKeyConverter(maybe_date_col, self.date_format)
+                converted_X = date_converter.convert(X)
+                min_date = converted_X[maybe_date_col].min()
+                max_date = converted_X[maybe_date_col].max()
                 self.logger.info(f"Dates interval is ({min_date}, {max_date})")
 
         except Exception:
@@ -2967,7 +2989,7 @@ class FeaturesEnricher(TransformerMixin):
 
     def __correct_target(self, df: pd.DataFrame) -> pd.DataFrame:
         target = df[self.TARGET_NAME]
-        if is_string_dtype(target):
+        if is_string_dtype(target) or is_object_dtype(target):
             maybe_numeric_target = pd.to_numeric(target, errors="coerce")
             # If less than 5% is non numeric then leave this rows with NaN target and later it will be dropped
             if maybe_numeric_target.isna().sum() <= _num_samples(df) * 0.05:
@@ -3240,6 +3262,8 @@ class FeaturesEnricher(TransformerMixin):
             descriptions = []
             for m in autofe_meta:
                 autofe_feature = Feature.from_formula(m.formula)
+                orig_to_hashed = {base_column.original_name: base_column.hashed_name for base_column in m.base_columns}
+                autofe_feature.rename_columns(orig_to_hashed)
                 autofe_feature.set_display_index(m.display_index)
                 if autofe_feature.op.is_vector:
                     continue
@@ -3367,7 +3391,8 @@ class FeaturesEnricher(TransformerMixin):
                 valid_search_keys[column_name] = SearchKey.CUSTOM_KEY
             else:
                 if x[column_name].isnull().all() or (
-                    is_string_dtype(x[column_name]) and (x[column_name].astype("string").str.strip() == "").all()
+                    (is_string_dtype(x[column_name]) or is_object_dtype(x[column_name]))
+                    and (x[column_name].astype("string").str.strip() == "").all()
                 ):
                     raise ValidationError(self.bundle.get("empty_search_key").format(column_name))
 
@@ -3674,7 +3699,7 @@ class FeaturesEnricher(TransformerMixin):
                 def sample(inp, sample_index):
                     if _num_samples(inp) <= 1000:
                         return inp
-                    if isinstance(inp, pd.DataFrame) or isinstance(inp, pd.Series):
+                    if isinstance(inp, (pd.DataFrame, pd.Series)):
                         return inp.sample(n=1000, random_state=random_state)
                     if isinstance(inp, np.ndarray):
                         return inp[sample_index]

upgini/http.py

@@ -22,6 +22,7 @@ from pydantic import BaseModel
 from pythonjsonlogger import jsonlogger
 from requests.exceptions import RequestException
 
+from upgini.__about__ import __version__
 from upgini.errors import (
     HttpError,
     UnauthorizedError,
@@ -38,17 +39,17 @@ from upgini.metadata import (
 from upgini.resource_bundle import bundle
 from upgini.utils.track_info import get_track_metrics
 
-try:
-    from importlib_metadata import version  # type: ignore
+# try:
+#     from importlib.metadata import version  # type: ignore
 
-    __version__ = version("upgini")
-except ImportError:
-    try:
-        from importlib.metadata import version  # type: ignore
+#     __version__ = version("upgini")
+# except ImportError:
+#     try:
+#         from importlib_metadata import version  # type: ignore
 
-        __version__ = version("upgini")
-    except ImportError:
-        __version__ = "Upgini wasn't installed"
+#         __version__ = version("upgini")
+#     except ImportError:
+#         __version__ = "Upgini wasn't installed"
 
 UPGINI_URL: str = "UPGINI_URL"
 UPGINI_API_KEY: str = "UPGINI_API_KEY"
@@ -925,7 +926,7 @@ def is_demo_api_key(api_token: Optional[str]) -> bool:
     return api_token is None or api_token == "" or api_token == DEMO_API_KEY
 
 
-@lru_cache()
+@lru_cache
 def _get_rest_client(
     backend_url: str, api_token: str, client_ip: Optional[str] = None, client_visitorid: Optional[str] = None
 ) -> _RestClient:

upgini/mdc/__init__.py

@@ -1,15 +1,13 @@
-# -*- coding: utf-8 -*-
 """
 .. module: mdc
 .. moduleauthor:: Aljosha Friemann a.friemann@automate.wtf
 """
-from __future__ import absolute_import, division, print_function, unicode_literals
 
 import logging
 
-from upgini.mdc.context import new_log_context, get_mdc_fields
 from pythonjsonlogger import jsonlogger
 
+from upgini.mdc.context import get_mdc_fields, new_log_context
 
 MDContext = new_log_context
 MDC = new_log_context

upgini/mdc/context.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """
 .. module: TODO
     :platform: TODO
@@ -7,12 +6,11 @@
 .. moduleauthor:: Aljosha Friemann a.friemann@automate.wtf
 """
 
-import time
-import uuid
+import collections
 import logging
 import threading
-import collections
-
+import time
+import uuid
 from contextlib import contextmanager
 
 LOGGER = logging.getLogger(__name__)
@@ -32,7 +30,7 @@ def get_mdc_fields():
 
 @contextmanager
 def new_log_context(**kwargs):
-    context_id = "mdc-{thread}-{context}".format(thread=threading.current_thread().ident, context=uuid.uuid4())
+    context_id = f"mdc-{threading.current_thread().ident}-{uuid.uuid4()}"
 
     LOGGER.debug("creating context %s", context_id)
 

upgini/metadata.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from enum import Enum
 from typing import Dict, List, Optional, Set
 
@@ -201,6 +203,7 @@ class FileMetadata(BaseModel):
         for c in self.columns:
             if c.name == name:
                 return c
+        return None
 
     def search_types(self) -> Set[SearchKey]:
         search_keys = set()