upgini 1.1.274a3388.post2__tar.gz → 1.1.275a1__tar.gz

{upgini-1.1.274a3388.post2/src/upgini.egg-info → upgini-1.1.275a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.274a3388.post2
+Version: 1.1.275a1
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers

{upgini-1.1.274a3388.post2 → upgini-1.1.275a1}/setup.py

@@ -40,7 +40,7 @@ def send_log(msg: str):
 
 
 here = Path(__file__).parent.resolve()
-version = "1.1.274a3388-2"
+version = "1.1.275a1"
 try:
     send_log(f"Start setup PyLib version {version}")
     setup(

{upgini-1.1.274a3388.post2 → upgini-1.1.275a1}/src/upgini/autofe/date.py

@@ -1,9 +1,9 @@
-from typing import Any, List, Optional, Union
+from typing import Any, Optional, Union
 import numpy as np
 import pandas as pd
 from pydantic import BaseModel
 
-from upgini.autofe.operand import MultiOperand, Operand, PandasOperand
+from upgini.autofe.operand import PandasOperand
 
 
 class DateDiffMixin(BaseModel):
@@ -24,7 +24,6 @@ class DateDiff(PandasOperand, DateDiffMixin):
     name = "date_diff"
     is_binary = True
     has_symmetry_importance = True
-    common_type = "date_diff"
 
     def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
         left = self._convert_to_date(left, self.left_unit)
@@ -40,7 +39,6 @@ class DateDiffType2(PandasOperand, DateDiffMixin):
     name = "date_diff_type2"
     is_binary = True
     has_symmetry_importance = True
-    common_type = "date_diff"
 
     def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
         left = self._convert_to_date(left, self.left_unit)
@@ -62,7 +60,6 @@ class DateListDiff(PandasOperand, DateDiffMixin):
     is_binary = True
     has_symmetry_importance = True
     aggregation: str
-    common_type = "date_diff_list"
 
     def __init__(self, **data: Any) -> None:
         if "name" not in data:
@@ -89,9 +86,6 @@ class DateListDiff(PandasOperand, DateDiffMixin):
 
         return method(x) if len(x) > 0 else default
 
-    def make_multi_operand(self, operands: List[Operand]) -> Optional[MultiOperand]:
-        return DateListDiffMulti(children=operands, aggregation="")
-
 
 class DateListDiffBounded(DateListDiff):
     lower_bound: Optional[int]
@@ -114,12 +108,3 @@ class DateListDiffBounded(DateListDiff):
     def _agg(self, x):
         x = x[(x >= (self.lower_bound or -np.inf)) & (x < (self.upper_bound or np.inf))]
         return super()._agg(x)
-
-
-class DateListDiffMulti(DateListDiff, MultiOperand):
-    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
-        left = self._convert_to_date(left, self.left_unit)
-        right = right.apply(lambda x: pd.arrays.DatetimeArray(self._convert_to_date(x, self.right_unit)))
-
-        diff = pd.Series(left - right.values).apply(self._diff)
-        return diff.apply(lambda x: [c._agg(x) for c in self.children])

{upgini-1.1.274a3388.post2 → upgini-1.1.275a1}/src/upgini/autofe/feature.py

@@ -1,15 +1,13 @@
 import hashlib
 import itertools
-import operator
 from typing import Dict, List, Optional, Tuple, Union
 
-from more_itertools import map_reduce
 import numpy as np
 import pandas as pd
 from pandas._typing import DtypeObj
 
 from upgini.autofe.all_operands import find_op
-from upgini.autofe.operand import MultiOperand, Operand, PandasOperand
+from upgini.autofe.operand import Operand, PandasOperand
 
 
 class Column:
@@ -31,9 +29,6 @@ class Column:
         else:
             return feature_name[2:last_component_idx]
 
-    def get_display_name(self, **kwargs):
-        return self.name
-
     def delete_data(self):
         self.data = None
 
@@ -162,8 +157,6 @@ class Feature:
         else:
             new_data = new_data.replace([-np.inf, np.inf], np.nan)
 
-        new_data = new_data.rename(self.get_display_name())
-
         if is_root:
             self.data = new_data
         return new_data
@@ -333,65 +326,3 @@ class FeatureGroup:
             self.main_column_node.delete_data()
         for child in self.children:
             child.delete_data()
-
-
-class OperandGroup:
-    def __init__(self, operand: MultiOperand, children: List[Union[Column, Feature]]):
-        self.op = operand
-        self.children = children
-        self.data: Optional[pd.DataFrame] = None
-
-    def get_columns(self, **kwargs) -> List[str]:
-        column_list = []
-        seen = set()
-        for child in self.children:
-            columns = child.get_columns(**kwargs)
-            column_list.extend([f for f in columns if f not in seen])
-            seen.update(columns)
-        return column_list
-
-    def get_display_names(self, **kwargs) -> List[str]:
-        names = [Feature(op, self.children).get_display_name(**kwargs) for op in self.op.children]
-        return names
-
-    def calculate(self, data: pd.DataFrame, is_root=False) -> pd.DataFrame:
-        if isinstance(self.op, PandasOperand):
-            if self.op.is_vector:
-                ds = [child.calculate(data) for child in self.children]
-                new_data = self.op.calculate(data=ds)
-            else:
-                d1 = self.children[0].calculate(data)
-                d2 = None if len(self.children) < 2 else self.children[1].calculate(data)
-                new_data = self.op.calculate(data=d1, left=d1, right=d2)
-
-            new_data = pd.DataFrame(new_data.values.tolist())
-            new_data.columns = self.get_display_names()
-        else:
-            raise NotImplementedError(f"Unrecognized operator {self.op.name}.")
-
-        if is_root:
-            self.data = new_data
-        return new_data
-
-    @staticmethod
-    def make_groups(candidates: List[Feature]) -> List[Union[Feature, "FeatureGroup"]]:
-        grouped_features = []
-
-        for _, features in sorted(
-            map_reduce(
-                candidates, lambda f: (f.op.common_type or "", ",".join([c.get_display_name() for c in f.children]))
-            ).items(),
-            key=operator.itemgetter(0),
-        ):
-            feature_list = list(features)
-            multi_op = feature_list[0].op.make_multi_operand([f.op for f in feature_list])
-            if multi_op is not None:
-                grouped_features.append(OperandGroup(multi_op, feature_list[0].children))
-            else:
-                grouped_features.extend(feature_list)
-        return grouped_features
-
-    def delete_data(self):
-        self.data = None
-        for child in self.children:
-            child.delete_data()

{upgini-1.1.274a3388.post2 → upgini-1.1.275a1}/src/upgini/autofe/operand.py

@@ -5,9 +5,6 @@ import pandas as pd
 import numpy as np
 
 
-MAIN_COLUMN = "main_column"
-
-
 class Operand(BaseModel):
     name: str
     alias: Optional[str]
@@ -21,7 +18,6 @@ class Operand(BaseModel):
     is_binary: bool = False
     is_vector: bool = False
     is_distribution_dependent: bool = False
-    common_type: Optional[str] = None
     params: Optional[Dict[str, str]]
 
     def set_params(self, params: Dict[str, str]):
@@ -31,8 +27,8 @@ class Operand(BaseModel):
     def get_params(self) -> Dict[str, str]:
         return self.params
 
-    def make_multi_operand(self, operands: List["Operand"]) -> Optional["MultiOperand"]:
-        return None
+
+MAIN_COLUMN = "main_column"
 
 
 class PandasOperand(Operand, abc.ABC):
@@ -86,7 +82,3 @@ class VectorizableMixin(Operand):
         value_columns = [col for col in input_columns if col != group_column]
 
         return group_column, value_columns
-
-
-class MultiOperand(Operand):
-    children: List[Operand]

{upgini-1.1.274a3388.post2 → upgini-1.1.275a1}/src/upgini/dataset.py

@@ -22,7 +22,9 @@ from pandas.api.types import (
 from upgini.errors import ValidationError
 from upgini.http import ProgressStage, SearchProgress, _RestClient
 from upgini.metadata import (
+    ENTITY_SYSTEM_RECORD_ID,
     EVAL_SET_INDEX,
+    SEARCH_KEY_UNNEST,
     SYSTEM_COLUMNS,
     SYSTEM_RECORD_ID,
     TARGET,
@@ -78,6 +80,7 @@ class Dataset:  # (pd.DataFrame):
         path: Optional[str] = None,
         meaning_types: Optional[Dict[str, FileColumnMeaningType]] = None,
         search_keys: Optional[List[Tuple[str, ...]]] = None,
+        unnest_search_keys: Optional[List[str]] = None,
         model_task_type: Optional[ModelTaskType] = None,
         random_state: Optional[int] = None,
         rest_client: Optional[_RestClient] = None,
@@ -112,6 +115,7 @@ class Dataset:  # (pd.DataFrame):
         self.description = description
         self.meaning_types = meaning_types
         self.search_keys = search_keys
+        self.unnest_search_keys = unnest_search_keys
         self.ignore_columns = []
         self.hierarchical_group_keys = []
         self.hierarchical_subgroup_keys = []
@@ -171,7 +175,7 @@ class Dataset:  # (pd.DataFrame):
         new_columns = []
         dup_counter = 0
         for column in self.data.columns:
-            if column in [TARGET, EVAL_SET_INDEX, SYSTEM_RECORD_ID]:
+            if column in [TARGET, EVAL_SET_INDEX, SYSTEM_RECORD_ID, ENTITY_SYSTEM_RECORD_ID, SEARCH_KEY_UNNEST]:
                 self.columns_renaming[column] = column
                 new_columns.append(column)
                 continue
@@ -352,7 +356,9 @@ class Dataset:  # (pd.DataFrame):
 
             if is_string_dtype(self.data[postal_code]):
                 try:
-                    self.data[postal_code] = self.data[postal_code].astype("float64").astype("Int64").astype("string")
+                    self.data[postal_code] = (
+                        self.data[postal_code].astype("string").astype("Float64").astype("Int64").astype("string")
+                    )
                 except Exception:
                     pass
             elif is_float_dtype(self.data[postal_code]):
@@ -802,6 +808,8 @@ class Dataset:  # (pd.DataFrame):
                     meaningType=meaning_type,
                     minMaxValues=min_max_values,
                 )
+                if self.unnest_search_keys and column_meta.originalName in self.unnest_search_keys:
+                    column_meta.isUnnest = True
 
                 columns.append(column_meta)