howso-engine 62.2.0__tar.gz → 62.3.0__tar.gz

{howso_engine-62.2.0/howso_engine.egg-info → howso_engine-62.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howso-engine
-Version: 62.2.0
+Version: 62.3.0
 Summary: The Howso Engine™ is a natively and fully explainable ML engine, serving as an alternative to black box AI neural networks.
 Author: Howso Incorporated
 Author-email: support@howso.com

{howso_engine-62.2.0 → howso_engine-62.3.0}/howso/client/base.py

@@ -4039,7 +4039,7 @@ class AbstractHowsoClient(ABC):
         num_robust_prediction_contributions_samples_per_case : int, optional
             Specifies the number of robust samples to use for each case for
             robust prediction contribution computations. Defaults to 300 +
-            2 * (number of features) when unspecified.
+            30 * (number of features) with a max of 3000 when unspecified.
         num_robust_residual_samples : int, optional
             Total sample size of model to use (using sampling with replacement)
             for robust mda and residual computation.

{howso_engine-62.2.0 → howso_engine-62.3.0}/howso/client/typing.py

@@ -128,6 +128,9 @@ class FeatureBounds(TypedDict, total=False):
     min: float | str
     """The minimum value to be output. May be a number or date string."""
 
+    nulls_observed: bool
+    """Whether any nulls were observed in this feature."""
+
     observed_max: float | str
     """The observed maximum value in the data. May be a number, string, or date string."""
 

{howso_engine-62.2.0 → howso_engine-62.3.0}/howso/engine/trainee.py

@@ -3990,7 +3990,7 @@ class Trainee(BaseTrainee):
         num_robust_prediction_contributions_samples_per_case : int, optional
             Specifies the number of robust samples to use for each case for
             robust prediction contribution computations. Defaults to 300 +
-            2 * (number of features) when unspecified.
+            30 * (number of features) with a max of 3000 when unspecified.
         num_robust_residual_samples : int, optional
             Total sample size of model to use (using sampling with replacement)
             for robust mda and residual computation.

{howso_engine-62.2.0 → howso_engine-62.3.0}/howso/howso-engine/version.json

@@ -1,10 +1,10 @@
 {
-  "version": "116.2.0",
+  "version": "116.2.3",
   "dependencies": {
     "amalgam": "81.1.2"
   },
   "display_title": "",
   "build_date": "",
-  "sha": "47f268d04853bf5a131b6cfd4f25891001aecad4",
-  "url": "https://github.com/howsoai/howso-engine/releases/tag/116.2.0"
+  "sha": "bce11a73c0c3d7a9c2e4f3d7ec445eff38bab9d2",
+  "url": "https://github.com/howsoai/howso-engine/releases/tag/116.2.3"
 }

{howso_engine-62.2.0 → howso_engine-62.3.0}/howso/utilities/__init__.py

@@ -38,6 +38,7 @@ from howso.utilities.utilities import (  # noqa: F401
     get_hash,
     get_kwargs,
     get_optimized_max_chunk_size,
+    is_null_value,
     is_valid_datetime_format,
     is_valid_uuid,
     ISO_8601_DATE_FORMAT,
@@ -98,6 +99,7 @@ __all__ = [
     "get_optimized_max_chunk_size",
     "infer_fanout_feature_config",
     "infer_feature_attributes",
+    "is_null_value",
     "is_valid_datetime_format",
     "is_valid_uuid",
     "lazy_map",

{howso_engine-62.2.0 → howso_engine-62.3.0}/howso/utilities/feature_attributes/abstract_data.py

@@ -401,13 +401,11 @@ class InferFeatureAttributesAbstractData(InferFeatureAttributesBase):
     ) -> dict | None:
         # prevent circular import
         output = dict()
-        allow_null = True
         original_type = feature_attributes[feature_name]['original_type']
         decimal_places = feature_attributes[feature_name].get('decimal_places')
 
-        # Only integers by default do not allow nulls.
-        if original_type.get('data_type') == FeatureType.INTEGER.value:
-            allow_null = False
+        # Rely on schema information about nullability
+        allow_null = self.data.is_nullable_column(feature_name)
 
         if feature_attributes[feature_name].get('type') == 'continuous':
             # Grab the natural feature_type and raw_feature_type
@@ -451,14 +449,14 @@ class InferFeatureAttributesAbstractData(InferFeatureAttributesBase):
                     return {
                         'min': 0, 'max': feature_attributes[feature_name]['cycle_length'],
                         'observed_min': time_to_seconds(min_time), 'observed_max': time_to_seconds(max_time),
-                        'allow_null': True
+                        'allow_null': allow_null
                     }
                 else:
                     # Tight bounds
                     return {
                         'min': time_to_seconds(min_time), 'max': time_to_seconds(max_time),
                         'observed_min': time_to_seconds(min_time), 'observed_max': time_to_seconds(max_time),
-                        'allow_null': True
+                        'allow_null': allow_null
                     }
 
             if 'date_time_format' in feature_attributes[feature_name]:
@@ -798,3 +796,7 @@ class InferFeatureAttributesAbstractData(InferFeatureAttributesBase):
     def _get_value_count(self, feature_name: str, value: t.Any) -> int:
         """Get the number of occurances of the provided value of the provided feature."""
         return self.data.get_value_count(feature_name, value, max_rows_to_eval=self.max_rows_to_eval)
+
+    def _contains_nulls(self, feature_name: str) -> bool:
+        """Get whether the provided feature contains any nulls."""
+        return self.data.contains_nulls(feature_name)

{howso_engine-62.2.0 → howso_engine-62.3.0}/howso/utilities/feature_attributes/base.py

@@ -1097,6 +1097,10 @@ class InferFeatureAttributesBase(ABC):
                     # Use `update` on the bounds dictionary in case `allowed` ordinal values have already been set
                     bounds.update(self.attributes[feature_name].get("bounds", {}))
                     _attributes["bounds"] = bounds
+                # Record whether we have observed any nulls in this column
+                if "bounds" not in _attributes:
+                    _attributes["bounds"] = {}
+                _attributes["bounds"]["nulls_observed"] = self._contains_nulls(feature_name)
 
         # Do any features contain data unsupported by the core?
         self._check_unsupported_data(self.attributes)
@@ -1190,6 +1194,10 @@ class InferFeatureAttributesBase(ABC):
 
         return ordered_attributes
 
+    @abstractmethod
+    def _contains_nulls(self, feature_name: str) -> bool:
+        """Get whether the provided feature has any nulls."""
+
     @abstractmethod
     def __call__(self) -> FeatureAttributesBase:
         """Process and return the feature attributes."""
@@ -1962,8 +1970,8 @@ class InferFeatureAttributesBase(ABC):
             )
         return prvc
 
-    def _process_rare_values(self, preserve_rare_values_map: PreserveRareValuesMap | t.Literal["all", "off"],  # noqa: PLR0912, PLR0915
-                             preserve_rare_values_config: PreserveRareValuesConfig, max_distilled_cases: int,
+    def _process_rare_values(self, preserve_rare_values_map: PreserveRareValuesMap | t.Literal["all", "off"] | None,  # noqa: PLR0912, PLR0915
+                             preserve_rare_values_config: PreserveRareValuesConfig | None, max_distilled_cases: int,
                              significance_threshold: int, enable_suggestions: bool = True) -> None:
         """Procesess `preserve_rare_values` configuration or make recommendation."""
         _prvc: FullPreserveRareValuesConfig = {}

{howso_engine-62.2.0 → howso_engine-62.3.0}/howso/utilities/feature_attributes/pandas.py

@@ -452,13 +452,6 @@ class InferFeatureAttributesDataFrame(InferFeatureAttributesBase):
         decimal_places = feature_attributes[feature_name].get('decimal_places')
         # only integers by default do not allow nulls
 
-        # Non-nullable integer
-        if column.dtype == np.int64:
-            allow_null = False
-        # Nullable integer
-        elif is_integer_dtype(column.dtype):
-            allow_null = True
-
         def _as_float(value, dtype):
             # Convert datatype to float
             if is_timedelta64_dtype(dtype):
@@ -984,3 +977,7 @@ class InferFeatureAttributesDataFrame(InferFeatureAttributesBase):
         if value in counts.index:
             return int(counts.loc[value])
         return 0
+
+    def _contains_nulls(self, feature_name: str) -> bool:
+        """Get whether the provided feature contains any null values."""
+        return bool(self.data[feature_name].isna().any())

{howso_engine-62.2.0 → howso_engine-62.3.0}/howso/utilities/feature_attributes/protocols.py

@@ -191,6 +191,10 @@ class IFACompatibleADCProtocol(t.Protocol):
     def get_unique_values(self, column_name: Hashable) -> Collection[t.Any]:
         """Return the unique values in `column_name`."""
 
+    @abstractmethod
+    def is_nullable_column(self, column_name: Hashable) -> bool:
+        """Return whether the provided column allows nulls."""
+
 
 class RelationshipProtocol(t.Protocol):
     """Protocol for an object representing a relationship in a database."""

{howso_engine-62.2.0 → howso_engine-62.3.0}/howso/utilities/features.py

@@ -4,7 +4,6 @@ from collections.abc import Iterable, Mapping, Sequence
 import decimal
 from enum import Enum
 from functools import partial
-import json
 import locale
 from typing import Any, cast
 import warnings
@@ -675,12 +674,14 @@ class FeatureSerializer:
     def format_tokenizable_string_column(
         cls,
         column: pd.Series,
-        feature: Mapping[str, Any],  # noqa: ARG003
+        feature: Mapping[str, Any],
         tokenizer: TokenizerProtocol | None = None,
     ) -> pd.Series:
         """
         Format tokenizable string column.
 
+        Null values are preserved as None, as there is nothing to detokenize.
+
         Parameters
         ----------
         column : pandas.Series
@@ -698,7 +699,10 @@ class FeatureSerializer:
             The formatted column.
         """
         tokenizer = tokenizer or HowsoTokenizer()
-        return pd.Series(tokenizer.detokenize(json.loads(case)) for case in column)
+        return pd.Series(
+            None if tokens is None else tokenizer.detokenize(tokens)
+            for tokens in destringify_json(column, feature)
+        )
 
     @classmethod
     def format_unknown_column(cls, column: pd.Series, feature: Mapping[str, Any]) -> pd.Series:  # noqa: ARG003

{howso_engine-62.2.0 → howso_engine-62.3.0}/howso/utilities/utilities.py

@@ -771,10 +771,33 @@ def serialize_datetimes(  # noqa: C901
             case[i] = dt_value
 
 
+def is_null_value(value: Any) -> bool:
+    """
+    Determine whether a single case value is null.
+
+    Unlike :func:`pandas.isna`, this is safe to call on container values such as lists and dicts,
+    for which :func:`pandas.isna` returns element-wise results rather than a single bool.
+
+    Parameters
+    ----------
+    value : Any
+        The case value to check.
+
+    Returns
+    -------
+    bool
+        True if the value is a null scalar (None, `np.nan`, `pd.NA`, ...).
+    """
+    return pd.api.types.is_scalar(value) and bool(pd.isna(value))
+
+
 def stringify_json(cases: list[list[Any]], features: Iterable[str], feature_attributes: Mapping) -> None:
     """
     Ensure that any JSON features have their cases stringified.
 
+    Null values are left as-is so that they are sent to the engine as nulls rather than as the
+    JSON string "null".
+
     Parameters
     ----------
     cases : list of list of Any
@@ -789,20 +812,25 @@ def stringify_json(cases: list[list[Any]], features: Iterable[str], feature_attr
             orig_type_info = feature_attributes[feature_name].get("original_type", {})
             # Applicable if original type is an object (Python list/dict) or string, tokenized into a list
             if orig_type_info.get("data_type", {}) in ["container", "tokenizable_string"]:
-                if orig_type_info.get("coercion") == "set":
-                    # If an original type indicates a set, convert to list for serialization
-                    for case_group in cases:
-                        case_group[idx] = json.dumps(list(case_group[idx]))
-                    # Deserialize only
-                else:
-                    for case_group in cases:
-                        case_group[idx] = json.dumps(case_group[idx])
+                coerce_set = orig_type_info.get("coercion") == "set"
+                for case_group in cases:
+                    value = case_group[idx]
+                    if is_null_value(value):
+                        case_group[idx] = None
+                        continue
+                    if coerce_set:
+                        # If an original type indicates a set, convert to list for serialization
+                        value = list(value)
+                    case_group[idx] = json.dumps(value)
 
 
-def destringify_json(cases: pd.Series, feature_attributes: Mapping) -> pd.Series:  # noqa: ARG001
+def destringify_json(cases: pd.Series, feature_attributes: Mapping) -> pd.Series:
     """
     Ensure that any JSON features have their cases destringified.
 
+    Null values are returned as None. A null may arrive either directly as a null value or JSON
+    encoded as the literal ``null`` (which `json.loads` decodes to None).
+
     Parameters
     ----------
     cases : pd.Series
@@ -813,8 +841,15 @@ def destringify_json(cases: pd.Series, feature_attributes: Mapping) -> pd.Series
     destringified_cases = []
     typing_info = feature_attributes.get("original_type", {})
     for case_to_destringify in cases:
-        formatted_case = json.loads(case_to_destringify)
-        if typing_info.get("coercion") == "set":
+        if is_null_value(case_to_destringify):
+            destringified_cases.append(None)
+            continue
+        if isinstance(case_to_destringify, (str, bytes, bytearray)):
+            # The JSON literal ``null`` decodes to None, which cannot be coerced any further
+            formatted_case = json.loads(case_to_destringify)
+        else:
+            formatted_case = case_to_destringify
+        if formatted_case is not None and typing_info.get("coercion") == "set":
             formatted_case = set(formatted_case)
         destringified_cases.append(formatted_case)
     return pd.Series(destringified_cases)
@@ -826,6 +861,8 @@ def tokenize_strings(
     """
     Process tokenizable strings in the provided cases by tokenizing or detokenizing them in place.
 
+    Null values are left as-is, as there is nothing to tokenize.
+
     Parameters
     ----------
     cases : list of list of Any
@@ -843,6 +880,8 @@ def tokenize_strings(
         if feature_attributes.get(feature_name, {}).get("original_type", {}).get("data_type") != "tokenizable_string":
             continue
         for case_group in cases:
+            if is_null_value(case_group[idx]):
+                continue
             case_group[idx] = tokenizer.tokenize(case_group[idx])
 
 

{howso_engine-62.2.0 → howso_engine-62.3.0/howso_engine.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howso-engine
-Version: 62.2.0
+Version: 62.3.0
 Summary: The Howso Engine™ is a natively and fully explainable ML engine, serving as an alternative to black box AI neural networks.
 Author: Howso Incorporated
 Author-email: support@howso.com

howso_engine-62.3.0/version.json

@@ -0,0 +1,5 @@
+{
+  "dependencies": {
+    "howso-engine": "116.2.3"
+  }
+}

howso_engine-62.2.0/version.json

@@ -1,5 +0,0 @@
-{
-  "dependencies": {
-    "howso-engine": "116.2.0"
-  }
-}