featrixsphere 0.2.8332__tar.gz → 0.2.8359__tar.gz

{featrixsphere-0.2.8332 → featrixsphere-0.2.8359}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: featrixsphere
-Version: 0.2.8332
+Version: 0.2.8359
 Summary: Transform any CSV into a production-ready ML model in minutes, not months.
 Home-page: https://github.com/Featrix/sphere
 Author: Featrix

featrixsphere-0.2.8359/VERSION

@@ -0,0 +1 @@
+0.2.8359

{featrixsphere-0.2.8332 → featrixsphere-0.2.8359}/featrixsphere/__init__.py

@@ -44,7 +44,7 @@ Usage:
 For more examples, see the documentation at https://docs.featrix.com
 """
 
-__version__ = "0.2.8332"
+__version__ = "0.2.8359"
 __author__ = "Featrix"
 __email__ = "support@featrix.com"
 __license__ = "MIT"
@@ -53,6 +53,7 @@ __license__ = "MIT"
 from .api import FeatrixSphere, FoundationalModel, Predictor, PredictionResult
 from .api.exceptions import FeatrixAuthenticationError, FeatrixPredictionError
 from .api.label_utils import find_similar_labels, collapse_similar_labels
+from .api.null_distribution import compute_null_distribution
 
 __all__ = [
     "FeatrixAuthenticationError",
@@ -63,6 +64,7 @@ __all__ = [
     "PredictionResult",
     "find_similar_labels",
     "collapse_similar_labels",
+    "compute_null_distribution",
     "__version__",
 ]
 

{featrixsphere-0.2.8332 → featrixsphere-0.2.8359}/featrixsphere/api/__init__.py

@@ -55,6 +55,7 @@ from .reference_record import ReferenceRecord
 from .api_endpoint import APIEndpoint
 from .notebook_helper import FeatrixNotebookHelper
 from .label_utils import find_similar_labels, collapse_similar_labels
+from .null_distribution import compute_null_distribution
 
 __all__ = [
     'FeatrixAuthenticationError',
@@ -72,4 +73,5 @@ __all__ = [
     'FeatrixNotebookHelper',
     'find_similar_labels',
     'collapse_similar_labels',
+    'compute_null_distribution',
 ]

featrixsphere-0.2.8359/featrixsphere/api/null_distribution.py

@@ -0,0 +1,176 @@
+#  -*- coding: utf-8 -*-
+#
+#  Copyright (c) 2023-2026 Featrix, Inc, All Rights Reserved
+#
+#  Proprietary and Confidential.  Unauthorized use, copying or dissemination
+#  of these materials is strictly prohibited.
+#
+
+"""
+Null distribution analysis for shaping ES masking to real-world data patterns.
+
+Compute per-column null rates from any data source — production exports, representative
+samples, or the training data itself — and save the result as JSON. The ES training
+pipeline can then load this distribution to bias its masking strategy, so the model
+learns representations that are robust to the null patterns it will actually see.
+
+Usage::
+
+    from featrixsphere import compute_null_distribution
+
+    # From a file
+    dist = compute_null_distribution("production_sample.csv")
+
+    # From a DataFrame
+    dist = compute_null_distribution(df, ignore_columns=["id", "timestamp"])
+
+    # Save for training
+    import json
+    with open("null_distribution.json", "w") as f:
+        json.dump(dist, f, indent=2)
+"""
+import json
+import logging
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Union
+
+import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+
+def compute_null_distribution(
+    data: Union[str, Path, pd.DataFrame],
+    ignore_columns: Optional[List[str]] = None,
+) -> Dict[str, Any]:
+    """
+    Compute per-column null distribution from a data file or DataFrame.
+
+    Point this at your production data, a representative sample, or any CSV/parquet/JSON
+    to learn the null patterns your model will see in the real world.  The returned dict
+    can be saved as JSON and passed to ES training via the ``null_distribution_path``
+    config key.
+
+    Args:
+        data: One of:
+            - ``str`` or ``Path``: path to a CSV, Parquet, or JSON file
+            - ``pd.DataFrame``: an already-loaded DataFrame
+        ignore_columns: Column names to exclude from analysis (e.g. IDs, timestamps).
+
+    Returns:
+        Dict with per-column null counts/rates and summary statistics::
+
+            {
+                "total_rows": 50000,
+                "total_columns": 25,
+                "per_column": {
+                    "phone": {"null_count": 12500, "null_rate": 0.25},
+                    "email": {"null_count": 5000, "null_rate": 0.10},
+                    ...
+                },
+                "summary": {
+                    "mean_null_rate": 0.12,
+                    "median_null_rate": 0.08,
+                    "max_null_rate": 0.45,
+                    "columns_always_present": 15,
+                    "columns_sometimes_null": 8,
+                    "columns_mostly_null": 2,
+                }
+            }
+
+    Raises:
+        FileNotFoundError: If *data* is a path that does not exist.
+        ValueError: If the resulting DataFrame is empty or has no analyzable columns.
+    """
+    df = _load_data(data)
+
+    if ignore_columns:
+        ignore_set = set(ignore_columns)
+    else:
+        ignore_set = set()
+
+    cols = [c for c in df.columns if c not in ignore_set]
+
+    if not cols:
+        raise ValueError(
+            "No columns to analyze after applying ignore_columns filter"
+        )
+
+    total_rows = len(df)
+    if total_rows == 0:
+        raise ValueError("DataFrame is empty — nothing to analyze")
+
+    # Per-column null counts
+    null_counts = df[cols].isna().sum(axis=0)
+
+    per_column = {}
+    null_rates = []
+    for col in cols:
+        nc = int(null_counts[col])
+        rate = nc / total_rows
+        per_column[col] = {
+            "null_count": nc,
+            "null_rate": round(rate, 6),
+        }
+        null_rates.append(rate)
+
+    # Summary statistics
+    import numpy as np
+
+    rates_arr = np.array(null_rates)
+    columns_always_present = int((rates_arr == 0.0).sum())
+    columns_sometimes_null = int(((rates_arr > 0.0) & (rates_arr <= 0.5)).sum())
+    columns_mostly_null = int((rates_arr > 0.5).sum())
+
+    result = {
+        "total_rows": total_rows,
+        "total_columns": len(cols),
+        "per_column": per_column,
+        "summary": {
+            "mean_null_rate": round(float(rates_arr.mean()), 6),
+            "median_null_rate": round(float(np.median(rates_arr)), 6),
+            "max_null_rate": round(float(rates_arr.max()), 6),
+            "columns_always_present": columns_always_present,
+            "columns_sometimes_null": columns_sometimes_null,
+            "columns_mostly_null": columns_mostly_null,
+        },
+    }
+
+    # Log a quick summary
+    logger.info(
+        "Null distribution: %d rows, %d columns — "
+        "%d always present, %d sometimes null, %d mostly null (>50%%)",
+        total_rows,
+        len(cols),
+        columns_always_present,
+        columns_sometimes_null,
+        columns_mostly_null,
+    )
+
+    return result
+
+
+def _load_data(data: Union[str, Path, pd.DataFrame]) -> pd.DataFrame:
+    """Load data from a file path or return the DataFrame as-is."""
+    if isinstance(data, pd.DataFrame):
+        return data
+
+    path = Path(data)
+    if not path.exists():
+        raise FileNotFoundError(f"Data file not found: {path}")
+
+    suffix = path.suffix.lower()
+    if suffix == ".csv":
+        return pd.read_csv(path)
+    elif suffix in (".parquet", ".pq"):
+        return pd.read_parquet(path)
+    elif suffix == ".json":
+        return pd.read_json(path)
+    elif suffix in (".tsv",):
+        return pd.read_csv(path, sep="\t")
+    elif suffix in (".xlsx", ".xls"):
+        return pd.read_excel(path)
+    else:
+        # Try CSV as fallback
+        logger.warning("Unknown file extension %r — attempting CSV parse", suffix)
+        return pd.read_csv(path)

{featrixsphere-0.2.8332 → featrixsphere-0.2.8359}/featrixsphere/api/predictor.py

@@ -770,6 +770,66 @@ class Predictor:
 
         return response
 
+    def encode(
+        self,
+        record: Union[Dict[str, Any], List[Dict[str, Any]], 'pd.DataFrame', None] = None,
+        records: Union[Dict[str, Any], List[Dict[str, Any]], 'pd.DataFrame', None] = None,
+        short: bool = False,
+    ) -> Union[List[Dict[str, Any]], List[List[float]]]:
+        """
+        Encode records to embedding vectors using the underlying embedding space.
+
+        Accepts either record= or records= (singular or plural). Each can be
+        a single dict, a list of dicts, or a DataFrame.
+
+        Args:
+            record: Single record or list of records (alias for records)
+            records: Single record or list of records
+            short: If True, return only the 3D embedding vectors (for visualization).
+                   If False (default), return full results with both embeddings.
+
+        Returns:
+            If short=False: List of dicts, each containing:
+                - "embedding": 3D embedding vector (for visualization)
+                - "embedding_long": Full-length embedding vector
+                - "query_record": The input record echoed back
+            If short=True: List of 3D embedding vectors (List[List[float]])
+
+        Example:
+            results = predictor.encode(record={"age": 35, "income": 50000})
+            results = predictor.encode(records=[{"age": 35}, {"age": 42}])
+            vectors_3d = predictor.encode(records=df, short=True)
+        """
+        if not self._ctx:
+            raise ValueError("Predictor not connected to client")
+
+        # Accept either kwarg
+        data = record if record is not None else records
+        if data is None:
+            raise ValueError("Must provide record= or records=")
+
+        # Normalize to list of dicts
+        if isinstance(data, dict):
+            data = [data]
+        elif hasattr(data, 'to_dict'):
+            data = data.to_dict('records')
+        elif not isinstance(data, list):
+            data = [data]
+
+        cleaned = [self._clean_record(r) for r in data]
+
+        response = self._ctx.post_json(
+            f"/session/{self.session_id}/encode_records",
+            data={"records": cleaned}
+        )
+
+        results = response.get('results', [])
+
+        if short:
+            return [r["embedding"] for r in results]
+
+        return results
+
     def _update_metrics(self) -> None:
         """Update metrics from server."""
         try:

{featrixsphere-0.2.8332 → featrixsphere-0.2.8359}/featrixsphere.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: featrixsphere
-Version: 0.2.8332
+Version: 0.2.8359
 Summary: Transform any CSV into a production-ready ML model in minutes, not months.
 Home-page: https://github.com/Featrix/sphere
 Author: Featrix

{featrixsphere-0.2.8332 → featrixsphere-0.2.8359}/featrixsphere.egg-info/SOURCES.txt

@@ -23,6 +23,7 @@ featrixsphere/api/foundational_model.py
 featrixsphere/api/http_client.py
 featrixsphere/api/label_utils.py
 featrixsphere/api/notebook_helper.py
+featrixsphere/api/null_distribution.py
 featrixsphere/api/prediction_grid.py
 featrixsphere/api/prediction_result.py
 featrixsphere/api/predictor.py
@@ -2491,6 +2492,7 @@ src/lib/featrix/neural/sp_loss_autotuner.py
 src/lib/featrix/neural/sp_multiclass_autotuner.py
 src/lib/featrix/neural/sp_multilabel_autotuner.py
 src/lib/featrix/neural/sp_probe.py
+src/lib/featrix/neural/sp_regression_autotuner.py
 src/lib/featrix/neural/sphere_config.py
 src/lib/featrix/neural/sphere_init.py
 src/lib/featrix/neural/split_utils.py
@@ -2632,6 +2634,7 @@ src/production_server/gateway_watchdog.sh
 src/production_server/generate_fixture_from_model.py
 src/production_server/generate_validation_fixture.py
 src/production_server/install.sh
+src/production_server/null_tracker.py
 src/production_server/publish_model.py
 src/production_server/server.py
 src/production_server/worker.py

{featrixsphere-0.2.8332 → featrixsphere-0.2.8359}/src/api.py

@@ -11933,6 +11933,8 @@ def create_app() -> FastAPI:
                 'column_loss_landscape': ('column_loss_landscape.png', 'Column loss landscape'),
                 'relationship_graph': ('relationship_graph.png', 'Relationship graph'),
                 'relationship_graph_data': ('relationship_graph.json', 'Relationship graph data'),
+                'relationship_graph_cumlift': ('relationship_graph_cumlift.png', 'Cumulative lift graph'),
+                'relationship_graph_cumlift_data': ('relationship_graph_cumlift.json', 'Cumulative lift data'),
                 'gradient_flow': ('gradient_flow.png', 'Gradient flow'),
                 'prediction_confidence': ('prediction_confidence.png', 'Prediction confidence'),
                 'attention_flow': ('attention_flow.png', 'Attention flow'),
@@ -11988,10 +11990,15 @@ def create_app() -> FastAPI:
 
             # Determine which file to serve
             file_ext = Path(filename).suffix  # e.g. '.png', '.json', '.html'
+            # Map viz_types that share epoch directories with another viz_type
+            _epoch_dir_aliases = {
+                'relationship_graph_data': 'relationship_graph',
+                'relationship_graph_cumlift_data': 'relationship_graph_cumlift',
+            }
+
             if epoch is not None:
                 # Specific epoch requested - check media subdirectory
-                # relationship_graph_data shares epoch dir/prefix with relationship_graph
-                epoch_dir_name = 'relationship_graph' if viz_type == 'relationship_graph_data' else viz_type
+                epoch_dir_name = _epoch_dir_aliases.get(viz_type, viz_type)
                 epoch_file_prefix = epoch_dir_name
                 media_file = job_dir / "media" / epoch_dir_name / f"{epoch_file_prefix}_epoch_{epoch:04d}{file_ext}"
             else:
@@ -12011,7 +12018,7 @@ def create_app() -> FastAPI:
                         break
                 else:
                     # Last resort: find the latest epoch file in media subdir
-                    fallback_dir_name = 'relationship_graph' if viz_type == 'relationship_graph_data' else viz_type
+                    fallback_dir_name = _epoch_dir_aliases.get(viz_type, viz_type)
                     fallback_prefix = fallback_dir_name
                     media_subdir = media_root / fallback_dir_name
                     if media_subdir.exists():
@@ -12096,6 +12103,8 @@ def create_app() -> FastAPI:
                 'column_loss_landscape': ('column_loss_landscape.png', 'Column loss landscape'),
                 'relationship_graph': ('relationship_graph.png', 'Relationship graph'),
                 'relationship_graph_data': ('relationship_graph.json', 'Relationship graph data'),
+                'relationship_graph_cumlift': ('relationship_graph_cumlift.png', 'Cumulative lift graph'),
+                'relationship_graph_cumlift_data': ('relationship_graph_cumlift.json', 'Cumulative lift data'),
                 'gradient_flow': ('gradient_flow.png', 'Gradient flow'),
                 'prediction_confidence': ('prediction_confidence.png', 'Prediction confidence'),
                 'attention_flow': ('attention_flow.png', 'Attention flow'),
@@ -12112,6 +12121,12 @@ def create_app() -> FastAPI:
                 'data_quality_report': ('data_quality_report.html', 'Interactive data quality report'),
             }
 
+            # Map viz_types that share epoch directories with another viz_type
+            _epoch_dir_aliases = {
+                'relationship_graph_data': 'relationship_graph',
+                'relationship_graph_cumlift_data': 'relationship_graph_cumlift',
+            }
+
             # Check which files actually exist (in media/ or job root)
             # Always check epoch subdirectory too — the main file is just the latest copy
             for viz_type, (filename, description) in viz_type_mapping.items():
@@ -12126,8 +12141,8 @@ def create_app() -> FastAPI:
                 )
 
                 # Check epoch subdirectory regardless of main file existence
-                # relationship_graph_data shares epoch dir/prefix with relationship_graph
-                epoch_dir_name = 'relationship_graph' if viz_type == 'relationship_graph_data' else viz_type
+                # _data variants share epoch dir/prefix with their parent viz_type
+                epoch_dir_name = _epoch_dir_aliases.get(viz_type, viz_type)
                 epoch_file_prefix = epoch_dir_name
                 viz_dir = media_dir / epoch_dir_name
                 epochs = []