sovai 0.2.56__tar.gz → 0.2.58__tar.gz

{sovai-0.2.56 → sovai-0.2.58}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sovai
-Version: 0.2.56
+Version: 0.2.58
 Summary: python for asset management
 License: Proprietary License
 Author: Derek Snow

{sovai-0.2.56 → sovai-0.2.58}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [project]
 name = "sovai"
-version = "0.2.56"
+version = "0.2.58"
 description = "python for asset management"
 authors = [
     {name = "Derek Snow", email = "d.snow@outlook.com"}

sovai-0.2.58/sovai/__init__.py

@@ -0,0 +1,167 @@
+"""
+Main SovAI SDK Tool Kit package
+"""
+from .api_config import read_key, save_key, ApiConfig
+from .basic_auth import basic_auth
+from .token_auth import token_auth
+from .utils.file_management import update_data_files # Keep update_data_files import
+
+
+import warnings
+warnings.filterwarnings("ignore")
+
+
+try:
+    from importlib.metadata import version
+    __version__ = version("sovai")
+except:
+    __version__ = "0.2.49"  # Fallback to current version in pyproject.toml
+
+
+# Lazy loading for core modules
+_data_module = None
+
+def data(*args, **kwargs):
+    """Lazy-loaded access to the data function."""
+    global _data_module
+    if _data_module is None:
+        from .get_data import data as _loaded_data
+        _data_module = _loaded_data
+    return _data_module(*args, **kwargs)
+
+
+# Lazy loading for full installation modules
+HAS_FULL_INSTALL = False
+_plot_module = None
+_report_module = None
+_compute_module = None
+_nowcast_module = None
+_extension_module = None
+_sec_search_module = None
+_sec_filing_module = None
+_code_module = None
+_sec_graph_module = None
+
+try:
+    # Attempt to import a module from the full installation to check if it's installed
+    # This import is just to check for ImportError, the actual function is lazy-loaded below
+    from .get_plots import plot as _ # Use _ to indicate this import is just for the check
+    HAS_FULL_INSTALL = True
+
+    def plot(*args, **kwargs):
+        """Lazy-loaded access to the plot function."""
+        global _plot_module
+        if _plot_module is None:
+            from .get_plots import plot as _loaded_plot
+            _plot_module = _loaded_plot
+        return _plot_module(*args, **kwargs)
+
+    def report(*args, **kwargs):
+        """Lazy-loaded access to the report function."""
+        global _report_module
+        if _report_module is None:
+            from .get_reports import report as _loaded_report
+            _report_module = _loaded_report
+        return _report_module(*args, **kwargs)
+
+    def compute(*args, **kwargs):
+        """Lazy-loaded access to the compute function."""
+        global _compute_module
+        if _compute_module is None:
+            from .get_compute import compute as _loaded_compute
+            _compute_module = _loaded_compute
+        return _compute_module(*args, **kwargs)
+
+    def nowcast(*args, **kwargs):
+        """Lazy-loaded access to the nowcast function."""
+        global _nowcast_module
+        if _nowcast_module is None:
+            from .studies.nowcasting import nowcast as _loaded_nowcast
+            _nowcast_module = _loaded_nowcast
+        return _nowcast_module(*args, **kwargs)
+
+    # extension is likely a class, so the lazy loader needs to return the class itself
+    def extension(*args, **kwargs):
+        """Lazy-loaded access to the CustomDataFrame extension class."""
+        global _extension_module
+        if _extension_module is None:
+            from .extensions.pandas_extensions import CustomDataFrame as _loaded_extension
+            _extension_module = _loaded_extension
+        # If called with args/kwargs, assume it's being instantiated
+        if args or kwargs:
+             return _extension_module(*args, **kwargs)
+        # Otherwise, return the class itself (e.g., for type checking or accessing class methods)
+        return _extension_module
+
+
+    def sec_search(*args, **kwargs):
+        """Lazy-loaded access to the sec_search function."""
+        global _sec_search_module
+        if _sec_search_module is None:
+            from .get_tools import sec_search as _loaded_sec_search
+            _sec_search_module = _loaded_sec_search
+        return _sec_search_module(*args, **kwargs)
+
+    def sec_filing(*args, **kwargs):
+        """Lazy-loaded access to the sec_filing function."""
+        global _sec_filing_module
+        if _sec_filing_module is None:
+            from .get_tools import sec_filing as _loaded_sec_filing
+            _sec_filing_module = _loaded_sec_filing
+        return _sec_filing_module(*args, **kwargs)
+
+    def code(*args, **kwargs):
+        """Lazy-loaded access to the code function."""
+        global _code_module
+        if _code_module is None:
+            from .get_tools import code as _loaded_code
+            _code_module = _loaded_code
+        return _code_module(*args, **kwargs)
+
+    def sec_graph(*args, **kwargs):
+        """Lazy-loaded access to the sec_graph function."""
+        global _sec_graph_module
+        if _sec_graph_module is None:
+            from .get_tools import sec_graph as _loaded_sec_graph
+            _sec_graph_module = _loaded_sec_graph
+        return _sec_graph_module(*args, **kwargs)
+
+
+except ImportError as e:
+    print("this is the lean installation, for full use sovai[full]")
+    # print(f"ImportError: {e}")
+    HAS_FULL_INSTALL = False
+
+    # Define dummy functions/objects for lean installation to avoid AttributeErrors
+    def plot(*args, **kwargs):
+        raise ImportError("Plotting requires the full installation. Install with 'pip install sovai[full]'")
+    def report(*args, **kwargs):
+        raise ImportError("Reporting requires the full installation. Install with 'pip install sovai[full]'")
+    def compute(*args, **kwargs):
+        raise ImportError("Compute requires the full installation. Install with 'pip install sovai[full]'")
+    def nowcast(*args, **kwargs):
+        raise ImportError("Nowcasting requires the full installation. Install with 'pip install sovai[full]'")
+    def extension(*args, **kwargs):
+         raise ImportError("Extensions require the full installation. Install with 'pip install sovai[full]'")
+    def sec_search(*args, **kwargs):
+        raise ImportError("SEC tools require the full installation. Install with 'pip install sovai[full]'")
+    def sec_filing(*args, **kwargs):
+        raise ImportError("SEC tools require the full installation. Install with 'pip install sovai[full]'")
+    def code(*args, **kwargs):
+        raise ImportError("SEC tools require the full installation. Install with 'pip install sovai[full]'")
+    def sec_graph(*args, **kwargs):
+        raise ImportError("SEC tools require the full installation. Install with 'pip install sovai[full]'")
+
+
+__all__ = [
+    'read_key', 'save_key', 'ApiConfig', 'basic_auth', 'token_auth',
+    'update_data_files', # Keep update_data_files in __all__
+    'data', # Add data to __all__
+]
+
+# Add full installation lazy-loaded attributes to __all__ if available
+if HAS_FULL_INSTALL:
+    __all__ += [
+        'plot', 'report', 'compute', 'nowcast', 'extension',
+        'sec_search', 'sec_filing', 'code', 'sec_graph'
+    ]

sovai-0.2.58/sovai/extensions/shapley_global_importance.py

@@ -0,0 +1,214 @@
+# Lightweight built-in imports can stay global
+import random
+import hashlib
+import warnings
+# import os # Removed as it was unused
+
+# Note: Imports below are moved into functions/methods to defer loading cost,
+# potentially speeding up the initial import of this script.
+
+def hash_of_df(df, sample_size=100):
+    """Calculates a SHA256 hash of a sampled portion of a DataFrame."""
+    # Import pandas here as it's needed for DataFrame operations
+    import pandas as pd
+
+    df_sample = (
+        df.sample(n=min(sample_size, len(df)), random_state=42).to_string().encode()
+    )
+    return hashlib.sha256(df_sample).hexdigest()
+
+class ClusteringExplainer:
+    """
+    Trains a classifier to predict cluster labels and provides SHAP explanations.
+    """
+    def __init__(self, random_state=42):
+        """Initializes the explainer and scaler."""
+        # Import StandardScaler here as it's used for self.scaler
+        from sklearn.preprocessing import StandardScaler
+
+        self.random_state = random_state
+        self.model = None
+        self.explainer = None
+        # Initialize scaler here after import
+        self.scaler = StandardScaler()
+
+    def fit(self, X, y):
+        """Fits the LGBM classifier and creates the SHAP explainer."""
+        # Imports needed specifically for fitting the model and explainer
+        from lightgbm import LGBMClassifier
+        from sklearn.utils.class_weight import compute_sample_weight
+        import shap
+
+        classes_weights = compute_sample_weight(class_weight="balanced", y=y)
+        self.model = LGBMClassifier(
+            objective="multiclass",
+            random_state=self.random_state,
+            verbose=-1,
+            force_col_wise=True,
+            min_gain_to_split=0.01
+        )
+        # Catch warnings during model fitting (e.g., from LightGBM)
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            self.model.fit(X, y, sample_weight=classes_weights)
+
+        # Create SHAP explainer after model is trained
+        self.explainer = shap.Explainer(self.model)
+
+    def get_shap_values(self, X):
+        """Gets SHAP values using the trained explainer."""
+        # No new imports needed here, uses self.explainer created in fit()
+        if self.explainer is None:
+            raise RuntimeError("Explainer not available. Call fit() method first.")
+        return self.explainer(X)
+
+def get_shap_values_for_dataset(df, clustering_method="KMEANS", n_clusters=10, random_state=42, sample_size=5000):
+    """
+    Performs clustering, trains a model, and calculates mean absolute SHAP values.
+    """
+    # Imports needed for this function's operations
+    import pandas as pd
+    import numpy as np
+    from sklearn.cluster import KMeans, MeanShift, HDBSCAN
+    from sklearn.preprocessing import StandardScaler # Needed again for local scaler instance
+
+    # Sample for clustering and model training
+    X_sample = df.sample(n=min(sample_size, len(df)), random_state=random_state)
+
+    # Create and fit a local scaler for the sample
+    scaler = StandardScaler()
+    X_sample_scaled = scaler.fit_transform(X_sample)
+
+    # Select and fit the clustering algorithm
+    if clustering_method == "KMEANS":
+        # Use random module (imported globally)
+        num_clusters = random.randint(5, 10)
+        clustering = KMeans(n_clusters=num_clusters, random_state=random_state, n_init=10) # Added n_init
+    elif clustering_method == "MEANSHIFT":
+        clustering = MeanShift()
+    elif clustering_method == "HDBSCAN":
+        clustering = HDBSCAN(min_cluster_size=5)
+    else:
+        raise ValueError(f"Unsupported clustering method: {clustering_method}")
+
+    # Catch warnings during clustering (e.g., convergence warnings)
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        clustering.fit(X_sample_scaled)
+
+    y_sample = clustering.labels_
+
+    # Handle noise points from HDBSCAN if necessary
+    if clustering_method == "HDBSCAN":
+        # Check if noise points exist (-1 label)
+        if -1 in y_sample:
+            noise_label = np.max(y_sample) + 1
+            y_sample[y_sample == -1] = noise_label # Assign noise points to a new cluster index
+
+    # Create and fit the explainer using the clustered sample
+    clust_explnr = ClusteringExplainer(random_state=random_state)
+    clust_explnr.fit(X_sample_scaled, y_sample) # This triggers imports within ClusteringExplainer.fit
+
+    # Transform the entire dataset using the scaler fitted on the sample
+    X_full_scaled = scaler.transform(df)
+
+    # Get SHAP values for the entire scaled dataset
+    shap_values = clust_explnr.get_shap_values(X_full_scaled)
+
+    # Calculate mean absolute SHAP values across classes for each feature
+    # shap_values.values is typically (n_samples, n_features, n_classes) or (n_samples, n_features)
+    if len(shap_values.values.shape) == 3:
+         mean_abs_shap = np.mean(np.abs(shap_values.values), axis=2)
+    else: # If it's already 2D (e.g., binary classification or regression SHAP)
+         mean_abs_shap = np.abs(shap_values.values)
+
+    # Create DataFrame for the results
+    mean_abs_shap_df = pd.DataFrame(mean_abs_shap, columns=df.columns, index=df.index)
+
+    return mean_abs_shap_df
+
+def run_simulations_frame_global(df, num_simulations=4, clustering_method="KMEANS"):
+    """
+    Runs multiple simulations of SHAP value calculation in parallel and averages.
+    """
+    # Imports needed for parallel execution and DataFrame manipulation
+    from joblib import Parallel, delayed
+    import pandas as pd
+
+    data_hash = hash_of_df(df) # Uses function defined above (triggers its pandas import if first call)
+    tasks = []
+    for i in range(num_simulations):
+        # Use random module (imported globally)
+        # Seed random state for reproducibility within the loop based on data hash
+        current_seed = int(data_hash, 16) + i
+        random.seed(current_seed)
+        # Generate distinct random states for clustering and model training per simulation
+        sim_random_state = random.randint(0, 2**32 - 1) # Use a large range for random state
+
+        tasks.append(
+            delayed(get_shap_values_for_dataset)(
+                df, clustering_method, 10, sim_random_state # Pass simulation-specific state
+            )
+        ) # This implicitly calls get_shap_values_for_dataset (triggering its imports)
+
+    # Catch warnings during parallel execution
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        # Run tasks in parallel, n_jobs=-1 uses all available CPU cores
+        all_shap_values = Parallel(n_jobs=-1, verbose=0)(tasks)
+
+    # Concatenate results and calculate the mean SHAP values grouped by index
+    # Assumes df.index is a MultiIndex with 'ticker' and 'date' levels
+    if isinstance(df.index, pd.MultiIndex) and all(level in df.index.names for level in ['ticker', 'date']):
+         avg_shap_values = pd.concat(all_shap_values).abs().groupby(level=["ticker", "date"]).mean()
+    else:
+         # Fallback if index is not the expected MultiIndex (might need adjustment)
+         # This assumes the index from the original df was preserved in mean_abs_shap_df
+         # The previous reset_index().groupby() might be safer if index structure varies
+         print("Warning: DataFrame index is not a MultiIndex with 'ticker' and 'date'. Averaging over the entire index.")
+         avg_shap_values = pd.concat(all_shap_values).abs().mean()
+
+
+    return avg_shap_values
+
+def run_simulations_global_importance(df, num_simulations=4, clustering_method="KMEANS"):
+    """
+    Calculates overall feature importance based on averaged SHAP values.
+    """
+    # Imports needed for DataFrame creation and statistical calculation
+    import pandas as pd
+    from scipy import stats
+
+    # Get the averaged SHAP values per instance/group
+    avg_shap_values = run_simulations_frame_global(
+        df, num_simulations=num_simulations, clustering_method=clustering_method
+        ) # Triggers imports in run_simulations_frame_global if not already loaded
+
+    # Calculate the mean importance across all instances/groups for each feature
+    feature_importance_values = avg_shap_values.mean(axis=0)
+
+    # Create the feature importance DataFrame
+    feature_importance = pd.DataFrame(
+        {
+            "feature": feature_importance_values.index,
+            "importance": feature_importance_values.values,
+        }
+    )
+
+    # Calculate percentile rank for each feature's importance
+    feature_importance["importance_percentile"] = feature_importance["importance"].apply(
+        lambda x: stats.percentileofscore(feature_importance["importance"], x, kind='rank')
+    )
+    # Alternative using numpy might be faster for large numbers of features:
+    # feature_importance["importance_percentile"] = feature_importance["importance"].rank(pct=True) * 100
+
+    # Sort by importance percentile descending
+    return feature_importance.sort_values(
+        "importance_percentile", ascending=False
+    ).reset_index(drop=True)
+
+
+# Usage example:
+# Assuming 'df' is your pandas DataFrame with features and a MultiIndex ('ticker', 'date')
+# importance_df = run_simulations_global_importance(df, num_simulations=10, clustering_method='KMEANS')
+# print(importance_df)

{sovai-0.2.56 → sovai-0.2.58}/sovai/get_data.py

@@ -808,6 +808,15 @@ def find_tickers(
    logger.debug(f"Mapped {len(sample_identifiers)} identifiers to {len(result)} unique tickers")
    return result
 
+@lru_cache(maxsize=1)
+def _get_ticker_codes_df():
+    logger.debug("Loading ticker mapping codes...")
+    try:
+        return pd.read_parquet("data/codes.parq")
+    except Exception as e:
+        logger.error(f"Failed to load ticker mapping data: {e}")
+        raise ValueError(f"Cannot perform ticker mapping: {e}")
+
 
 def ticker_mapper(params: Dict[str, Any], verbose: bool = False) -> Dict[str, Any]:
    """
@@ -828,7 +837,7 @@ def ticker_mapper(params: Dict[str, Any], verbose: bool = False) -> Dict[str, An
 
    # Load ticker mapping data
    try:
-       df_codes = pd.read_parquet("data/codes.parq")
+       df_codes = _get_ticker_codes_df()
    except Exception as e:
        logger.error(f"Failed to load ticker mapping data: {e}")
        raise ValueError(f"Cannot perform ticker mapping: {e}")

sovai-0.2.56/sovai/__init__.py

@@ -1,39 +0,0 @@
-"""
-Main SovAI SDK Tool Kit package
-"""
-from .api_config import read_key, save_key, ApiConfig
-from .get_data import data
-from .basic_auth import basic_auth
-from .token_auth import token_auth
-from .utils.file_management import save_or_update_tickers, save_or_update_codes, update_data_files
-
-
-import warnings
-warnings.filterwarnings("ignore")
-
-
-try:
-    from importlib.metadata import version
-    __version__ = version("sovai")
-except:
-    __version__ = "0.2.49"  # Fallback to current version in pyproject.toml
-
-
-try:
-    from .get_plots import plot
-    from .get_reports import report
-    from .get_compute import compute
-    from .studies.nowcasting import nowcast
-    from .extensions.pandas_extensions import CustomDataFrame as extension
-    from .get_tools import sec_search, sec_filing, code, sec_graph
-    HAS_FULL_INSTALL = True
-except ImportError as e:
-    print("this is the lean installation, for full use sovai[full]")
-    # print(f"ImportError: {e}")
-    HAS_FULL_INSTALL = False
-
-
-__all__ = ['read_key', 'save_key', 'ApiConfig', 'data', 'basic_auth', 'token_auth', 'update_data_files']
-
-if HAS_FULL_INSTALL:
-    __all__ += ['plot', 'report', 'compute', 'nowcast', 'extension', 'sec_search', 'sec_filing', 'code', 'sec_graph']

sovai-0.2.56/sovai/extensions/shapley_global_importance.py

@@ -1,127 +0,0 @@
-import pandas as pd
-import numpy as np
-import random
-from lightgbm import LGBMClassifier
-from sklearn.cluster import KMeans, MeanShift, HDBSCAN
-from sklearn.utils.class_weight import compute_sample_weight
-from sklearn.preprocessing import StandardScaler
-import shap
-from joblib import Parallel, delayed
-import hashlib
-from scipy import stats
-import os
-import warnings
-
-
-def hash_of_df(df, sample_size=100):
-    df_sample = (
-        df.sample(n=min(sample_size, len(df)), random_state=42).to_string().encode()
-    )
-    return hashlib.sha256(df_sample).hexdigest()
-
-class ClusteringExplainer:
-    def __init__(self, random_state=42):
-        self.random_state = random_state
-        self.model = None
-        self.explainer = None
-        self.scaler = StandardScaler()
-
-    def fit(self, X, y):
-        classes_weights = compute_sample_weight(class_weight="balanced", y=y)
-        self.model = LGBMClassifier(
-            objective="multiclass",
-            random_state=self.random_state,
-            verbose=-1,
-            force_col_wise=True,
-            min_gain_to_split=0.01
-        )
-        with warnings.catch_warnings():
-            warnings.simplefilter("ignore")
-            self.model.fit(X, y, sample_weight=classes_weights)
-        self.explainer = shap.Explainer(self.model)
-
-    def get_shap_values(self, X):
-        return self.explainer(X)
-
-def get_shap_values_for_dataset(df, clustering_method="KMEANS", n_clusters=10, random_state=42, sample_size=5000):
-    # Sample for clustering and model training
-    X_sample = df.sample(n=min(sample_size, len(df)), random_state=random_state)
-    
-    scaler = StandardScaler()
-    X_sample_scaled = scaler.fit_transform(X_sample)
-
-    if clustering_method == "KMEANS":
-        num_clusters = random.randint(5, 10)
-        clustering = KMeans(n_clusters=num_clusters, random_state=random_state)
-    elif clustering_method == "MEANSHIFT":
-        clustering = MeanShift()
-    elif clustering_method == "HDBSCAN":
-        clustering = HDBSCAN(min_cluster_size=5)
-    else:
-        raise ValueError(f"Unsupported clustering method: {clustering_method}")
-
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore")
-        clustering.fit(X_sample_scaled)
-
-    y_sample = clustering.labels_
-
-    if clustering_method == "HDBSCAN":
-        noise_label = max(y_sample) + 1
-        y_sample[y_sample == -1] = noise_label
-
-    clust_explnr = ClusteringExplainer(random_state=random_state)
-    clust_explnr.fit(X_sample_scaled, y_sample)
-
-    # Transform the entire dataset
-    X_full_scaled = scaler.transform(df)
-
-    # Get SHAP values for the entire dataset
-    shap_values = clust_explnr.get_shap_values(X_full_scaled)
-    mean_abs_shap = np.mean(np.abs(shap_values.values), axis=2)
-    mean_abs_shap_df = pd.DataFrame(mean_abs_shap, columns=df.columns, index=df.index)
-
-    return mean_abs_shap_df
-
-def run_simulations_frame_global(df, num_simulations=4, clustering_method="KMEANS"):
-    data_hash = hash_of_df(df)
-    tasks = []
-    for i in range(num_simulations):
-        random.seed(int(data_hash, 16) + i)
-        kmeans_random_state = random.randint(0, 1000)
-        lgbm_random_state = random.randint(0, 1000)
-        tasks.append(
-            delayed(get_shap_values_for_dataset)(
-                df, clustering_method, 10, kmeans_random_state
-            )
-        )
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore")
-        all_shap_values = Parallel(n_jobs=-1, verbose=0)(tasks)
-    # print(all_shap_values)
-    # print(all_shap_values.shape)
-
-    # print(all_shap_values[0].index)
-    # print(all_shap_values[2].index)
-    avg_shap_values = pd.concat(all_shap_values).abs().reset_index().groupby(by=["ticker","date"]).mean()
-    # avg_shap_values.index = df.index
-    return avg_shap_values
-
-def run_simulations_global_importance(df, num_simulations=4, clustering_method="KMEANS"):
-    avg_shap_values = run_simulations_frame_global(df, num_simulations=num_simulations, clustering_method=clustering_method)
-    feature_importance = pd.DataFrame(
-        {
-            "feature": avg_shap_values.columns,
-            "importance": avg_shap_values.mean().values,
-        }
-    )
-    feature_importance["importance_percentile"] = stats.percentileofscore(
-        feature_importance["importance"], feature_importance["importance"]
-    )
-    return feature_importance.sort_values(
-        "importance_percentile", ascending=False
-    ).reset_index(drop=True)
-
-
-# Usage example:
-# importance_df = run_simulations(df, num_simulations=4, clustering_method='KMEANS')