sovai 0.2.54__tar.gz → 0.2.56__tar.gz

{sovai-0.2.54 → sovai-0.2.56}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sovai
-Version: 0.2.54
+Version: 0.2.56
 Summary: python for asset management
 License: Proprietary License
 Author: Derek Snow
@@ -8,8 +8,6 @@ Author-email: d.snow@outlook.com
 Requires-Python: >=3.10
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Provides-Extra: full
@@ -51,6 +49,7 @@ Requires-Dist: scikit-learn ; extra == "full"
 Requires-Dist: scipy ; extra == "full"
 Requires-Dist: seaborn (>=0.13.2)
 Requires-Dist: shap ; extra == "full"
+Requires-Dist: skfolio (>=0.9.0,<0.10.0)
 Requires-Dist: skfolio ; extra == "full"
 Requires-Dist: statsforecast ; extra == "full"
 Requires-Dist: tensorly ; extra == "full"

{sovai-0.2.54 → sovai-0.2.56}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [project]
 name = "sovai"
-version = "0.2.54"
+version = "0.2.56"
 description = "python for asset management"
 authors = [
     {name = "Derek Snow", email = "d.snow@outlook.com"}
@@ -37,6 +37,7 @@ dependencies = [
   "fastapi-cache>=0.1.0",
   "fastapi-cache2[redis]>=0.2.1",
   "tqdm (>=4.67.1,<5.0.0)",
+  "skfolio (>=0.9.0,<0.10.0)"
 ]
 
 [tool.poetry]
@@ -66,7 +67,7 @@ exclude = [
 
 # Keep Poetry's dependencies section for detailed version specs
 [tool.poetry.dependencies]
-python = "^3.10"
+python = "^3.12"
 pandas = ">=1.0"
 numpy = ">=1.20"
 pyarrow = ">=5.0"
@@ -145,3 +146,5 @@ traitlets = "^5.14.3"
 jupyterlab = "^4.2.5"
 pytest = "^7.0"
 black = "^22.0"
+jupyter = "^1.1.1"
+ipykernel = "^6.29.5"

{sovai-0.2.54 → sovai-0.2.56}/sovai/__init__.py

@@ -5,7 +5,7 @@ from .api_config import read_key, save_key, ApiConfig
 from .get_data import data
 from .basic_auth import basic_auth
 from .token_auth import token_auth
-from .utils.file_management import save_or_update_tickers, save_or_update_codes
+from .utils.file_management import save_or_update_tickers, save_or_update_codes, update_data_files
 
 
 import warnings
@@ -18,10 +18,6 @@ try:
 except:
     __version__ = "0.2.49"  # Fallback to current version in pyproject.toml
 
-# Call the function to ensure tickers data is up-to-date
-save_or_update_tickers()
-
-save_or_update_codes()
 
 try:
     from .get_plots import plot
@@ -32,11 +28,12 @@ try:
     from .get_tools import sec_search, sec_filing, code, sec_graph
     HAS_FULL_INSTALL = True
 except ImportError as e:
-    print(f"ImportError: {e}")
+    print("this is the lean installation, for full use sovai[full]")
+    # print(f"ImportError: {e}")
     HAS_FULL_INSTALL = False
 
 
-__all__ = ['read_key', 'save_key', 'ApiConfig', 'data', 'basic_auth', 'token_auth']
+__all__ = ['read_key', 'save_key', 'ApiConfig', 'data', 'basic_auth', 'token_auth', 'update_data_files']
 
 if HAS_FULL_INSTALL:
     __all__ += ['plot', 'report', 'compute', 'nowcast', 'extension', 'sec_search', 'sec_filing', 'code', 'sec_graph']

{sovai-0.2.54 → sovai-0.2.56}/sovai/plots/news/news_plots.py

@@ -4,7 +4,34 @@ import ipywidgets as widgets
 from ipywidgets import interact
 from sovai import data
 
+
+import pandas as pd
+import numpy as np
+import plotly.graph_objects as go
+from plotly.subplots import make_subplots
+import dash
+from dash import dcc, html
+from dash.dependencies import Input, Output, State
 import random
+from scipy.optimize import minimize
+from scipy import stats
+from sovai.utils.port_manager import get_unique_port
+
+
+import pandas as pd
+import numpy as np
+
+import random
+
+
+import dash
+from dash import dcc, html, Input, Output, State
+import dash_bootstrap_components as dbc
+import plotly.graph_objs as go
+import pandas as pd
+import numpy as np
+
+
 
 # Global dictionary to keep track of apps and their ports
 app_ports = {}
@@ -118,23 +145,6 @@ def plot_above_sentiment_returns(df_news=None, tickers=None):
         fig.show()
 
 
-
-import pandas as pd
-import numpy as np
-import plotly.graph_objects as go
-from plotly.subplots import make_subplots
-import dash
-from dash import dcc, html
-from dash.dependencies import Input, Output, State
-import random
-from scipy.optimize import minimize
-from scipy import stats
-from sovai.utils.port_manager import get_unique_port
-
-
-
-import pandas as pd
-import numpy as np
 # from statsmodels.tsa.stattools import adfuller, coint
 # from statsmodels.tsa.vector_ar.vecm import VECM
 
@@ -550,12 +560,6 @@ def plot_news_daily(ticker='NVDA'):
 # plot_news_daily(ticker='AAPL')
 
 
-import dash
-from dash import dcc, html, Input, Output, State
-import dash_bootstrap_components as dbc
-import plotly.graph_objs as go
-import pandas as pd
-import numpy as np
 
 def dash_news_ts_analysis(df_sentiment, df_polarity, df_topic):
     # Dynamically get all topics from df_sentiment

{sovai-0.2.54 → sovai-0.2.56}/sovai/utils/client_side_s3_part_high.py

@@ -115,7 +115,7 @@ class PartitionFinder:
     
     @staticmethod
     def find_ticker_partitions(
-        ticker_base_path: str, 
+        ticker_base_path: str,
         tickers: List[str],
         has_year: bool = True,
         start_year: Optional[int] = None,
@@ -123,93 +123,141 @@ class PartitionFinder:
     ) -> List[Tuple[str, str]]:
         """
         Find all valid ticker partitions with optional year filtering.
-        
+
         Args:
             ticker_base_path: Base path to ticker partitions
             tickers: List of tickers to search for
             has_year: Whether ticker partitions include year subdirectories
             start_year: Optional starting year to filter by
             end_year: Optional ending year to filter by
-            
+
         Returns:
             List of tuples (path, ticker) for loading
         """
         fs = get_s3_filesystem()
         partitions = []
-        
+
         # Process each ticker
         for ticker in tickers:
-            ticker_path = f"{ticker_base_path}/ticker_partitioned={ticker}"
-            
+            # Construct the base path for this ticker's partitions
+            ticker_path = f"{ticker_base_path}/ticker_partitioned={ticker}" # e.g., .../base/ticker_partitioned=AAPL
+
             try:
-                # Case 1: No year partitioning
+                # Case 1: If the endpoint is configured with has_year=False
                 if not has_year:
+                    # Add the base ticker path directly
                     partitions.append((ticker_path, ticker))
+                    # Move to the next ticker
                     continue
-                    
-                # Case 2: Year partitioning
+
+                # Case 2: Endpoint has year partitioning (has_year=True)
+                # Call _extract_years. If start_year/end_year are None,
+                # _extract_years is responsible for returning ALL found years.
+                # If start_year/end_year have values, it returns filtered years.
+                verbose_mode.log(f"Extracting years for {ticker} at {ticker_path} (Range: {start_year}-{end_year})")
                 years = PartitionFinder._extract_years(
                     fs, ticker_path, start_year, end_year
                 )
-                    
-                # Generate specific year paths
+                verbose_mode.log(f"Found years for {ticker}: {years}")
+
+                # Generate specific year paths for EACH year returned by _extract_years
+                # If start_year/end_year were None, 'years' should contain ALL available years.
+                # If start_year/end_year were set, 'years' contains only the filtered years.
                 for year in years:
-                    year_path = f"{ticker_path}/year={year}"
+                    year_path = f"{ticker_path}/year={year}" # e.g., .../base/ticker_partitioned=AAPL/year=2023
                     partitions.append((year_path, ticker))
-                    
-                # If no years found but we know they exist, generate paths for requested years
+                    verbose_mode.log(f"Added partition task for: {year_path}")
+
+                # Fallback: This only runs if _extract_years found NO years AND a specific range WAS given.
+                # It does NOT run if start_year/end_year were None.
                 if not years and start_year and end_year:
+                    verbose_mode.log(f"No years found via listing for {ticker} in range {start_year}-{end_year}. Constructing paths based on range.")
                     for year in range(start_year, end_year + 1):
                         year_path = f"{ticker_path}/year={year}"
+                        # Note: These paths might not actually exist.
                         partitions.append((year_path, ticker))
-                        
+                        verbose_mode.log(f"Added potential partition task (fallback): {year_path}")
+
             except Exception as e:
-                verbose_mode.log(f"Error processing ticker {ticker}: {e}")
-        
+                # Log errors encountered for a specific ticker
+                logger.error(f"Error processing ticker {ticker} at path {ticker_path}: {e}", exc_info=True) # Use logger.error
+
         if not partitions:
-            verbose_mode.log(f"No ticker partitions found at {ticker_base_path} for tickers {tickers}")
-            
+            # Log if no partitions were found across ALL requested tickers matching the criteria
+            logger.warning(f"No ticker partitions ultimately found matching criteria. Base Path: {ticker_base_path}, Tickers: {tickers}, Has Year: {has_year}, Year Range: {start_year}-{end_year}") # Use logger.warning
+
         return partitions
     
     @staticmethod
     def _extract_years(
-        fs: S3FileSystem, 
-        ticker_path: str, 
-        start_year: Optional[int], 
+        fs: S3FileSystem,
+        ticker_path: str,
+        start_year: Optional[int],
         end_year: Optional[int]
     ) -> List[int]:
         """
-        Extract valid years from ticker subdirectories.
-        
+        Extract valid years from ticker subdirectories. [CORRECTED VERSION]
+
         Args:
             fs: S3 filesystem
-            ticker_path: Path to search for year directories
+            ticker_path: Path to search for year directories (e.g., .../ticker_partitioned=PFE)
             start_year: Optional starting year filter
             end_year: Optional ending year filter
-            
+
         Returns:
-            List of valid years within the specified range
+            List of valid years within the specified range, sorted.
         """
         years = []
-        
+        logger.debug(f"[_extract_years] Input: path='{ticker_path}', start={start_year}, end={end_year}")
         try:
-            subdirs = fs.ls(ticker_path)
-            for subdir in subdirs:
-                basename = os.path.basename(subdir.rstrip('/'))
+            logger.debug(f"[_extract_years] Listing contents of: {ticker_path} using get_file_info")
+            # Correct way to list directory contents using pyarrow fs API
+            selector = pa.fs.FileSelector(ticker_path, recursive=False)
+            subdirs_info = fs.get_file_info(selector) # Get list of FileInfo objects
+            logger.info(f"[_extract_years] Found {len(subdirs_info)} items in '{ticker_path}'")
+
+            for item_info in subdirs_info:
+                # Process only items that are directories
+                if item_info.type != pa.fs.FileType.Directory:
+                    logger.debug(f"[_extract_years] Skipping non-directory item: {item_info.path}")
+                    continue
+
+                subdir_path = item_info.path
+                basename = os.path.basename(subdir_path.rstrip('/')) # Get last component
+                logger.debug(f"[_extract_years] Processing directory item: '{basename}'")
+
                 if basename.startswith('year='):
-                    year_str = basename.split('=')[1]
+                    logger.debug(f"[_extract_years] Found potential year directory: {basename}")
+                    year_str = basename.split('=', 1)[1] # Extract year string
                     try:
                         year = int(year_str)
-                        # Apply year filtering if needed
-                        if ((start_year is None or year >= start_year) and
-                            (end_year is None or year <= end_year)):
+                        logger.debug(f"[_extract_years] Parsed year: {year}. Checking range: start={start_year}, end={end_year}")
+
+                        # Apply year filtering logic
+                        start_ok = (start_year is None or year >= start_year)
+                        end_ok = (end_year is None or year <= end_year)
+                        logger.debug(f"[_extract_years] Filter check for year {year}: start_ok={start_ok}, end_ok={end_ok}")
+
+                        if start_ok and end_ok:
+                            logger.info(f"[_extract_years] Year {year} is WITHIN range/criteria. Appending.")
                             years.append(year)
+                        else:
+                            logger.debug(f"[_extract_years] Year {year} is OUTSIDE range. Skipping.")
+
                     except ValueError:
-                        verbose_mode.log(f"Invalid year format: {year_str}")
+                        logger.warning(f"[_extract_years] Invalid year format: '{year_str}' in path {subdir_path}")
+                else:
+                    logger.debug(f"[_extract_years] Directory '{basename}' does not start with 'year='. Skipping.")
+
+        except FileNotFoundError:
+             # If the base ticker_path itself doesn't exist
+             logger.warning(f"[_extract_years] Directory not found for listing: {ticker_path}")
         except Exception as e:
-            verbose_mode.log(f"Error listing years: {e}")
-            
-        return years
+            # Catch any other errors during listing/processing
+            logger.error(f"[_extract_years] Error listing or processing directory {ticker_path}: {e}", exc_info=True)
+
+        logger.info(f"[_extract_years] Returning extracted years: {sorted(years)} for path '{ticker_path}'")
+        return sorted(years)
             
     @staticmethod
     def find_date_partitions(

{sovai-0.2.54 → sovai-0.2.56}/sovai/utils/file_management.py

@@ -70,3 +70,10 @@ def save_or_update_codes():
     codes_meta = pd.read_parquet(download_url)
     codes_meta.to_parquet(output_path)
 
+
+def update_data_files():
+    """
+    Manually updates the tickers and codes data files if they are outdated.
+    """
+    save_or_update_tickers()
+    save_or_update_codes()