sovai 0.2.55__tar.gz → 0.2.57__tar.gz

{sovai-0.2.55 → sovai-0.2.57}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sovai
-Version: 0.2.55
+Version: 0.2.57
 Summary: python for asset management
 License: Proprietary License
 Author: Derek Snow

{sovai-0.2.55 → sovai-0.2.57}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [project]
 name = "sovai"
-version = "0.2.55"
+version = "0.2.57"
 description = "python for asset management"
 authors = [
     {name = "Derek Snow", email = "d.snow@outlook.com"}

sovai-0.2.57/sovai/__init__.py

@@ -0,0 +1,167 @@
+"""
+Main SovAI SDK Tool Kit package
+"""
+from .api_config import read_key, save_key, ApiConfig
+from .basic_auth import basic_auth
+from .token_auth import token_auth
+from .utils.file_management import update_data_files # Keep update_data_files import
+
+
+import warnings
+warnings.filterwarnings("ignore")
+
+
+try:
+    from importlib.metadata import version
+    __version__ = version("sovai")
+except:
+    __version__ = "0.2.49"  # Fallback to current version in pyproject.toml
+
+
+# Lazy loading for core modules
+_data_module = None
+
+def data(*args, **kwargs):
+    """Lazy-loaded access to the data function."""
+    global _data_module
+    if _data_module is None:
+        from .get_data import data as _loaded_data
+        _data_module = _loaded_data
+    return _data_module(*args, **kwargs)
+
+
+# Lazy loading for full installation modules
+HAS_FULL_INSTALL = False
+_plot_module = None
+_report_module = None
+_compute_module = None
+_nowcast_module = None
+_extension_module = None
+_sec_search_module = None
+_sec_filing_module = None
+_code_module = None
+_sec_graph_module = None
+
+try:
+    # Attempt to import a module from the full installation to check if it's installed
+    # This import is just to check for ImportError, the actual function is lazy-loaded below
+    from .get_plots import plot as _ # Use _ to indicate this import is just for the check
+    HAS_FULL_INSTALL = True
+
+    def plot(*args, **kwargs):
+        """Lazy-loaded access to the plot function."""
+        global _plot_module
+        if _plot_module is None:
+            from .get_plots import plot as _loaded_plot
+            _plot_module = _loaded_plot
+        return _plot_module(*args, **kwargs)
+
+    def report(*args, **kwargs):
+        """Lazy-loaded access to the report function."""
+        global _report_module
+        if _report_module is None:
+            from .get_reports import report as _loaded_report
+            _report_module = _loaded_report
+        return _report_module(*args, **kwargs)
+
+    def compute(*args, **kwargs):
+        """Lazy-loaded access to the compute function."""
+        global _compute_module
+        if _compute_module is None:
+            from .get_compute import compute as _loaded_compute
+            _compute_module = _loaded_compute
+        return _compute_module(*args, **kwargs)
+
+    def nowcast(*args, **kwargs):
+        """Lazy-loaded access to the nowcast function."""
+        global _nowcast_module
+        if _nowcast_module is None:
+            from .studies.nowcasting import nowcast as _loaded_nowcast
+            _nowcast_module = _loaded_nowcast
+        return _nowcast_module(*args, **kwargs)
+
+    # extension is likely a class, so the lazy loader needs to return the class itself
+    def extension(*args, **kwargs):
+        """Lazy-loaded access to the CustomDataFrame extension class."""
+        global _extension_module
+        if _extension_module is None:
+            from .extensions.pandas_extensions import CustomDataFrame as _loaded_extension
+            _extension_module = _loaded_extension
+        # If called with args/kwargs, assume it's being instantiated
+        if args or kwargs:
+             return _extension_module(*args, **kwargs)
+        # Otherwise, return the class itself (e.g., for type checking or accessing class methods)
+        return _extension_module
+
+
+    def sec_search(*args, **kwargs):
+        """Lazy-loaded access to the sec_search function."""
+        global _sec_search_module
+        if _sec_search_module is None:
+            from .get_tools import sec_search as _loaded_sec_search
+            _sec_search_module = _loaded_sec_search
+        return _sec_search_module(*args, **kwargs)
+
+    def sec_filing(*args, **kwargs):
+        """Lazy-loaded access to the sec_filing function."""
+        global _sec_filing_module
+        if _sec_filing_module is None:
+            from .get_tools import sec_filing as _loaded_sec_filing
+            _sec_filing_module = _loaded_sec_filing
+        return _sec_filing_module(*args, **kwargs)
+
+    def code(*args, **kwargs):
+        """Lazy-loaded access to the code function."""
+        global _code_module
+        if _code_module is None:
+            from .get_tools import code as _loaded_code
+            _code_module = _loaded_code
+        return _code_module(*args, **kwargs)
+
+    def sec_graph(*args, **kwargs):
+        """Lazy-loaded access to the sec_graph function."""
+        global _sec_graph_module
+        if _sec_graph_module is None:
+            from .get_tools import sec_graph as _loaded_sec_graph
+            _sec_graph_module = _loaded_sec_graph
+        return _sec_graph_module(*args, **kwargs)
+
+
+except ImportError as e:
+    print("this is the lean installation, for full use sovai[full]")
+    # print(f"ImportError: {e}")
+    HAS_FULL_INSTALL = False
+
+    # Define dummy functions/objects for lean installation to avoid AttributeErrors
+    def plot(*args, **kwargs):
+        raise ImportError("Plotting requires the full installation. Install with 'pip install sovai[full]'")
+    def report(*args, **kwargs):
+        raise ImportError("Reporting requires the full installation. Install with 'pip install sovai[full]'")
+    def compute(*args, **kwargs):
+        raise ImportError("Compute requires the full installation. Install with 'pip install sovai[full]'")
+    def nowcast(*args, **kwargs):
+        raise ImportError("Nowcasting requires the full installation. Install with 'pip install sovai[full]'")
+    def extension(*args, **kwargs):
+         raise ImportError("Extensions require the full installation. Install with 'pip install sovai[full]'")
+    def sec_search(*args, **kwargs):
+        raise ImportError("SEC tools require the full installation. Install with 'pip install sovai[full]'")
+    def sec_filing(*args, **kwargs):
+        raise ImportError("SEC tools require the full installation. Install with 'pip install sovai[full]'")
+    def code(*args, **kwargs):
+        raise ImportError("SEC tools require the full installation. Install with 'pip install sovai[full]'")
+    def sec_graph(*args, **kwargs):
+        raise ImportError("SEC tools require the full installation. Install with 'pip install sovai[full]'")
+
+
+__all__ = [
+    'read_key', 'save_key', 'ApiConfig', 'basic_auth', 'token_auth',
+    'update_data_files', # Keep update_data_files in __all__
+    'data', # Add data to __all__
+]
+
+# Add full installation lazy-loaded attributes to __all__ if available
+if HAS_FULL_INSTALL:
+    __all__ += [
+        'plot', 'report', 'compute', 'nowcast', 'extension',
+        'sec_search', 'sec_filing', 'code', 'sec_graph'
+    ]

{sovai-0.2.55 → sovai-0.2.57}/sovai/utils/client_side_s3_part_high.py

@@ -115,7 +115,7 @@ class PartitionFinder:
     
     @staticmethod
     def find_ticker_partitions(
-        ticker_base_path: str, 
+        ticker_base_path: str,
         tickers: List[str],
         has_year: bool = True,
         start_year: Optional[int] = None,
@@ -123,93 +123,141 @@ class PartitionFinder:
     ) -> List[Tuple[str, str]]:
         """
         Find all valid ticker partitions with optional year filtering.
-        
+
         Args:
             ticker_base_path: Base path to ticker partitions
             tickers: List of tickers to search for
             has_year: Whether ticker partitions include year subdirectories
             start_year: Optional starting year to filter by
             end_year: Optional ending year to filter by
-            
+
         Returns:
             List of tuples (path, ticker) for loading
         """
         fs = get_s3_filesystem()
         partitions = []
-        
+
         # Process each ticker
         for ticker in tickers:
-            ticker_path = f"{ticker_base_path}/ticker_partitioned={ticker}"
-            
+            # Construct the base path for this ticker's partitions
+            ticker_path = f"{ticker_base_path}/ticker_partitioned={ticker}" # e.g., .../base/ticker_partitioned=AAPL
+
             try:
-                # Case 1: No year partitioning
+                # Case 1: If the endpoint is configured with has_year=False
                 if not has_year:
+                    # Add the base ticker path directly
                     partitions.append((ticker_path, ticker))
+                    # Move to the next ticker
                     continue
-                    
-                # Case 2: Year partitioning
+
+                # Case 2: Endpoint has year partitioning (has_year=True)
+                # Call _extract_years. If start_year/end_year are None,
+                # _extract_years is responsible for returning ALL found years.
+                # If start_year/end_year have values, it returns filtered years.
+                verbose_mode.log(f"Extracting years for {ticker} at {ticker_path} (Range: {start_year}-{end_year})")
                 years = PartitionFinder._extract_years(
                     fs, ticker_path, start_year, end_year
                 )
-                    
-                # Generate specific year paths
+                verbose_mode.log(f"Found years for {ticker}: {years}")
+
+                # Generate specific year paths for EACH year returned by _extract_years
+                # If start_year/end_year were None, 'years' should contain ALL available years.
+                # If start_year/end_year were set, 'years' contains only the filtered years.
                 for year in years:
-                    year_path = f"{ticker_path}/year={year}"
+                    year_path = f"{ticker_path}/year={year}" # e.g., .../base/ticker_partitioned=AAPL/year=2023
                     partitions.append((year_path, ticker))
-                    
-                # If no years found but we know they exist, generate paths for requested years
+                    verbose_mode.log(f"Added partition task for: {year_path}")
+
+                # Fallback: This only runs if _extract_years found NO years AND a specific range WAS given.
+                # It does NOT run if start_year/end_year were None.
                 if not years and start_year and end_year:
+                    verbose_mode.log(f"No years found via listing for {ticker} in range {start_year}-{end_year}. Constructing paths based on range.")
                     for year in range(start_year, end_year + 1):
                         year_path = f"{ticker_path}/year={year}"
+                        # Note: These paths might not actually exist.
                         partitions.append((year_path, ticker))
-                        
+                        verbose_mode.log(f"Added potential partition task (fallback): {year_path}")
+
             except Exception as e:
-                verbose_mode.log(f"Error processing ticker {ticker}: {e}")
-        
+                # Log errors encountered for a specific ticker
+                logger.error(f"Error processing ticker {ticker} at path {ticker_path}: {e}", exc_info=True) # Use logger.error
+
         if not partitions:
-            verbose_mode.log(f"No ticker partitions found at {ticker_base_path} for tickers {tickers}")
-            
+            # Log if no partitions were found across ALL requested tickers matching the criteria
+            logger.warning(f"No ticker partitions ultimately found matching criteria. Base Path: {ticker_base_path}, Tickers: {tickers}, Has Year: {has_year}, Year Range: {start_year}-{end_year}") # Use logger.warning
+
         return partitions
     
     @staticmethod
     def _extract_years(
-        fs: S3FileSystem, 
-        ticker_path: str, 
-        start_year: Optional[int], 
+        fs: S3FileSystem,
+        ticker_path: str,
+        start_year: Optional[int],
         end_year: Optional[int]
     ) -> List[int]:
         """
-        Extract valid years from ticker subdirectories.
-        
+        Extract valid years from ticker subdirectories. [CORRECTED VERSION]
+
         Args:
             fs: S3 filesystem
-            ticker_path: Path to search for year directories
+            ticker_path: Path to search for year directories (e.g., .../ticker_partitioned=PFE)
             start_year: Optional starting year filter
             end_year: Optional ending year filter
-            
+
         Returns:
-            List of valid years within the specified range
+            List of valid years within the specified range, sorted.
         """
         years = []
-        
+        logger.debug(f"[_extract_years] Input: path='{ticker_path}', start={start_year}, end={end_year}")
         try:
-            subdirs = fs.ls(ticker_path)
-            for subdir in subdirs:
-                basename = os.path.basename(subdir.rstrip('/'))
+            logger.debug(f"[_extract_years] Listing contents of: {ticker_path} using get_file_info")
+            # Correct way to list directory contents using pyarrow fs API
+            selector = pa.fs.FileSelector(ticker_path, recursive=False)
+            subdirs_info = fs.get_file_info(selector) # Get list of FileInfo objects
+            logger.info(f"[_extract_years] Found {len(subdirs_info)} items in '{ticker_path}'")
+
+            for item_info in subdirs_info:
+                # Process only items that are directories
+                if item_info.type != pa.fs.FileType.Directory:
+                    logger.debug(f"[_extract_years] Skipping non-directory item: {item_info.path}")
+                    continue
+
+                subdir_path = item_info.path
+                basename = os.path.basename(subdir_path.rstrip('/')) # Get last component
+                logger.debug(f"[_extract_years] Processing directory item: '{basename}'")
+
                 if basename.startswith('year='):
-                    year_str = basename.split('=')[1]
+                    logger.debug(f"[_extract_years] Found potential year directory: {basename}")
+                    year_str = basename.split('=', 1)[1] # Extract year string
                     try:
                         year = int(year_str)
-                        # Apply year filtering if needed
-                        if ((start_year is None or year >= start_year) and
-                            (end_year is None or year <= end_year)):
+                        logger.debug(f"[_extract_years] Parsed year: {year}. Checking range: start={start_year}, end={end_year}")
+
+                        # Apply year filtering logic
+                        start_ok = (start_year is None or year >= start_year)
+                        end_ok = (end_year is None or year <= end_year)
+                        logger.debug(f"[_extract_years] Filter check for year {year}: start_ok={start_ok}, end_ok={end_ok}")
+
+                        if start_ok and end_ok:
+                            logger.info(f"[_extract_years] Year {year} is WITHIN range/criteria. Appending.")
                             years.append(year)
+                        else:
+                            logger.debug(f"[_extract_years] Year {year} is OUTSIDE range. Skipping.")
+
                     except ValueError:
-                        verbose_mode.log(f"Invalid year format: {year_str}")
+                        logger.warning(f"[_extract_years] Invalid year format: '{year_str}' in path {subdir_path}")
+                else:
+                    logger.debug(f"[_extract_years] Directory '{basename}' does not start with 'year='. Skipping.")
+
+        except FileNotFoundError:
+             # If the base ticker_path itself doesn't exist
+             logger.warning(f"[_extract_years] Directory not found for listing: {ticker_path}")
         except Exception as e:
-            verbose_mode.log(f"Error listing years: {e}")
-            
-        return years
+            # Catch any other errors during listing/processing
+            logger.error(f"[_extract_years] Error listing or processing directory {ticker_path}: {e}", exc_info=True)
+
+        logger.info(f"[_extract_years] Returning extracted years: {sorted(years)} for path '{ticker_path}'")
+        return sorted(years)
             
     @staticmethod
     def find_date_partitions(

{sovai-0.2.55 → sovai-0.2.57}/sovai/utils/file_management.py

@@ -70,3 +70,10 @@ def save_or_update_codes():
     codes_meta = pd.read_parquet(download_url)
     codes_meta.to_parquet(output_path)
 
+
+def update_data_files():
+    """
+    Manually updates the tickers and codes data files if they are outdated.
+    """
+    save_or_update_tickers()
+    save_or_update_codes()

sovai-0.2.55/sovai/__init__.py

@@ -1,43 +0,0 @@
-"""
-Main SovAI SDK Tool Kit package
-"""
-from .api_config import read_key, save_key, ApiConfig
-from .get_data import data
-from .basic_auth import basic_auth
-from .token_auth import token_auth
-from .utils.file_management import save_or_update_tickers, save_or_update_codes
-
-
-import warnings
-warnings.filterwarnings("ignore")
-
-
-try:
-    from importlib.metadata import version
-    __version__ = version("sovai")
-except:
-    __version__ = "0.2.49"  # Fallback to current version in pyproject.toml
-
-# Call the function to ensure tickers data is up-to-date
-save_or_update_tickers()
-
-save_or_update_codes()
-
-try:
-    from .get_plots import plot
-    from .get_reports import report
-    from .get_compute import compute
-    from .studies.nowcasting import nowcast
-    from .extensions.pandas_extensions import CustomDataFrame as extension
-    from .get_tools import sec_search, sec_filing, code, sec_graph
-    HAS_FULL_INSTALL = True
-except ImportError as e:
-    print("this is the lean installation, for full use sovai[full]")
-    # print(f"ImportError: {e}")
-    HAS_FULL_INSTALL = False
-
-
-__all__ = ['read_key', 'save_key', 'ApiConfig', 'data', 'basic_auth', 'token_auth']
-
-if HAS_FULL_INSTALL:
-    __all__ += ['plot', 'report', 'compute', 'nowcast', 'extension', 'sec_search', 'sec_filing', 'code', 'sec_graph']