sovai 0.2.55__tar.gz → 0.2.56__tar.gz

{sovai-0.2.55 → sovai-0.2.56}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sovai
-Version: 0.2.55
+Version: 0.2.56
 Summary: python for asset management
 License: Proprietary License
 Author: Derek Snow

{sovai-0.2.55 → sovai-0.2.56}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [project]
 name = "sovai"
-version = "0.2.55"
+version = "0.2.56"
 description = "python for asset management"
 authors = [
     {name = "Derek Snow", email = "d.snow@outlook.com"}

{sovai-0.2.55 → sovai-0.2.56}/sovai/__init__.py

@@ -5,7 +5,7 @@ from .api_config import read_key, save_key, ApiConfig
 from .get_data import data
 from .basic_auth import basic_auth
 from .token_auth import token_auth
-from .utils.file_management import save_or_update_tickers, save_or_update_codes
+from .utils.file_management import save_or_update_tickers, save_or_update_codes, update_data_files
 
 
 import warnings
@@ -18,10 +18,6 @@ try:
 except:
     __version__ = "0.2.49"  # Fallback to current version in pyproject.toml
 
-# Call the function to ensure tickers data is up-to-date
-save_or_update_tickers()
-
-save_or_update_codes()
 
 try:
     from .get_plots import plot
@@ -37,7 +33,7 @@ except ImportError as e:
     HAS_FULL_INSTALL = False
 
 
-__all__ = ['read_key', 'save_key', 'ApiConfig', 'data', 'basic_auth', 'token_auth']
+__all__ = ['read_key', 'save_key', 'ApiConfig', 'data', 'basic_auth', 'token_auth', 'update_data_files']
 
 if HAS_FULL_INSTALL:
     __all__ += ['plot', 'report', 'compute', 'nowcast', 'extension', 'sec_search', 'sec_filing', 'code', 'sec_graph']

{sovai-0.2.55 → sovai-0.2.56}/sovai/utils/client_side_s3_part_high.py

@@ -115,7 +115,7 @@ class PartitionFinder:
     
     @staticmethod
     def find_ticker_partitions(
-        ticker_base_path: str, 
+        ticker_base_path: str,
         tickers: List[str],
         has_year: bool = True,
         start_year: Optional[int] = None,
@@ -123,93 +123,141 @@ class PartitionFinder:
     ) -> List[Tuple[str, str]]:
         """
         Find all valid ticker partitions with optional year filtering.
-        
+
         Args:
             ticker_base_path: Base path to ticker partitions
             tickers: List of tickers to search for
             has_year: Whether ticker partitions include year subdirectories
             start_year: Optional starting year to filter by
             end_year: Optional ending year to filter by
-            
+
         Returns:
             List of tuples (path, ticker) for loading
         """
         fs = get_s3_filesystem()
         partitions = []
-        
+
         # Process each ticker
         for ticker in tickers:
-            ticker_path = f"{ticker_base_path}/ticker_partitioned={ticker}"
-            
+            # Construct the base path for this ticker's partitions
+            ticker_path = f"{ticker_base_path}/ticker_partitioned={ticker}" # e.g., .../base/ticker_partitioned=AAPL
+
             try:
-                # Case 1: No year partitioning
+                # Case 1: If the endpoint is configured with has_year=False
                 if not has_year:
+                    # Add the base ticker path directly
                     partitions.append((ticker_path, ticker))
+                    # Move to the next ticker
                     continue
-                    
-                # Case 2: Year partitioning
+
+                # Case 2: Endpoint has year partitioning (has_year=True)
+                # Call _extract_years. If start_year/end_year are None,
+                # _extract_years is responsible for returning ALL found years.
+                # If start_year/end_year have values, it returns filtered years.
+                verbose_mode.log(f"Extracting years for {ticker} at {ticker_path} (Range: {start_year}-{end_year})")
                 years = PartitionFinder._extract_years(
                     fs, ticker_path, start_year, end_year
                 )
-                    
-                # Generate specific year paths
+                verbose_mode.log(f"Found years for {ticker}: {years}")
+
+                # Generate specific year paths for EACH year returned by _extract_years
+                # If start_year/end_year were None, 'years' should contain ALL available years.
+                # If start_year/end_year were set, 'years' contains only the filtered years.
                 for year in years:
-                    year_path = f"{ticker_path}/year={year}"
+                    year_path = f"{ticker_path}/year={year}" # e.g., .../base/ticker_partitioned=AAPL/year=2023
                     partitions.append((year_path, ticker))
-                    
-                # If no years found but we know they exist, generate paths for requested years
+                    verbose_mode.log(f"Added partition task for: {year_path}")
+
+                # Fallback: This only runs if _extract_years found NO years AND a specific range WAS given.
+                # It does NOT run if start_year/end_year were None.
                 if not years and start_year and end_year:
+                    verbose_mode.log(f"No years found via listing for {ticker} in range {start_year}-{end_year}. Constructing paths based on range.")
                     for year in range(start_year, end_year + 1):
                         year_path = f"{ticker_path}/year={year}"
+                        # Note: These paths might not actually exist.
                         partitions.append((year_path, ticker))
-                        
+                        verbose_mode.log(f"Added potential partition task (fallback): {year_path}")
+
             except Exception as e:
-                verbose_mode.log(f"Error processing ticker {ticker}: {e}")
-        
+                # Log errors encountered for a specific ticker
+                logger.error(f"Error processing ticker {ticker} at path {ticker_path}: {e}", exc_info=True) # Use logger.error
+
         if not partitions:
-            verbose_mode.log(f"No ticker partitions found at {ticker_base_path} for tickers {tickers}")
-            
+            # Log if no partitions were found across ALL requested tickers matching the criteria
+            logger.warning(f"No ticker partitions ultimately found matching criteria. Base Path: {ticker_base_path}, Tickers: {tickers}, Has Year: {has_year}, Year Range: {start_year}-{end_year}") # Use logger.warning
+
         return partitions
     
     @staticmethod
     def _extract_years(
-        fs: S3FileSystem, 
-        ticker_path: str, 
-        start_year: Optional[int], 
+        fs: S3FileSystem,
+        ticker_path: str,
+        start_year: Optional[int],
         end_year: Optional[int]
     ) -> List[int]:
         """
-        Extract valid years from ticker subdirectories.
-        
+        Extract valid years from ticker subdirectories. [CORRECTED VERSION]
+
         Args:
             fs: S3 filesystem
-            ticker_path: Path to search for year directories
+            ticker_path: Path to search for year directories (e.g., .../ticker_partitioned=PFE)
             start_year: Optional starting year filter
             end_year: Optional ending year filter
-            
+
         Returns:
-            List of valid years within the specified range
+            List of valid years within the specified range, sorted.
         """
         years = []
-        
+        logger.debug(f"[_extract_years] Input: path='{ticker_path}', start={start_year}, end={end_year}")
         try:
-            subdirs = fs.ls(ticker_path)
-            for subdir in subdirs:
-                basename = os.path.basename(subdir.rstrip('/'))
+            logger.debug(f"[_extract_years] Listing contents of: {ticker_path} using get_file_info")
+            # Correct way to list directory contents using pyarrow fs API
+            selector = pa.fs.FileSelector(ticker_path, recursive=False)
+            subdirs_info = fs.get_file_info(selector) # Get list of FileInfo objects
+            logger.info(f"[_extract_years] Found {len(subdirs_info)} items in '{ticker_path}'")
+
+            for item_info in subdirs_info:
+                # Process only items that are directories
+                if item_info.type != pa.fs.FileType.Directory:
+                    logger.debug(f"[_extract_years] Skipping non-directory item: {item_info.path}")
+                    continue
+
+                subdir_path = item_info.path
+                basename = os.path.basename(subdir_path.rstrip('/')) # Get last component
+                logger.debug(f"[_extract_years] Processing directory item: '{basename}'")
+
                 if basename.startswith('year='):
-                    year_str = basename.split('=')[1]
+                    logger.debug(f"[_extract_years] Found potential year directory: {basename}")
+                    year_str = basename.split('=', 1)[1] # Extract year string
                     try:
                         year = int(year_str)
-                        # Apply year filtering if needed
-                        if ((start_year is None or year >= start_year) and
-                            (end_year is None or year <= end_year)):
+                        logger.debug(f"[_extract_years] Parsed year: {year}. Checking range: start={start_year}, end={end_year}")
+
+                        # Apply year filtering logic
+                        start_ok = (start_year is None or year >= start_year)
+                        end_ok = (end_year is None or year <= end_year)
+                        logger.debug(f"[_extract_years] Filter check for year {year}: start_ok={start_ok}, end_ok={end_ok}")
+
+                        if start_ok and end_ok:
+                            logger.info(f"[_extract_years] Year {year} is WITHIN range/criteria. Appending.")
                             years.append(year)
+                        else:
+                            logger.debug(f"[_extract_years] Year {year} is OUTSIDE range. Skipping.")
+
                     except ValueError:
-                        verbose_mode.log(f"Invalid year format: {year_str}")
+                        logger.warning(f"[_extract_years] Invalid year format: '{year_str}' in path {subdir_path}")
+                else:
+                    logger.debug(f"[_extract_years] Directory '{basename}' does not start with 'year='. Skipping.")
+
+        except FileNotFoundError:
+             # If the base ticker_path itself doesn't exist
+             logger.warning(f"[_extract_years] Directory not found for listing: {ticker_path}")
         except Exception as e:
-            verbose_mode.log(f"Error listing years: {e}")
-            
-        return years
+            # Catch any other errors during listing/processing
+            logger.error(f"[_extract_years] Error listing or processing directory {ticker_path}: {e}", exc_info=True)
+
+        logger.info(f"[_extract_years] Returning extracted years: {sorted(years)} for path '{ticker_path}'")
+        return sorted(years)
             
     @staticmethod
     def find_date_partitions(

{sovai-0.2.55 → sovai-0.2.56}/sovai/utils/file_management.py

@@ -70,3 +70,10 @@ def save_or_update_codes():
     codes_meta = pd.read_parquet(download_url)
     codes_meta.to_parquet(output_path)
 
+
+def update_data_files():
+    """
+    Manually updates the tickers and codes data files if they are outdated.
+    """
+    save_or_update_tickers()
+    save_or_update_codes()