sovai 0.2.40__tar.gz → 0.2.42__tar.gz

{sovai-0.2.40 → sovai-0.2.42}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sovai
-Version: 0.2.40
+Version: 0.2.42
 Summary: python for asset management
 License: Proprietary License
 Author: Derek Snow
@@ -17,6 +17,9 @@ Requires-Dist: boto3 (>=1.20)
 Requires-Dist: dash (>=2.0) ; extra == "full"
 Requires-Dist: dash-bootstrap-components (>=1.6.0) ; extra == "full"
 Requires-Dist: edgar-tool (>=1.3.3,<2.0.0)
+Requires-Dist: fastapi-cache (>=0.1.0,<0.2.0)
+Requires-Dist: fastapi-cache2[redis] (>=0.2.1,<0.3.0)
+Requires-Dist: gcsfs (>=2024.10.0,<2025.0.0)
 Requires-Dist: great-tables (>=0.9) ; extra == "full"
 Requires-Dist: ipywidgets (>=8.1.3) ; extra == "full"
 Requires-Dist: joblib (>=1.0)

{sovai-0.2.40 → sovai-0.2.42}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sovai"
-version = "0.2.40"
+version = "0.2.42"
 description = "python for asset management"
 authors = ["Derek Snow <d.snow@outlook.com>"]
 license = "Proprietary License"
@@ -66,6 +66,9 @@ pyvis = "^0.3.2"
 networkx = "^3.3"
 seaborn = "^0.13.2"
 s3fs = "^2024.10.0"
+gcsfs = "^2024.10.0"
+fastapi-cache = "^0.1.0"
+fastapi-cache2 = {extras = ["redis"], version = "^0.2.1"}
 
 
 [tool.poetry.extras]
@@ -103,5 +106,3 @@ black = "^22.0"
 
 
 [build-system]
-requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"

{sovai-0.2.40 → sovai-0.2.42}/sovai/get_data.py

@@ -23,7 +23,7 @@ import plotly.graph_objects as go
 # Local imports
 from sovai.api_config import ApiConfig
 from sovai.errors.sovai_errors import InvalidInputData
-from sovai.utils.converter import convert_data2df
+from sovai.utils.converter import convert_data2df, process_dataframe
 from sovai.utils.stream import stream_data, stream_data_pyarrow
 from sovai.utils.datetime_formats import datetime_format
 from sovai.utils.client_side import client_side_frame
@@ -307,7 +307,7 @@ ENDPOINT_TO_TICKER = {
    "/news/activeness": "",
    "/news/pronouns": "",
    "/news/word_count": "",
-   "/news/sentiment_score": None,
+   "/news/sentiment_score": "", # was None before?
    "/insider/trading": "",
    "/wikipedia/views": "",
    "/accounting/weekly": "",
@@ -343,7 +343,9 @@ ENDPOINT_TO_TICKER = {
 
     "/patents/applications": "",
     "/patents/grants": "",
-    "/clinical_trials": "",
+    
+    "/clinical/trials": "",
+
     "/spending/awards": "",
     "/spending/compensation": "",
     "/spending/competition": "",
@@ -358,11 +360,11 @@ ENDPOINT_TO_TICKER = {
 
 # Define endpoint sets for different processing methods
 CLIENT_SIDE_ENDPOINTS_GCS = {
-   "ratios/relative",
+#    "ratios/relative",
    "market/prices",
    "market/closeadj",
    "short/volume",
-   "complaints/public",
+#    "complaints/public",
    "complaints/private",
    "lobbying/public",
 }
@@ -379,7 +381,10 @@ CLIENT_SIDE_ENDPOINTS_S3 = {
 CLIENT_SIDE_ENDPOINTS_S3_PART_HIGH = {
    "patents/applications",
    "patents/grants",
-   "clinical_trials",
+
+   "clinical/trials",
+
+
    "spending/awards",
    "spending/compensation",
    "spending/competition",
@@ -387,8 +392,29 @@ CLIENT_SIDE_ENDPOINTS_S3_PART_HIGH = {
    "spending/entities", 
    "spending/location",
    "spending/product",
+
    "spending/transactions",
-   "lobbying/data"
+   "lobbying/data",
+
+   "accounting/weekly",
+   "ratios/normal",
+   "ratios/relative",
+
+   "complaints/public",
+
+   "factors/accounting",
+   "factors/alternative",
+   "factors/comprehensive",
+   "factors/coefficients",
+   "factors/standard_errors",
+   "factors/t_statistics",
+   "factors/model_metrics",
+
+   "breakout",
+
+   "corprisk/risks",
+
+   "visas/h1b"
 }
 
 
@@ -999,7 +1025,6 @@ def get_api_handler() -> ApiRequestHandler:
         logger=logger
     )
 
-
 def data(
     endpoint: str,
     tickers: Optional[Union[str, List[str]]] = None,
@@ -1018,6 +1043,7 @@ def data(
     frequency: Optional[str] = None,
     verbose: bool = False,
     full_history: bool = False,
+    source: Optional[str] = None,
 ) -> Union[DataFrameType, go.Figure, None]:
     """
     Main function to retrieve data from the API.
@@ -1057,6 +1083,8 @@ def data(
     # Set verbose mode based on parameter
     verbose_mode.toggle_verbose(verbose)
     verbose_mode.log(f"Starting data request for endpoint: {endpoint}")
+    
+    result_data = None  # Initialize the return value
 
     # Initialize and process parameters
     params = params or {}
@@ -1081,6 +1109,7 @@ def data(
                 parquet=params.get("parquet", parquet),
                 frequency=params.get("frequency", frequency),
                 full_history=params.get("full_history", full_history),
+                source=params.get("source", source),  # Add source parameter here
             )
         )
     except Exception as e:
@@ -1109,169 +1138,195 @@ def data(
 
     if not purge_cache and cache_key in _query_cache:
         verbose_mode.log("Using cached data")
-        return _query_cache[cache_key]
-
-    # Process normalized endpoint for handler matching
-    normalized_endpoint = normalize_endpoint(endpoint)
-
-    # Try client-side handlers for specific endpoints
-    for endpoint_set, handler_func, message in ENDPOINT_HANDLERS:
-        if (
-            normalized_endpoint in endpoint_set and
-            (tickers is not None or start_date is not None or end_date is not None) and
-            frequency is None
-        ):
-            verbose_mode.log(message)
-            verbose_mode.log(f"Calling client-side handler with tickers={tickers}, start_date={start_date}, end_date={end_date}")
-            try:
-                result = handler_func(
-                    normalized_endpoint, tickers, columns, start_date, end_date
-                )
-                _query_cache[cache_key] = result
-                return result
-            except Exception as e:
-                logger.error(f"Client-side handler error for {normalized_endpoint}: {e}")
-                # Fall through to API request if client-side handler fails
-
-    try:
-        # Perform ticker mapping if needed
-        if tickers is not None and not is_all(tickers):
-            verbose_mode.log("Mapping ticker symbols")
-            params = ticker_mapper(params, verbose)
-
-        # Create API request handler
-        api_handler = get_api_handler()
-        
-        # Make API request with retry logic
-        verbose_mode.log("Sending API request")
-        res = api_handler.get(
-            endpoint=endpoint,
-            params=params,
-            body=body,
-            stream=True
-        )
-        
-        verbose_mode.log(f"Response received - Status: {res.status_code}, Content-Type: {res.headers.get('content-type')}")
-        
-        # Get appropriate ticker for endpoint
-        tickers = get_ticker_from_endpoint(endpoint, tickers, ENDPOINT_TO_TICKER)
-        verbose_mode.log(f"Using tickers: {tickers}")
-
-        # Get response metadata
-        data_format = res.headers.get("X-Data-Format")
-        content_type = res.headers["content-type"]
-        plot_header = res.headers.get("X-Plotly-Data")
-        result_data = None
-        
-        # Handle binary data (parquet)
-        if (content_type == "application/octet-stream") and not plot_header:
-            if data_format == "pyarrow":
-                verbose_mode.log("Processing pyarrow data stream")
-                result_data = stream_data_pyarrow(res)
-            else:
-                verbose_mode.log("Processing binary data stream")
-                result_data = stream_data(res)
-            
-            # Wrap in CustomDataFrame if available
-            if HAS_FULL_INSTALL:
-                result_data = CustomDataFrame(result_data)
-            else:
-                result_data = pd.DataFrame(result_data)
-            
-            _query_cache[cache_key] = result_data
-            return result_data
-        
-        # Handle 'all tickers' case
-        if is_all(tickers):
-            verbose_mode.log("Processing 'all tickers' response")
-            urls = [u.strip() for u in res.text.strip('"').split(',') if u.strip()]
-            result_data = None
-            
-            # Try each URL until successful
-            for i, url in enumerate(urls):
-                verbose_mode.log(f"Attempting URL {i+1}/{len(urls)}: {url}")
+        result_data = _query_cache[cache_key]
+    else:
+        # Process normalized endpoint for handler matching
+        normalized_endpoint = normalize_endpoint(endpoint)
+
+        # Try client-side handlers for specific endpoints
+        handler_result = None
+        for endpoint_set, handler_func, message in ENDPOINT_HANDLERS:
+            if (
+                normalized_endpoint in endpoint_set and
+                (tickers is not None or start_date is not None or end_date is not None) and
+                frequency is None
+            ):
+                verbose_mode.log(message)
+                verbose_mode.log(f"Calling client-side handler with tickers={tickers}, start_date={start_date}, end_date={end_date}")
                 try:
-                    result_data = read_parquet(url, use_polars=use_polars)
-                    verbose_mode.log(f"Successfully downloaded data from URL {i+1}")
+                    handler_result = handler_func(
+                        normalized_endpoint, tickers, columns, start_date, end_date
+                    )
+                    _query_cache[cache_key] = handler_result
+                    result_data = handler_result
                     break
                 except Exception as e:
-                    verbose_mode.log(f"Failed to download from URL {i+1}: {str(e)}")
-            
-            if result_data is None:
-                error_msg = "Failed to download data from all provided URLs"
-                logger.error(error_msg)
-                raise Exception(error_msg)
-            
-            # Apply filters
-            verbose_mode.log("Applying filters to data")
-            result_data = filter_data(
-                result_data, 
-                columns=columns, 
-                start_date=start_date, 
-                end_date=end_date, 
-                use_polars=use_polars
-            )
+                    logger.error(f"Client-side handler error for {normalized_endpoint}: {e}")
+                    # Fall through to API request if client-side handler fails
 
-            # Wrap in CustomDataFrame if available
-            if HAS_FULL_INSTALL and not use_polars:
-                result_data = CustomDataFrame(result_data)
+        # Only proceed with API request if no handler result
+        if result_data is None:
+            try:
+                # Perform ticker mapping if needed
+                if tickers is not None and not is_all(tickers):
+                    verbose_mode.log("Mapping ticker symbols")
+                    params = ticker_mapper(params, verbose)
 
-            _query_cache[cache_key] = result_data
-            
-        # Handle JSON data
-        elif not plot_header:
-            verbose_mode.log("Processing JSON response")
-            json_data = res.json()
-            if HAS_FULL_INSTALL:
-                result_data = CustomDataFrame(convert_data2df(json_data))
-            else:
-                result_data = pd.DataFrame(convert_data2df(json_data))
-
-            _query_cache[cache_key] = result_data
-
-        # Handle plot data
-        if plot_header:
-            verbose_mode.log("Processing plot data")
-            if HAS_FULL_INSTALL:
-                import pickle
-                try:
-                    # Unpickle the data
-                    pickle_bytes = res.content
-                    fig = pickle.loads(pickle_bytes)
-                    fig = go.Figure(json.loads(fig))
-                    fig = set_dark_mode(fig)
-                    return fig
-                except Exception as e:
-                    logger.error(f"Failed to process plot data: {e}")
-                    return None
-            else:
-                logger.warning("Plotting is only available with the full installation. Please install 'sovai[full]' to use this feature.")
-                return None
-        
-        # Generate plot if requested
-        if plot:
-            verbose_mode.log("Generating plot from data")
-            if HAS_FULL_INSTALL and result_data is not None:
-                return _draw_graphs(result_data)
-            else:
-                logger.warning("Plotting is only available with the full installation. Please install 'sovai[full]' to use this feature.")
-                return None
+                # Create API request handler
+                api_handler = get_api_handler()
                 
-        return result_data
-        
-    except InvalidInputData as err:
-        logger.error(f"Invalid input data: {err}")
-        raise
-        
-    except (ConnectionError, TimeoutError) as err:
-        logger.error(f"Network error: {err}")
-        raise ConnectionError(f"Could not connect to API: {err}")
-        
-    except Exception as err:
-        logger.error(f"API request error: {err}")
-        raise
-            
-    return None
+                # Make API request with retry logic
+                verbose_mode.log("Sending API request")
+                res = api_handler.get(
+                    endpoint=endpoint,
+                    params=params,
+                    body=body,
+                    stream=True
+                )
+                
+                verbose_mode.log(f"Response received - Status: {res.status_code}, Content-Type: {res.headers.get('content-type')}")
+                
+                # Get appropriate ticker for endpoint
+                tickers = get_ticker_from_endpoint(endpoint, tickers, ENDPOINT_TO_TICKER)
+                verbose_mode.log(f"Using tickers: {tickers}")
+
+                # Get response metadata
+                data_format = res.headers.get("X-Data-Format")
+                content_type = res.headers["content-type"]
+                plot_header = res.headers.get("X-Plotly-Data")
+                
+                # Handle binary data (parquet)
+                if (content_type == "application/octet-stream") and not plot_header:
+                    if data_format == "pyarrow":
+                        verbose_mode.log("Processing pyarrow data stream")
+                        data_result = stream_data_pyarrow(res)
+                    else:
+                        verbose_mode.log("Processing binary data stream")
+                        data_result = stream_data(res)
+                    
+                    # Wrap in CustomDataFrame if available
+                    if HAS_FULL_INSTALL:
+                        data_result = CustomDataFrame(data_result)
+                    else:
+                        data_result = pd.DataFrame(data_result)
+                    
+                    _query_cache[cache_key] = data_result
+                    result_data = data_result
+
+                    
+                
+                # Handle 'all tickers' case
+                elif is_all(tickers):
+                    verbose_mode.log("Processing 'all tickers' response")
+                    urls = [u.strip() for u in res.text.strip('"').split(',') if u.strip()]
+                    data_result = None
+                    
+                    # Try each URL until successful
+                    for i, url in enumerate(urls):
+                        verbose_mode.log(f"Attempting URL {i+1}/{len(urls)}: {url}")
+                        try:
+                            data_result = read_parquet(url, use_polars=use_polars)
+                            verbose_mode.log(f"Successfully downloaded data from URL {i+1}")
+                            break
+                        except Exception as e:
+                            verbose_mode.log(f"Failed to download from URL {i+1}: {str(e)}")
+                    
+                    if data_result is None:
+                        error_msg = "Failed to download data from all provided URLs"
+                        logger.error(error_msg)
+                        raise Exception(error_msg)
+                    
+                    # Apply filters
+                    verbose_mode.log("Applying filters to data")
+                    data_result = filter_data(
+                        data_result, 
+                        columns=columns, 
+                        start_date=start_date, 
+                        end_date=end_date, 
+                        use_polars=use_polars
+                    )
+
+                    # Wrap in CustomDataFrame if available
+                    if HAS_FULL_INSTALL and not use_polars:
+                        data_result = CustomDataFrame(data_result)
+
+                    _query_cache[cache_key] = data_result
+                    result_data = data_result
+
+
+                    # result_data = process_dataframe(result_data)
+
+
+                    # Before returning, apply source filtering if applicable
+                    if (result_data is not None and 
+                        source is not None and 
+                        hasattr(result_data, 'filter') and 
+                        'source' not in result_data.columns):
+                        
+                        if source == "delisted":
+                            verbose_mode.log("Filtering for delisted securities")
+                            result_data = result_data.filter(["isdelisted=Y"])
+                        elif source == "listed":
+                            verbose_mode.log("Filtering for listed securities")
+                            result_data = result_data.filter(["isdelisted=N"])
+                    
+                # Handle JSON data
+                elif not plot_header:
+                    verbose_mode.log("Processing JSON response")
+                    json_data = res.json()
+                    if HAS_FULL_INSTALL:
+                        data_result = CustomDataFrame(convert_data2df(json_data))
+                    else:
+                        data_result = pd.DataFrame(convert_data2df(json_data))
+
+                    _query_cache[cache_key] = data_result
+                    result_data = data_result
+
+
+                # Handle plot data
+                if plot_header:
+                    verbose_mode.log("Processing plot data")
+                    if HAS_FULL_INSTALL:
+                        import pickle
+                        try:
+                            # Unpickle the data
+                            pickle_bytes = res.content
+                            fig = pickle.loads(pickle_bytes)
+                            fig = go.Figure(json.loads(fig))
+                            fig = set_dark_mode(fig)
+                            result_data = fig
+                        except Exception as e:
+                            logger.error(f"Failed to process plot data: {e}")
+                            result_data = None
+                    else:
+                        logger.warning("Plotting is only available with the full installation. Please install 'sovai[full]' to use this feature.")
+                        result_data = None
+                
+                # Generate plot if requested
+                if plot and result_data is not None and not plot_header:
+                    verbose_mode.log("Generating plot from data")
+                    if HAS_FULL_INSTALL:
+                        result_data = _draw_graphs(result_data)
+                    else:
+                        logger.warning("Plotting is only available with the full installation. Please install 'sovai[full]' to use this feature.")
+                        result_data = None
+                    
+            except InvalidInputData as err:
+                logger.error(f"Invalid input data: {err}")
+                raise
+                
+            except (ConnectionError, TimeoutError) as err:
+                logger.error(f"Network error: {err}")
+                raise ConnectionError(f"Could not connect to API: {err}")
+                
+            except Exception as err:
+                logger.error(f"API request error: {err}")
+                raise
+
+
+    result_data = process_dataframe(result_data)
+
+    return result_data
 
 
 async def fetch_multiple_tickers(

{sovai-0.2.40 → sovai-0.2.42}/sovai/utils/client_side_s3_part_high.py

@@ -585,7 +585,7 @@ ENDPOINT_CONFIG = {
         "date_path": "sovai/sovai-patents-bulk/grants/date",
         "has_year": True
     },
-    "clinical_trials": {
+    "clinical/trials": {
         "ticker_path": "sovai/sovai-clinical-trials-export/partitioned/ticker/ticker",
         "date_path": "sovai/sovai-clinical-trials-export/partitioned/date/date",
         "has_year": False
@@ -610,10 +610,9 @@ ENDPOINT_CONFIG = {
         "date_path": "sovai/sovai-government/partitioned/contract/date",
         "has_year": False
     },
-
     "spending/product": {
         "ticker_path": "sovai/sovai-government/partitioned/product/ticker",
-         "date_path": "sovai/sovai-government/partitioned/product/date",
+        "date_path": "sovai/sovai-government/partitioned/product/date",
         "has_year": False
     },
     "spending/transactions": {
@@ -631,9 +630,69 @@ ENDPOINT_CONFIG = {
     },
     "lobbying/data": {
         "ticker_path": "sovai/sovai-lobbying/partitioned/ticker",
-         "date_path": "sovai/sovai-lobbying/partitioned/date",
+        "date_path": "sovai/sovai-lobbying/partitioned/date",
+        "has_year": False
+    },
+    "accounting/weekly": {
+        "ticker_path": "sovai/sovai-accounting/partitioned/ticker",
+        "date_path": "sovai/sovai-accounting/partitioned/date",
+        "has_year": False
+    },
+    "ratios/normal": {
+        "ticker_path": "sovai/sovai-ratios/partitioned/ticker",
+        "date_path": "sovai/sovai-ratios/partitioned/date",
+        "has_year": False
+    },
+    "complaints/public": {
+        "ticker_path": "sovai/sovai-complaints/partitioned/ticker",
+        "date_path": "sovai/sovai-complaints/partitioned/date",
+        "has_year": False
+    },
+    "factors/accounting": {
+        "ticker_path": "sovai/sovai-factors/accounting/partitioned/ticker",
+        "date_path": "sovai/sovai-factors/accounting/partitioned/date",
+        "has_year": False
+    },
+    "factors/alternative": {
+        "ticker_path": "sovai/sovai-factors/alternative/partitioned/ticker",
+        "date_path": "sovai/sovai-factors/alternative/partitioned/date",
+        "has_year": False
+    },
+    "factors/comprehensive": {
+        "ticker_path": "sovai/sovai-factors/comprehensive/partitioned/ticker",
+        "date_path": "sovai/sovai-factors/comprehensive/partitioned/date",
+        "has_year": False
+    },
+    "factors/coefficients": {
+        "ticker_path": "sovai/sovai-factors/coefficients/partitioned/ticker",
+        "date_path": "sovai/sovai-factors/coefficients/partitioned/date",
+        "has_year": False
+    },
+    "factors/standard_errors": {
+        "ticker_path": "sovai/sovai-factors/standard_errors/partitioned/ticker",
+        "date_path": "sovai/sovai-factorss/standard_errors/partitioned/date",
+        "has_year": False
+    },
+    "factors/t_statistics": {
+        "ticker_path": "sovai/sovai-factors/t_statistics/partitioned/ticker",
+        "date_path": "sovai/sovai-factors/t_statistics/partitioned/date",
+        "has_year": False
+    },
+    "factors/model_metrics": {
+        "ticker_path": "sovai/sovai-factors/model_metrics/partitioned/ticker",
+        "date_path": "sovai/sovai-factors/model_metrics/partitioned/date",
         "has_year": False
     },
+    "breakout": {
+        "ticker_path": "sovai/sovai-breakout-price/partitioned/ticker",
+        "date_path": "sovai/sovai-breakout-price/partitioned/date",
+        "has_year": False
+    },
+    "visas/h1b": {
+        "ticker_path": "sovai/sovai-employment/partitioned/ticker",
+        "date_path": "sovai/sovai-employment/partitioned/date",
+        "has_year": False
+    }
 }