sovai 0.2.32__tar.gz → 0.2.34__tar.gz

{sovai-0.2.32 → sovai-0.2.34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sovai
-Version: 0.2.32
+Version: 0.2.34
 Summary: python for asset management
 License: Proprietary License
 Author: Derek Snow

{sovai-0.2.32 → sovai-0.2.34}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sovai"
-version = "0.2.32"
+version = "0.2.34"
 description = "python for asset management"
 authors = ["Derek Snow <d.snow@outlook.com>"]
 license = "Proprietary License"

{sovai-0.2.32 → sovai-0.2.34}/sovai/api_config.py

@@ -3,7 +3,7 @@ from typing import Optional, Union
 
 from dotenv import dotenv_values
 
-### save
+### save le chose a faire.
 
 class ApiConfig:
     """

{sovai-0.2.32 → sovai-0.2.34}/sovai/extensions/filter_df.py

@@ -1,211 +1,3 @@
-import pandas as pd
-import re
-import time
-from typing import Union, List, Tuple
-from functools import wraps
-from rapidfuzz import process, fuzz
-from sovai.tools.sec.llm_code_generator import get_openai_key
-from collections import deque
-from datetime import datetime, timedelta
-import openai
-from textwrap import dedent
-from functools import lru_cache
-
-def with_openai_key(func):
-    @wraps(func)
-    def wrapper(*args, **kwargs):
-        openai.api_key = get_openai_key()
-        try:
-            return func(*args, **kwargs)
-        finally:
-            openai.api_key = None
-    return wrapper
-
-class Filter:
-    _call_timestamps = deque()
-    _max_calls_per_minute = 10
-    _ask_cache = {}
-    verbose = False
-    mutable = False
-    models = ['gpt-3.5-turbo', 'gpt-4']
-    completion_config = {}
-
-
-    @staticmethod
-    @lru_cache(maxsize=1)
-    def _load_combined_df():
-        return pd.read_parquet("https://storage.googleapis.com/sovai-public/concats/filters/latest.parquet")
-
-
-    @classmethod
-    def _check_rate_limit(cls):
-        now = datetime.now()
-        while cls._call_timestamps and cls._call_timestamps[0] < now - timedelta(minutes=1):
-            cls._call_timestamps.popleft()
-        
-        if len(cls._call_timestamps) >= cls._max_calls_per_minute:
-            raise ValueError("Slow down mate! You've reached the maximum number of filtering attempts per minute.")
-        
-        cls._call_timestamps.append(now)
-
-    @staticmethod
-    def rate_limit_decorator(func):
-        @wraps(func)
-        def wrapper(*args, **kwargs):
-            Filter._check_rate_limit()
-            return func(*args, **kwargs)
-        return wrapper
-
-    @staticmethod
-    def _find_best_match(query: str, choices: List[str], threshold: int = 80) -> Union[str, None]:
-        best_match = process.extractOne(query, choices, scorer=fuzz.ratio, score_cutoff=threshold)
-        return best_match[0] if best_match else None
-
-    @staticmethod
-    def _parse_value(value: str) -> Union[float, int]:
-        multipliers = {'k': 1e3, 'm': 1e6, 'b': 1e9}
-        if value.endswith('%'):
-            return float(value.rstrip('%')) / 100
-        for suffix, multiplier in multipliers.items():
-            if value.lower().endswith(suffix):
-                return float(value[:-1]) * multiplier
-        return float(value)
-
-    @classmethod
-    def _create_standard_query(cls, condition: str, df: pd.DataFrame) -> Tuple[str, str]:
-        pattern = r'(\w+)\s*([<>=]+|like|contains)\s*(.+)'
-        match = re.match(pattern, condition, re.IGNORECASE)
-        if not match:
-            raise ValueError(f"Invalid condition format: {condition}")
-
-        column, operator, value = match.groups()
-        best_match = cls._find_best_match(column, df.columns.tolist())
-        if not best_match:
-            raise ValueError(f"No close match found for column: {column}")
-
-        original_column = column
-        column = best_match
-        
-        operator = operator.lower().strip()
-        value = value.strip()
-
-        is_string_type = df[column].dtype == 'object'
-
-        if operator in ['like', 'contains'] or (operator == '=' and is_string_type):
-            query = f"`{column}`.str.contains('{value}', case=False, na=False)"
-            display_operator = 'like'
-        else:
-            parsed_value = cls._parse_value(value)
-            query = f"`{column}` {operator} {parsed_value}"
-            display_operator = operator
-
-        display_condition = f"{column} ({original_column}) {display_operator} {value}" if column != original_column else condition
-        return query, display_condition
-    
-    @staticmethod
-    def _is_simple_condition(condition: str) -> bool:
-        simple_pattern = r'\w+\s*([<>=]+|like|contains)\s*.+'
-        return bool(re.match(simple_pattern, condition, re.IGNORECASE))
-
-    @staticmethod
-    def _split_conditions(condition: str) -> List[str]:
-        pattern = r'\s+(?:and|or)\s+(?![^()]*\))'
-        conditions = re.split(pattern, condition, flags=re.IGNORECASE)
-        return [cond.strip() for cond in conditions if cond.strip()]
-
-    @classmethod
-    @rate_limit_decorator
-    @with_openai_key
-    def _apply_condition_with_retry(cls, df: pd.DataFrame, condition: str, max_retries: int = 3, verbose: bool = False) -> pd.DataFrame:
-        original_len = len(df)
-        
-        sub_conditions = cls._split_conditions(condition)
-        if len(sub_conditions) > 1:
-            if verbose:
-                print(f"Detected multiple conditions: {sub_conditions}")
-            for sub_condition in sub_conditions:
-                df = cls._apply_condition_with_retry(df, sub_condition, max_retries, verbose)
-            return df
-        
-        if cls._is_simple_condition(condition):
-            try:
-                query, display_condition = cls._create_standard_query(condition, df)
-                filtered_df = df.query(query)
-                if len(filtered_df) < original_len:
-                    if verbose:
-                        print(f"Applied standard filter: {display_condition}")
-                    return filtered_df
-            except Exception as e:
-                if verbose:
-                    print(f"Standard filtering failed: {str(e)}. Falling back to GPT method.")
-        
-        for attempt in range(max_retries):
-            try:
-                filtered_df = df.ask(condition)
-                if isinstance(filtered_df, pd.DataFrame) and len(filtered_df) < original_len:
-                    return filtered_df
-                elif verbose:
-                    print(f"Attempt {attempt + 1}: GPT filtering didn't remove any rows. Retrying...")
-                
-                alternative_conditions = [f"filter {condition}", f"select {condition}", f"find {condition}"]
-                for alt_condition in alternative_conditions:
-                    try:
-                        filtered_df = df.ask(alt_condition)
-                        if isinstance(filtered_df, pd.DataFrame) and len(filtered_df) < original_len:
-                            return filtered_df
-                    except Exception:
-                        pass
-                
-                columns_str = ", ".join(df.columns)
-                general_condition = f"Based on the columns {columns_str}, {condition}"
-                filtered_df = df.ask(general_condition)
-                if isinstance(filtered_df, pd.DataFrame) and len(filtered_df) < original_len:
-                    return filtered_df
-            except Exception as e:
-                if verbose:
-                    print(f"Error on GPT attempt {attempt + 1}: {str(e)}")
-            time.sleep(1)
-        
-        if verbose:
-            print("All filtering attempts failed. Returning original DataFrame.")
-        return df
-
-
-    @staticmethod
-    def create_table(data, headers):
-        col_widths = [max(len(str(x)) for x in col) for col in zip(*data, headers)]
-        col_widths = [max(15, width) for width in col_widths]
-        
-        row_format = '│ {:<{}} │ {:>{}} │ {:>{}} │ {:>{}} │'
-        header = row_format.format(headers[0], col_widths[0], 
-                                   headers[1], col_widths[1], 
-                                   headers[2], col_widths[2], 
-                                   headers[3], col_widths[3])
-        
-        separator = '┼' + '─' * (col_widths[0] + 2) + '┼' + '─' * (col_widths[1] + 2) + '┼' + '─' * (col_widths[2] + 2) + '┼' + '─' * (col_widths[3] + 2) + '┼'
-        top_border = '┌' + '─' * (col_widths[0] + 2) + '┬' + '─' * (col_widths[1] + 2) + '┬' + '─' * (col_widths[2] + 2) + '┬' + '─' * (col_widths[3] + 2) + '┐'
-        bottom_border = '└' + '─' * (col_widths[0] + 2) + '┴' + '─' * (col_widths[1] + 2) + '┴' + '─' * (col_widths[2] + 2) + '┴' + '─' * (col_widths[3] + 2) + '┘'
-        
-        rows = []
-        for i, row in enumerate(data):
-            if i == 0:  # Initial row
-                formatted_row = row_format.format(row[0], col_widths[0],
-                                                  f"{row[1]:,}", col_widths[1],
-                                                  '-', col_widths[2],
-                                                  '-', col_widths[3])
-            elif i == len(data) - 1:  # Final row
-                formatted_row = row_format.format(row[0], col_widths[0],
-                                                  f"{data[0][1]:,}", col_widths[1],
-                                                  f"{row[2]:,}", col_widths[2],
-                                                  f"{row[1]:,}", col_widths[3])
-            else:  # Other rows
-                formatted_row = row_format.format(row[0], col_widths[0],
-                                                  f"{row[1]:,}", col_widths[1],
-                                                  f"{row[2]:,}", col_widths[2],
-                                                  f"{row[1]:,}", col_widths[3])
-            rows.append(formatted_row)
-        
-        return '\n'.join([top_border, header, separator] + rows[:-1] + [separator, rows[-1], bottom_border])
 
 import pandas as pd
 import re
@@ -243,7 +35,7 @@ class Filter:
     @staticmethod
     @lru_cache(maxsize=1)
     def _load_combined_df():
-        return pd.read_parquet("https://storage.googleapis.com/sovai-public/combined/latest_values.parquet")
+        return pd.read_parquet("https://storage.googleapis.com/sovai-public/concats/filters/latest.parquet")
 
 
     @classmethod

{sovai-0.2.32 → sovai-0.2.34}/sovai/get_compute.py

@@ -1,4 +1,4 @@
-# Inside plot/get_plot.py
+# Inside plot/get_plot.py ##
 from .computations.functions import *
 
 COMPUTE_FUNCTION_MAPPER = {

{sovai-0.2.32 → sovai-0.2.34}/sovai/get_data.py

@@ -10,7 +10,7 @@ import pyarrow.parquet as pq
 from io import BytesIO
 import boto3
 import polars as pl
-
+import plotly.graph_objects as go
 # from fastapi import HTTPException ## Dee
 
 import pyarrow.parquet as pq
@@ -243,7 +243,7 @@ endpoint_to_ticker = {
     "/spending/competition": "",
     "/spending/entities": "",
     "/accounting/weekly": "",
-    "/visa/h1b": "",
+    "/visas/h1b": "",
     "/factors/accounting": "",
     "/factors/alternative": "",
     "/factors/comprehensive": "",
@@ -257,6 +257,7 @@ endpoint_to_ticker = {
     "/complaints/private": "",
     "/complaints/public": "",
     "/short/over_shorted": "",
+    "/short/volume": "",
     "/earnings/surprise": "",
     "/news/sentiment_score": "",
     "/news/topic_probability": "",
@@ -264,8 +265,15 @@ endpoint_to_ticker = {
     "/macro/features": "",
     "/congress": "",
     "/market/closeadj": "",
+    "/lobbying/public": "",
 
 
+    ## remember to add leading "/"
+    "/trials/predict": "",
+    "/trials/describe": "",
+    "/trials/all/predict": "",
+    "/trials/all/decribe": "",
+    "/trials/all": ""
 
 
 }
@@ -282,6 +290,8 @@ def normalize_endpoint(endpoint):
     return endpoint.strip("/").strip()
 
 
+## If you also want access by full file, please also add to endpoint_to_ticker
+
 client_side_endpoints_gcs = {
     "ratios/relative",
     "market/prices",
@@ -292,10 +302,19 @@ client_side_endpoints_gcs = {
     "lobbying/public",
 }
 
+## If you also want access by full file, please also add to endpoint_to_ticker
 client_side_endpoints_s3 = {
-    "sec/10k"
+    "sec/10k",
+    "trials/predict",
+    "trials/describe",
+    "trials/all/predict",
+    "trials/all/decribe",
+    "trials/all",
+
 }
 
+
+
 # In your main function or wherever this logic is implemented:
 
 
@@ -594,7 +613,7 @@ def data(
 
     elif (
         normalized_endpoint in client_side_endpoints_s3
-        # and tickers is not None
+        and tickers is not None
         and frequency is None
     ):
         verbose_mode.vprint("Grabbing s3 client side")
@@ -605,6 +624,7 @@ def data(
         if tickers is not None and not is_all(tickers):
             params = ticker_mapper(params,verbose)
 
+
         res = requests.get(
             url=url,
             headers=headers,
@@ -613,7 +633,7 @@ def data(
             stream=True,
             verify=ApiConfig.verify_ssl,
         )
-        res.raise_for_status()
+        
         verbose_mode.vprint(f"Response Status: {res.status_code}")
         verbose_mode.vprint(f"Response Content-Type: {res.headers.get('content-type')}")
 
@@ -624,12 +644,20 @@ def data(
         tickers = get_ticker_from_endpoint(endpoint, tickers, endpoint_to_ticker)
 
         print_tickers_value(tickers)
+  
+
+        res.raise_for_status()
 
         data_format = res.headers.get("X-Data-Format")
         content_type = res.headers["content-type"]
+
+        plot_header = res.headers.get("X-Plotly-Data")
+        # print(data_format)
+        # print(plot_header)
+
         # print(content_type)
 
-        if content_type == "application/octet-stream":
+        if (content_type == "application/octet-stream") and not plot_header:
             if data_format == "pyarrow":
                 verbose_mode.vprint(f"header: {data_format}")
                 data = stream_data_pyarrow(res)
@@ -670,25 +698,54 @@ def data(
                 data = CustomDataFrame(data)
 
             verbose_mode.vprint("It reached the DF")
-        else:
+
+            _query_cache[cache_key] = data
+
+            verbose_mode.vprint("It passed the DF")
+
+        elif not plot_header:
             if HAS_FULL_INSTALL:
                 data = CustomDataFrame(convert_data2df(res.json()))
             else:
                 data = pd.DataFrame(convert_data2df(res.json()))
 
-        _query_cache[cache_key] = data
+            _query_cache[cache_key] = data
+
+            verbose_mode.vprint("It passed the DF")
 
-        verbose_mode.vprint("It passed the DF")
+        import pickle
+        from plotly.graph_objs import Figure
+        def set_dark_mode(fig: Figure):
+            return fig.update_layout(
+                template="plotly_dark",
+                plot_bgcolor="rgba(10, 10, 10, 1)",
+                paper_bgcolor="rgba(10, 10, 10, 1)",
+            )
 
+        if plot_header:
+            # print(res.content[:100])
+            if HAS_FULL_INSTALL:
+                # Decompress the data
+                pickle_bytes = res.content
+                # Unpickle the data
+                fig = pickle.loads(pickle_bytes)
+                fig = go.Figure(json.loads(fig))
+                fig = set_dark_mode(fig)
+
+                return fig
+            else:
+                print("Plotting is only available with the full installation. Please install 'sovai[full]' to use this feature.")
+            return None
+        
         if plot:
             if HAS_FULL_INSTALL:
-                _draw_graphs(data)
+                return _draw_graphs(data)
             else:
                 print("Plotting is only available with the full installation. Please install 'sovai[full]' to use this feature.")
             return None
         return data
     except Exception as err:
-        verbose_mode.vprint("An error occurred:", err)
+        verbose_mode.vprint("An error occurred, check dictaionries for 'data/name':'None':", err)
         return data
         if res.status_code == 404:
             msg = res.json()
@@ -747,9 +804,7 @@ def _draw_graphs(data: Union[Dict, List[Dict]]):
     if isinstance(data, list):
         for plot in data:
             for _, val in plot.items():
-                plotting_data(val)
-                break
+                return plotting_data(val)
     else:
         for _, val in data.items():
-            plotting_data(val)
-            break
+            return plotting_data(val)

{sovai-0.2.32 → sovai-0.2.34}/sovai/get_plots.py

@@ -44,12 +44,12 @@ def _draw_graphs(data: Union[Dict, List[Dict]]):
     if isinstance(data, list):
         for plot in data:
             for _, val in plot.items():
-                plotting_data(val)
+                return plotting_data(val)
                 break
     else:
         for _, val in data.items():
-            plotting_data(val)
-            break
+            return plotting_data(val)
+            
 
 
 from typing import Optional, Union, Tuple, List, Dict

{sovai-0.2.32 → sovai-0.2.34}/sovai/plots/earnings_surprise/earnings_surprise_plots.py

@@ -127,6 +127,7 @@ def earnings_tree():
     if 'ticker' not in df_earn.columns:
         df_earn = df_earn.reset_index()
     
+    df_earn['market_cap'] = df_earn['market_cap'].replace(0, 1e-6)
     # Calculate market cap deciles and handle NaN values
     df_earn['market_cap_decile'] = pd.qcut(df_earn['market_cap'].rank(method='first'), q=10, labels=range(1, 11))
     df_earn['market_cap_decile'] = df_earn['market_cap_decile'].astype(str).replace('nan', 'Unknown')
@@ -134,6 +135,7 @@ def earnings_tree():
     # Update decile labels
     df_earn['market_cap_decile'] = df_earn['market_cap_decile'].apply(lambda x: f"{x} Cap Decile" if x != 'Unknown' else x)
     
+    
     # Create the main treemap
     fig = px.treemap(
         df_earn,
@@ -183,4 +185,6 @@ def earnings_tree():
     )
     
     # Show the plot
-    fig.show()
+    fig
+
+    return fig

{sovai-0.2.32 → sovai-0.2.34}/sovai/utils/client_side_s3.py

@@ -63,6 +63,9 @@ def process_partition(storage_provider, base_path, identifier_column, identifier
     
     if identifier:
         df[identifier_column] = identifier
+
+        df = df[[identifier_column] + [col for col in df.columns if col != identifier_column]]
+
     
     return df
 
@@ -165,6 +168,8 @@ def map_identifier(input_values, input_identifier='ticker', output_identifier='c
     else:
         return result.tolist()
 
+
+
 def load_frame_s3(endpoint, tickers=None, columns=None, start_date=None, end_date=None):
 
     endpoint_config = {
@@ -177,12 +182,44 @@ def load_frame_s3(endpoint, tickers=None, columns=None, start_date=None, end_dat
             "partitioned": True,
     
         },
-        "future/enpdoint": {
+        "trials/predict": {
             "storage_provider": {
-                "digitalocean": "sovai/sovai-filings/tenk",
-                "wasabi": "sovai-filings/tenk"
+                "digitalocean": "sovai/sovai-clinical-trials-export/prediction_public",
+                "wasabi": "sovai-clinical-trials-export/prediction_public"
             },
-            "identifier_column": "cik",
+            "identifier_column": "ticker",
+            "partitioned": True,
+        },
+        "trials/describe": {
+            "storage_provider": {
+                "digitalocean": "sovai/sovai-clinical-trials-export/describe_public",
+                "wasabi": "sovai-clinical-trials-export/describe_public"
+            },
+            "identifier_column": "ticker",
+            "partitioned": True,
+        },
+        "trials/all": {
+            "storage_provider": {
+                "digitalocean": "sovai/sovai-clinical-trials-export/describe_all",
+                "wasabi": "sovai-clinical-trials-export/describe_all"
+            },
+            "identifier_column": "ticker",
+            "partitioned": True,
+        },
+        "trials/all/predict": {
+            "storage_provider": {
+                "digitalocean": "sovai/sovai-clinical-trials-export/prediction_all",
+                "wasabi": "sovai-clinical-trials-export/prediction_all"
+            },
+            "identifier_column": "ticker",
+            "partitioned": True,
+        },
+        "trials/all/decribe": {
+            "storage_provider": {
+                "digitalocean": "sovai/sovai-clinical-trials-export/describe_all",
+                "wasabi": "sovai-clinical-trials-export/describe_all"
+            },
+            "identifier_column": "ticker",
             "partitioned": True,
         }
     }
@@ -193,17 +230,15 @@ def load_frame_s3(endpoint, tickers=None, columns=None, start_date=None, end_dat
     config = endpoint_config[endpoint]
 
     
-    if endpoint == "sec/10k":
-        df_frame = client_side_s3_frame(
-            config,
-            tickers,
-            columns,
-            start_date,
-            end_date
-        )
-    else:
-        # Implement logic for other endpoints if needed
-        raise NotImplementedError(f"Endpoint {endpoint} is not yet implemented")
+    
+    df_frame = client_side_s3_frame(
+        config,
+        tickers,
+        columns,
+        start_date,
+        end_date
+    )
+
     
     if HAS_CUSTOM_DATAFRAME:
         return CustomDataFrame(df_frame)

{sovai-0.2.32 → sovai-0.2.34}/sovai/utils/plot.py

@@ -52,3 +52,4 @@ def plotting_data(
         elif chart_type == "scatter":
             fig = px.scatter(df, x=x, y=y, **kwargs)
             fig.show()
+    return fig