strideutils 0.4.5__tar.gz → 0.5.0__tar.gz

{strideutils-0.4.5 → strideutils-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: strideutils
-Version: 0.4.5
+Version: 0.5.0
 Summary: 
 Author: Your Name
 Author-email: you@example.com
@@ -18,6 +18,7 @@ Requires-Dist: gspread (>=6.0.2)
 Requires-Dist: pandas (>=2.1.1)
 Requires-Dist: pycryptodome (>=3.20.0)
 Requires-Dist: python-dateutil (>=2.8.2)
+Requires-Dist: python-dotenv (==1.0.1)
 Requires-Dist: pytz (>=2023.3)
 Requires-Dist: pyyaml (>=6.0.1)
 Requires-Dist: redis (>=4.5.4)
@@ -49,12 +50,11 @@ Strideutils requires three different environment variables that can be added to
 
 ```
 export STRIDEUTILS_CONFIG_PATH=
-export STRIDEUTILS_SECRETS_PATH=
-export ENV=DEV
+export STRIDEUTILS_ENV_PATH=
 ```
 
 Examples of these files are included under strideutils/config_examples.
-Stride Labs employees can find config.yaml in launchpad and strideutils_secrets.yamls in lastpass.
+Stride Labs employees can find config.yaml in launchpad and .env.local in lastpass. We recommend you place .env.local in your launchpad repo.
 
 Any configuration or secrets that aren't consumed don't need to be set. However if one is accessed but unset, an error will be thrown for easier debugging.
 
@@ -77,6 +77,14 @@ import sys
 sys.path = ['/path/to/strideutils/'] + sys.path
 ```
 
+Alternatively, you can install strideutils through local mode. However, please be warned that this will make your local Python environment _always_ use your local strideutils. This might lead to unexpected behavior if you local changes.
+
+To install in local mode, run the following:
+
+```
+pip3 install -e /Users/username/Documents/strideutils
+```
+
 Confirm the location of where it's being imported from by printing the module.
 After making changes to strideutils, reload it before testing your application.
 

{strideutils-0.4.5 → strideutils-0.5.0}/README.md

@@ -20,12 +20,11 @@ Strideutils requires three different environment variables that can be added to
 
 ```
 export STRIDEUTILS_CONFIG_PATH=
-export STRIDEUTILS_SECRETS_PATH=
-export ENV=DEV
+export STRIDEUTILS_ENV_PATH=
 ```
 
 Examples of these files are included under strideutils/config_examples.
-Stride Labs employees can find config.yaml in launchpad and strideutils_secrets.yamls in lastpass.
+Stride Labs employees can find config.yaml in launchpad and .env.local in lastpass. We recommend you place .env.local in your launchpad repo.
 
 Any configuration or secrets that aren't consumed don't need to be set. However if one is accessed but unset, an error will be thrown for easier debugging.
 
@@ -48,6 +47,14 @@ import sys
 sys.path = ['/path/to/strideutils/'] + sys.path
 ```
 
+Alternatively, you can install strideutils through local mode. However, please be warned that this will make your local Python environment _always_ use your local strideutils. This might lead to unexpected behavior if you local changes.
+
+To install in local mode, run the following:
+
+```
+pip3 install -e /Users/username/Documents/strideutils
+```
+
 Confirm the location of where it's being imported from by printing the module.
 After making changes to strideutils, reload it before testing your application.
 

{strideutils-0.4.5 → strideutils-0.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "strideutils"
-version = "0.4.5"
+version = "0.5.0"
 description = ""
 authors = ["Your Name <you@example.com>"]
 readme = "README.md"
@@ -21,6 +21,7 @@ slack-sdk = ">=3.23.0"
 pandas = ">=2.1.1"
 twilio = ">=8.10.0"
 tabulate = ">=0.9.0"
+python-dotenv = "==1.0.1"
 pyyaml = ">=6.0.1"
 python-dateutil = ">=2.8.2"
 pycryptodome = ">=3.20.0"

{strideutils-0.4.5 → strideutils-0.5.0}/strideutils/coingecko.py

@@ -79,11 +79,8 @@ def get_token_price_history(token: str, num_days=90, use_key=False, memo={}) ->
     price_key = (token, num_days)
     if price_key not in memo:
         tstr = get_coingecko_name(config.get_chain(ticker=token))
-        rurl = (
-            f"https://api.coingecko.com/api/v3/coins/{tstr}/market_chart?vs_currency=usd&days={num_days}"  # noqa:E231
-        )
+        rurl = f"{COINGECKO_ENDPOINT}api/v3/coins/{tstr}/market_chart?vs_currency=usd&days={num_days}"  # noqa:E231
         rel_headers = {"x-cg-pro-api-key": config.COINGECKO_API_TOKEN} if use_key else None
-
         r = requests.get(
             rurl,
             headers=rel_headers,
@@ -98,7 +95,7 @@ def get_token_price_history(token: str, num_days=90, use_key=False, memo={}) ->
     return memo[token]
 
 
-def get_token_volume_history(token: str, num_days=90, use_key=False, memo={}) -> pd.Series:
+def get_token_volume_history(token: str, num_days=90, use_key=True, memo={}) -> pd.Series:
     """
     returns a Pandas Series with token volume data going back num_days
     e.g. get_token_volume_history('uatom', 30) returns a Series with 30 days of ATOM volume data
@@ -107,11 +104,8 @@ def get_token_volume_history(token: str, num_days=90, use_key=False, memo={}) ->
     volume_key = (token, num_days)
     if volume_key not in memo:
         tstr = get_coingecko_name(config.get_chain(ticker=token))
-        rurl = (
-            f"https://api.coingecko.com/api/v3/coins/{tstr}/market_chart?vs_currency=usd&days={num_days}"  # noqa:E231
-        )
+        rurl = f"{COINGECKO_ENDPOINT}api/v3/coins/{tstr}/market_chart?vs_currency=usd&days={num_days}"  # noqa:E231
         rel_headers = {"x-cg-pro-api-key": config.COINGECKO_API_TOKEN} if use_key else None
-
         r = requests.get(
             rurl,
             headers=rel_headers,
@@ -126,7 +120,7 @@ def get_token_volume_history(token: str, num_days=90, use_key=False, memo={}) ->
     return memo[token]
 
 
-def get_dataframe_of_prices(token_list: List[str], num_days: int = 90) -> pd.DataFrame:
+def get_dataframe_of_prices(token_list: List[str], num_days: int = 90, use_key: bool = True) -> pd.DataFrame:
     """
     returns a Pandas DataFrame with token price data going back num_days, for the given tokens
     e.g. get_dataframe_of_prices(["ATOM", "OSMO", "STRD"], 30) returns a DataFrame of 30 days of price data
@@ -134,7 +128,7 @@ def get_dataframe_of_prices(token_list: List[str], num_days: int = 90) -> pd.Dat
     # gather the dataframe
     df = {}
     for token in token_list:
-        df[token] = get_token_price_history(token, num_days)["price"]
+        df[token] = get_token_price_history(token, num_days, use_key=use_key)["price"]
     df = pd.DataFrame(df)
     # resample based on how much data we're gathering
     if num_days > 90:

strideutils-0.5.0/strideutils/config_examples/env.local.example

@@ -0,0 +1,24 @@
+ENV=DEV
+CONFIGURE_STRIDEUTILS_LOGS=true
+
+# Slack
+SLACK_CHANNELS={"test": "U33D4351C2V"}
+
+# API Tokens
+NUMIA_API_TOKEN=token
+COINGECKO_API_TOKEN=cgtoken
+
+# Upstash
+UPSTASH_PUBLIC_PASSWORD=password
+
+# Twilio
+TWILIO_ALERTS_NUMBER=+11111111111
+
+# Slashbot twitter
+TWITTER_CONSUMER_KEY=consumerkey
+
+# Epoch Summary
+EPOCH_SUMMARY_SLACK_BEARER_TOKEN=token
+
+# Google Sheets
+PUBLICSHEETS_AUTH={}

{strideutils-0.4.5 → strideutils-0.5.0}/strideutils/exec_tx.py

@@ -18,14 +18,16 @@ class TxResponse:
     raw_log: Optional[str] = None
 
 
-def execute_tx(command: str, api_endpoint: str = config.stride.api_endpoint) -> TxResponse:
+def execute_tx(command: str, api_endpoint: str = config.stride.api_endpoint, verbose=False) -> TxResponse:
     """
     Execute a tx and query the tx hash
     """
     output = subprocess.run(command, shell=True, capture_output=True)
     standard_out = re.sub(r'\x1b\[[0-9;]*[mGKH]', '', output.stdout.decode('utf-8'))
     error_out = re.sub(r'\x1b\[[0-9;]*[mGKH]', '', output.stderr.decode('utf-8'))
-
+    if verbose:
+        print("Standard Out: ", standard_out)
+        print("Error Out: ", error_out)
     tx_hash = ""
     for output_line in standard_out.split('\n'):
         if 'txhash' in output_line:

{strideutils-0.4.5 → strideutils-0.5.0}/strideutils/slack_connector.py

@@ -1,4 +1,5 @@
 import os
+import traceback
 from typing import List, Optional, Union
 
 from slack_sdk import WebClient
@@ -51,3 +52,23 @@ def post_msg(
             print("Error sending message: ", e)
             raise
     return thread_ts
+
+
+def upload_file(file_name: str, content: str) -> str:
+    '''
+    This uploads a file called "file_name" with the string contents "content"
+
+    This will return a link that can be used to embed a file in slack.
+    For example:
+        url = upload_file("test.txt", "Hello World")
+        post_msg(f"<{url}|This is a file>", channel="#alerts-debug")
+    '''
+    client = WebClient(token=config.STRIDEBOT_API_TOKEN)
+    try:
+        slack_file = client.files_upload_v2(filename=file_name, content=content)
+    except Exception as e:
+        print(f"Error uploading file to slack: {e}")
+        traceback.print_exc()
+        raise ValueError(f"Error uploading file to slack: {e}")
+    file_link = slack_file['file']['permalink']
+    return file_link

{strideutils-0.4.5 → strideutils-0.5.0}/strideutils/stride_config.py

@@ -8,6 +8,7 @@ from dataclasses import dataclass, field, fields
 from typing import Any, List, Optional, Union
 
 import yaml
+from dotenv import load_dotenv
 
 MISSING_CONFIG_MSG = "Configuration or environment variable '{x}' is not set or unknown"
 
@@ -191,6 +192,8 @@ class Config(ConfigObj):
     UPSTASH_STRIDE_FRONTEND_PASSWORD: str = ""
     UPSTASH_STRIDE_BACKEND_PASSWORD: str = ""
     UPSTASH_STRIDE_DYDX_PUBLIC_PASSWORD: str = ""
+    UPSTASH_STRIDE_DYDX_AIRDROP_PASSWORD: str = ""
+    UPSTASH_STRIDE_MILESTONES_PASSWORD: str = ""
 
     # Stride internal secrets
     SLACK_BEARER_TOKEN: str = ""
@@ -317,7 +320,6 @@ def _load_raw_config() -> dict[str, Any]:
             'Shell var STRIDEUTILS_CONFIG_PATH is missing or the file does not exist in the default location.'
             'The default file and location is config/config.yaml in the current working directory'
         )
-
     with open(strideutils_config_path, 'r') as config_file:
         raw_config = yaml.load(config_file, Loader)
 
@@ -327,20 +329,13 @@ def _load_raw_config() -> dict[str, Any]:
 def _load_raw_secrets() -> dict[str, Optional[str]]:
     """
     Loads the secret variables into a dictionary
-    If this is running locally in dev mode, secrets are grabbed from the path defined by STRIDEUTILS_SECRETS_PATH
+    If this is running locally in dev mode, secrets are grabbed from the path defined by STRIDEUTILS_ENV_PATH
     Otherwise, their loaded as environment variables
     """
-    if ENV == "PROD":
-        return {secret.name: os.environ.get(secret.name) for secret in fields(Config) if os.environ.get(secret.name)}
-
-    if not (strideutils_secrets_path := os.environ.get('STRIDEUTILS_SECRETS_PATH')):
-        raise ValueError(MISSING_CONFIG_MSG.format(x='STRIDEUTILS_SECRETS_PATH'))
-
-    with open(strideutils_secrets_path, 'r') as f:
-        raw_secrets = yaml.load(f, Loader)
-        raw_secrets.update({'ENV': ENV})
-
-        return raw_secrets
+    strideutils_env_path = os.environ.get('STRIDEUTILS_ENV_PATH', '.env.local')
+    if os.path.exists(strideutils_env_path):
+        load_dotenv(strideutils_env_path)
+    return {secret.name: os.environ.get(secret.name) for secret in fields(Config) if os.environ.get(secret.name)}
 
 
 def _parse_raw_configs(raw_config: dict[str, Any], raw_secrets: dict[str, Optional[str]]) -> Config:

{strideutils-0.4.5 → strideutils-0.5.0}/strideutils/stride_requests.py

@@ -973,18 +973,20 @@ def get_delegations(delegator_address: str, api_endpoint: str = config.stride.ap
     return delegations_by_validator
 
 
-def get_host_zone_delegations(host_zone_id: str, include_ground_truth: bool = True) -> pd.DataFrame:
+def _parse_validator_dict(raw_validators: List[dict[str, Any]]) -> pd.DataFrame:
     """
-    Given a host zone id, returns a Dataframe of validator delegations.
+    Given the output of a host_zone's "validator" field,
+    this will parse the data into a Pandas DataFrame.
 
-    If `include_ground_truth=True`, then will also return the "true" delegations
-    on the host zone delegaiton account.
+    Will also cast the relevant columns to ints and floats.
     """
-    int_cols = [
+    int_fields = [
         "weight",
         "delegation",
-        "slash_query_progress_tracker",
         "slash_query_checkpoint",
+        "slash_query_progress_tracker",
+    ]
+    float_cols = [
         "shares_to_tokens_rate",
     ]
     output_cols = [
@@ -997,12 +999,26 @@ def get_host_zone_delegations(host_zone_id: str, include_ground_truth: bool = Tr
         "shares_to_tokens_rate",
     ]
 
-    # Query the validator delegations on the host zone
-    host_zone = get_host_zone(host_zone_id)
-    validator_df = pd.DataFrame(host_zone["validators"])
-    validator_df[int_cols] = validator_df[int_cols].map(lambda i: int(float(i)))  # type: ignore
-    validator_df = validator_df[output_cols]
+    def parse_validator(field_name: str, field_value: Any) -> Any:
+        if field_name in int_fields:
+            return int(field_value)
+        if field_name in float_cols:
+            return float(field_value)
+        return field_value
+
+    cast_validators = [{k: parse_validator(k, v) for k, v in val_dict.items()} for val_dict in raw_validators]
+    return pd.DataFrame(cast_validators, dtype=object)[output_cols]
 
+
+def get_host_zone_delegations(host_zone_id: str, include_ground_truth: bool = True) -> pd.DataFrame:
+    """
+    Given a host zone id, returns a Dataframe of validator delegations.
+
+    If `include_ground_truth=True`, then will also return the "true" delegations
+    on the host zone delegaiton account.
+    """
+    host_zone = get_host_zone(host_zone_id)
+    validator_df = _parse_validator_dict(host_zone["validators"])
     # If the ground truth isn't requested, return as is
     if not include_ground_truth:
         return validator_df

{strideutils-0.4.5 → strideutils-0.5.0}/strideutils/stride_upstash.py

@@ -1,4 +1,4 @@
-from typing import Dict, Optional
+from typing import Dict, List, Optional
 
 import redis
 
@@ -43,7 +43,20 @@ def get_redis_db(redis_db_name: str) -> redis.Redis:
                 password=config.UPSTASH_STRIDE_DYDX_PUBLIC_PASSWORD,
                 ssl=True,
             )
-
+        elif redis_db_name == 'dydx_airdrop':
+            r = redis.Redis(
+                host='pleasant-pipefish-54904.upstash.io',
+                port=6379,
+                password=config.UPSTASH_STRIDE_DYDX_AIRDROP_PASSWORD,
+                ssl=True,
+            )
+        elif redis_db_name == "milestones":
+            r = redis.Redis(
+                host='adapting-snail-56076.upstash.io',
+                port=6379,
+                password=config.UPSTASH_STRIDE_MILESTONES_PASSWORD,
+                ssl=True,
+            )
         else:
             raise ValueError(f'Invalid Redis DB: {redis_db_name}')
 
@@ -64,6 +77,35 @@ def get(redis_key: str, db_name='frontend') -> Optional[str]:
     return value.decode('utf-8') if value is not None else None
 
 
+def get_multiple_keys(redis_keys: List[str], db_name='frontend') -> List[Optional[str]]:
+    """
+    This function will read multiple keys in at once.
+    Value will be "None" if the key does not exist.
+
+    Pulls from the specified redis db.
+    """
+    db = get_redis_db(db_name)
+    values = db.mget(redis_keys)
+
+    return [value.decode('utf-8') if value is not None else None for value in values]
+
+
+def get_all_keys(db_name='dydx_airdrop') -> List[str]:
+    """
+    This function will return all keys in the redis db specified.
+    """
+    db = get_redis_db(db_name)
+
+    out = []
+
+    cursor = '0'
+    while cursor != 0:
+        cursor, keys = db.scan(cursor, count=1000)
+        if keys:
+            out.extend(keys)
+    return [key.decode('utf-8') for key in out]
+
+
 def set(redis_key: str, redis_val: str, db_name='frontend'):
     """
     This function will set the given key to value in the redis_db specified.
@@ -79,5 +121,7 @@ def set_keys(dict_to_upload: Dict[str, str], db_name='frontend', prefix=''):
     Will append "prefix" to all keys in the dict.
     """
     db = get_redis_db(db_name)
+    pipe = db.pipeline()
     for k, v in dict_to_upload.items():
-        db.set(prefix + k, v)
+        pipe.set(prefix + k, v)
+    pipe.execute()

strideutils-0.4.5/strideutils/config_examples/strideutils_secrets.yaml.example

@@ -1,28 +0,0 @@
-# Twilio
-TWILIO_ALERTS_NUMBER: +18663326515
-TWILIO_ACCOUNT_ID:
-TWILIO_API_TOKEN:
-PHONE_NUMBERS: {}
-
-# Slack
-STRIDEBOT_API_TOKEN:
-
-# GSHEETS
-PUBLICSHEETS_AUTH:
-
-# Coingecko
-COINGECKO_API_TOKEN:
-
-# Stride Internal secrets
-SLACK_BEARER_TOKEN:
-SLACK_SUCCESS_CHANNEL_ID:
-SLACK_INVARIANT_FAILURE_CHANNEL_ID:
-SLACK_PACKET_FAILURE_CHANNEL_ID:
-NUMIA_API_TOKEN:
-UPSTASH_PUBLIC_PASSWORD:
-UPSTASH_STRIDE_FRONTEND_PASSWORD:
-UPSTASH_STRIDE_BACKEND_PASSWORD:
-
-# AWS
-AWS_ACCESS_KEY_ID:
-AWS_SECRET_ACCESS_KEY: