abstract-solana 0.0.2.81__py3-none-any.whl → 0.0.2.83__py3-none-any.whl

abstract_solana/abstract_rpcs/__init__.py

@@ -1,4 +1,4 @@
 from .get_body import *
 from .db_templates import *
-from .solana_rpc_client import get_rpc_dict,abstract_solana_rate_limited_call
+from .solana_rpc_client import get_rpc_dict,abstract_solana_rate_limited_call,make_call
 from .rate_limiter import *

abstract_solana/abstract_rpcs/rate_limiter.py

@@ -1,6 +1,10 @@
-import time,os,json
+import time
+import os
+import json
+from datetime import datetime
+
 from abstract_utilities import *
-from abstract_security import *
+from abstract_security import get_env_value
 def getAbsFile():
   return os.path.abspath(__file__)
 def getAbsDir():
@@ -9,12 +13,12 @@ def getAbsPath(path):
   return os.path.join(getAbsDir(),path)
 def getSaveStatePath():
   return getAbsPath('rate_limiter_state.json')
-def readSaveState():
-  path= getSaveStatePath()
+def readSaveState(url1,url2,path=None):
+  path= path or getSaveStatePath()
   if not os.path.isfile(path):
-    state = {'last_method':None,'rate_limit': [],'last_mb': {},'cooldown_time': False,'last_url':None}
+    state = {'last_method':None,'rate_limits': {url1: [], url2: []},'last_mb': {url1: {}, url2: {}},'cooldown_times': {url1: {}, url2: {}},'last_url':url1}
     safe_dump_to_file(data=state,file_path=path)
-  return safe_read_from_json(getSaveStatePath())
+  return safe_read_from_json(path)
 def is_time_interval(time_obj, interval):
     return (time.time() - time_obj) < interval-1
 
@@ -23,120 +27,151 @@ def get_mb(sum_list, limit, last_mb):
 
 def datasize(data):
     if isinstance(data, str):
-        return len(data.encode('utf-8'))
+        size = len(data.encode('utf-8'))
     elif isinstance(data, (bytes, bytearray)):
-        return len(data)
+        size = len(data)
     elif isinstance(data, list) or isinstance(data, dict):
-        return len(json.dumps(data).encode('utf-8'))
+        size = len(json.dumps(data).encode('utf-8'))
     else:
-        return len(str(data).encode('utf-8'))
-
+        size = len(str(data).encode('utf-8'))
+    return size/1000
 class RateLimiter(metaclass=SingletonMeta):
-    def __init__(self,rpc_url = None,fallback_rpc_url=None,env_directory=None):
+    def __init__(self, rpc_url=None, fallback_rpc_url=None, env_directory=None,save_state_path = None):
         if not hasattr(self, 'initialized'):  # Prevent reinitialization
-          self.initialized = True
-          self.rpc_url = rpc_url or get_env_value(key="solana_primary_rpc_url",path=env_directory) or "https://api.mainnet-beta.solana.com"
-          self.fallback_rpc_url = fallback_rpc_url or get_env_value(key="solana_fallback_rpc_url",path=env_directory)
-          self.initialized = True
-          self.rate_limit = []
-          self.last_mb = {}
-          self.cooldown_time = False
-          self.url1 = self.rpc_url
-          self.url2 = self.fallback_rpc_url
-          self.state_file = getSaveStatePath()
-          self.last_url = None
-          self.last_method=None
-          self.load_state()
-    def get_url_2(self):
-        return self.url2
+            self.initialized = True
+            self.rpc_url = rpc_url or get_env_value(key="solana_primary_rpc_url", path=env_directory) or "https://api.mainnet-beta.solana.com"
+            self.fallback_rpc_url = fallback_rpc_url or get_env_value(key="solana_fallback_rpc_url", path=env_directory)
+            self.state_file = save_state_path or getSaveStatePath()
+            self.url1 = self.rpc_url
+            self.url2 = self.fallback_rpc_url
+            self.rate_limits = {self.url1: [], self.url2: []}  # Separate rate limits for each URL
+            self.last_mb = {self.url1: {}, self.url2: {}}
+            self.cooldown_times = {self.url1: {}, self.url2: {}}  # Separate cooldowns for each URL
+            self.last_url = None
+            self.last_method = None
+            self.load_state()
 
     def save_state(self):
         state = {
-            'last_method':self.last_method,
-            'rate_limit': self.rate_limit,
+            'last_method': self.last_method,
+            'rate_limits': self.rate_limits,
             'last_mb': self.last_mb,
-            'cooldown_time': self.cooldown_time,
+            'cooldown_times': self.cooldown_times,
             'last_url': self.last_url
         }
-        safe_dump_to_file(data=state,file_path=self.state_file)
+        safe_dump_to_file(data=state, file_path=self.state_file)
 
     def load_state(self):
-        state = readSaveState()
+        state = readSaveState(self.url1,self.url2)
         self.last_method = state.get('last_method')
-        self.rate_limit = state.get('rate_limit', [])
-        self.last_mb = state.get('last_mb', {})
-        self.last_url = state.get('last_url')
-        self.cooldown_time = state.get('cooldown_time', False)
-
-    def set_cooldown(self, add=False):
-        if add:
-            self.cooldown_time = time.time() + add
-        if self.cooldown_time and (time.time() > self.cooldown_time):
-            self.cooldown_time = False
-        return bool(self.cooldown_time)
-
-    def get_last_rate_limit(self):
-        if self.rate_limit:
-            return self.rate_limit[-1]
+        self.rate_limits = state.get('rate_limits', {self.url1: [], self.url2: []})
+        self.last_mb = state.get('last_mb', {self.url1: {}, self.url2: {}})
+        self.cooldown_times = state.get('cooldown_times', {self.url1: {}, self.url2: {}})
+
+    def set_cooldown(self, url, method=None, add=False):
+        if method:
+            if add:
+                self.cooldown_times[url][method] = time.time() + add
+            if method in self.cooldown_times[url] and time.time() > self.cooldown_times[url][method]:
+                del self.cooldown_times[url][method]
+            return method in self.cooldown_times[url]
+        return False
+
+    def get_last_rate_limit(self, url):
+        if self.rate_limits[url]:
+            return self.rate_limits[url][-1]
         return {}
 
-    def is_all_limit(self, method):
-        if method not in self.last_mb:
-            self.last_mb[method] = 0
+    def is_all_limit(self, url, method):
+        if method not in self.last_mb[url]:
+            self.last_mb[url][method] = 0
 
-        if self.set_cooldown():
+        if self.set_cooldown(url, method):
+            print(f'set_cooldown for method {method} in {url} hit')
             return True
 
-        self.rate_limit = [query for query in self.rate_limit if is_time_interval(query.get('time') or 0, 30)]
-        last_rate_limit = self.get_last_rate_limit()
-
-        # Check if data size exceeds limit
-        if get_mb(sum(query.get('data', 0) for query in self.rate_limit), 100, self.last_mb[method]):
+        # Clean up expired queries for the current URL
+        self.rate_limits[url] = [
+            query for query in self.rate_limits[url] if is_time_interval(query.get('time') or 0, 30)
+        ]
+        last_rate_limit = self.get_last_rate_limit(url)
+
+        # Check data size limits
+        total_mb = sum(query.get('data', 0) for query in self.rate_limits[url])
+        mb = get_mb(total_mb, 100, self.last_mb[url][method])
+        if mb:
+            print(f'mb {total_mb} of limit 100 hit')
             return True
 
         # Check if the last request for the same method was within 10 seconds
-        if self.last_method == method and is_time_interval(last_rate_limit.get('time') or 0, 10):
-            return True
-
-        # Check if more than 100 requests in the last 10 seconds
-        time_rate = [query for query in self.rate_limit if is_time_interval(query.get('time') or 0, 10)]
+        time_rate = [
+            query for query in self.rate_limits[url] if is_time_interval(query.get('time') or 0, 10)
+        ]
         if len(time_rate) > 100:
+            print(f'time_rate {time_rate} of timerate limit 100 hit')
             return True
 
-        # Check if more than 40 requests for the same method in the last 10 seconds
-        method_specific_time_rate = [query for query in time_rate if query['method'] == method]
+        method_specific_time_rate = [
+            query for query in time_rate if query['method'] == method
+        ]
         if len(method_specific_time_rate) > 40:
+            print(f'method_specific_time_rate {len(method_specific_time_rate)} of method_specific_time_rate limit 40 hit')
             return True
 
         return False
 
-    def log_response(self, method=None, response=None):
+    def log_response(self, method=None, response=None, retry_after=None):
         method = method or 'default_method'
         response = response or {}
         data_size = datasize(response)
-        self.last_mb[method] = data_size
-
-        if self.last_url == self.url1:
-            self.rate_limit.append({'method': method, 'data': data_size, 'time': time.time()})
-
-        self.rate_limit = [query for query in self.rate_limit if is_time_interval(query['time'], 30)]
+        active_url = self.last_url
+
+        # Handle Retry-After logic
+        if retry_after:
+            try:
+                wait_time = int(retry_after)
+            except ValueError:
+                retry_after_date = datetime.strptime(retry_after, '%a, %d %b %Y %H:%M:%S GMT')
+                wait_time = (retry_after_date - datetime.utcnow()).total_seconds()
+            self.set_cooldown(active_url, method, add=max(wait_time, 0))
+
+        if active_url == self.url1:
+            self.rate_limits[active_url].append({'method': method, 'data': data_size, 'time': time.time()})
+
+        # Clean up expired entries for the current URL
+        self.rate_limits[active_url] = [
+            query for query in self.rate_limits[active_url] if is_time_interval(query['time'], 30)
+        ]
         self.save_state()
-
+    def get_cooldown_for_method(self,url,method):
+        wait_time = 0
+        if self.set_cooldown(url,method):
+          wait_time = int(self.cooldown_times[url][method]) - time.time()
+          if wait_time <= 0:
+            del self.cooldown_times[url][method]
+           
+          else:
+            return wait_time
+        return False
     def get_url(self, method=None):
         method = method or 'default_method'
-        if self.url2 and method == 'get_url_2':
-            self.last_url = self.url2
-            return self.url2
+        wait_time = self.get_cooldown_for_method(self.url1,method)
         
-        if not self.is_all_limit(method):
+        if wait_time:
+          wait_time = int(self.cooldown_times[self.url1][method]) - time.time()
+          if wait_time > 0:
+            self.last_url = self.url2
+            #retry_after_date = datetime.strptime(str(int(self.cooldown_times[method])), '%a, %d %b %Y %H:%M:%S GMT')
+            print(f"{method} is on cooldown for {wait_time} more seconds")
+        if method == 'get_url2':
+          self.last_url = self.url2
+          return self.last_url
+        # If fallback URL is selected, skip all limits
+ 
+        is_limit = self.is_all_limit(self.url1, method)
+        if not is_limit:
             self.last_method = method
-            
             self.last_url = self.url1
-        elif self.url2:
-            self.last_url = self.url2
-        else:
-          return {"rate_limited":"limit has been reached"}
+        print([is_limit,self.last_url])
         return self.last_url
 
-
-

abstract_solana/abstract_rpcs/solana_rpc_client.py

@@ -2,7 +2,7 @@ from solana.rpc.core import _ClientCore
 from typing import Dict, List, Optional, Sequence, Union
 from solana.rpc.commitment import Commitment, Finalized
 from .rate_limiter import RateLimiter
-from ..abstract_utils.utils import *
+from ..abstract_utils.pubkey_utils import *
 import inspect,asyncio,json,requests
 from abstract_apis import *
 from abstract_utilities import is_number
@@ -85,10 +85,17 @@ def get_rpc_dict(endpoint,*args,**kwargs):
     kwargs = get_conversions(variables,*args,**kwargs)
     kwargs = json.loads(str(call_function(function,**kwargs)))
     return kwargs
-def abstract_solana_rate_limited_call(endpoint,*args,**kwargs):
-    rpc_dict = get_rpc_dict(endpoint,*args,**kwargs)
-    url = rate_limiter.get_url(rpc_dict.get('method'))
-    if isinstance(url,dict):
-        url=url.get('url')
-    response = postRpcRequest(url=url,**rpc_dict)
+def make_call(url,body):
+    return postRpcRequest(url=url,**body,retry_after =True,status_code=True, headers=get_headers())
+def abstract_solana_rate_limited_call(method, *args, **kwargs):
+    # Build the request body
+    body = get_rpc_dict(method, *args, **kwargs)
+    body_method = body.get('method')
+    url = rate_limiter.get_url(body_method)
+    response,status_code,retry_after = make_call(url,body)   
+    rate_limiter.log_response(body_method, response, retry_after)
+    if url == rate_limiter.url1 and status_code == 429:
+        url = rate_limiter.get_url('get_url2')
+        response,status_code,retry_after = make_call(url,body)
+        rate_limiter.log_response(body_method, response)
     return response

{abstract_solana-0.0.2.81.dist-info → abstract_solana-0.0.2.83.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: abstract_solana
-Version: 0.0.2.81
+Version: 0.0.2.83
 Home-page: https://github.com/AbstractEndeavors/abstract_solana
 Author: putkoff
 Author-email: partners@abstractendeavors.com

{abstract_solana-0.0.2.81.dist-info → abstract_solana-0.0.2.83.dist-info}/RECORD

@@ -1,10 +1,10 @@
 abstract_solana/__init__.py,sha256=uEPtTF-WPCIwpwR8OgAI5prloYQpPbx-YExFgzZvFkQ,59
-abstract_solana/abstract_rpcs/__init__.py,sha256=DP-TZ8GGPUu4ETVDuFFLFxkJH2_QY-j8R19zB0reA9E,158
+abstract_solana/abstract_rpcs/__init__.py,sha256=LIkUCWcuzUWVN1WjzcXjQ9Pl7cbcN8TwknXir-Uk34c,168
 abstract_solana/abstract_rpcs/db_templates.py,sha256=sjdHfHIq9bO6VuDm3hwzn46NUrXXrGnB0knYNeVU7k8,29839
 abstract_solana/abstract_rpcs/get_api_gui.py,sha256=OZ61HHb0gwdYjirwFFmnwk8z9x5np3cecCj9MdmuF8U,12372
 abstract_solana/abstract_rpcs/get_body.py,sha256=UV85217q7mIpYOhVZdnzfmgZxD3QM0w0J0oevXyYtdE,51272
-abstract_solana/abstract_rpcs/rate_limiter.py,sha256=AuHtjpCfTw5os-v5kW0bt1wWOvaRvZIkoWjlexlego8,5216
-abstract_solana/abstract_rpcs/solana_rpc_client.py,sha256=69hynDmVyYtqoATK9n2Fqq6PzvO0m0mOxG1R-DdUDXg,3704
+abstract_solana/abstract_rpcs/rate_limiter.py,sha256=yfUwmdMXty05VN_A-baTrOKY4EH9fYF1Vc7SKPZgcTk,7248
+abstract_solana/abstract_rpcs/solana_rpc_client.py,sha256=s-Mg0mDcDwjU3dm6nK_U6wWysaWTLoC4eXpSHbYsbyY,4132
 abstract_solana/abstract_utils/__init__.py,sha256=HCbBVQ5BIFCVkFqqTMHp1Y__YQAO4HTq_KHVdMCe89U,296
 abstract_solana/abstract_utils/account_key_utils.py,sha256=VMJd4GOTK1vn8UZsfXDnjxDOGoQWGY6fvflJqPZ7Xvs,877
 abstract_solana/abstract_utils/constants.py,sha256=cSmCKzQiNZocX1YkKYrdY-O449aYhi7BT_j-45HZN-E,1418
@@ -20,7 +20,7 @@ abstract_solana/pump_functions/__init__.py,sha256=BiRxwJd1JWwEft63zqYwZ_Xs6UDp4h
 abstract_solana/pump_functions/buy_sell_pump.py,sha256=gjv_1et20s1Li0ygcURofO29VPkO1v-a5G5Bo_sZs_c,7860
 abstract_solana/pump_functions/pump_fun_keys.py,sha256=BeWbV9_wd-c6ydF33drW-gZBDPWolbsMZL4cNhP3eOU,8537
 abstract_solana/pump_functions/token_utils.py,sha256=O-Fgj3L1NhND-k4INa3WvLAEXg2N9u1fVqyLFzn1PwM,2714
-abstract_solana-0.0.2.81.dist-info/METADATA,sha256=gDlvhikUAtAHNrTswQTsjwrbUohER7Fy2E2pnoLgH0U,981
-abstract_solana-0.0.2.81.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
-abstract_solana-0.0.2.81.dist-info/top_level.txt,sha256=SsJYent8eZQ0FU2jmP8wTj7aFZFhNwxxP-5cCTQ2B-o,16
-abstract_solana-0.0.2.81.dist-info/RECORD,,
+abstract_solana-0.0.2.83.dist-info/METADATA,sha256=7UPZnKtp2vufYZBBU9NuBOu43FL8ba4ULsUuifP9Ubc,981
+abstract_solana-0.0.2.83.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
+abstract_solana-0.0.2.83.dist-info/top_level.txt,sha256=SsJYent8eZQ0FU2jmP8wTj7aFZFhNwxxP-5cCTQ2B-o,16
+abstract_solana-0.0.2.83.dist-info/RECORD,,