abstract-solana 0.0.2.6__tar.gz → 0.0.2.9__tar.gz

{abstract_solana-0.0.2.6 → abstract_solana-0.0.2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: abstract_solana
-Version: 0.0.2.6
+Version: 0.0.2.9
 Home-page: https://github.com/AbstractEndeavors/abstract_solana
 Author: putkoff
 Author-email: partners@abstractendeavors.com

abstract_solana-0.0.2.9/pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools>=42", "wheel"]
+build-backend = "setuptools.build_meta"

{abstract_solana-0.0.2.6 → abstract_solana-0.0.2.9}/setup.py

@@ -1,34 +1,34 @@
-from time import time
-import setuptools
-with open("README.md", "r", encoding="utf-8") as fh:
-    long_description = fh.read()
-setuptools.setup(
-    name='abstract_solana',
-    version='0.0.2.06',
-    author='putkoff',
-    author_email='partners@abstractendeavors.com',
-    description="",
-    long_description=long_description,
-    long_description_content_type='text/markdown',
-    url='https://github.com/AbstractEndeavors/abstract_solana',
-    classifiers=[
-        'Development Status :: 3 - Alpha',
-        'Intended Audience :: Developers',
-        'Topic :: Software Development :: Libraries',
-        'License :: OSI Approved :: MIT License',
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.6',
-        'Programming Language :: Python :: 3.7',
-        'Programming Language :: Python :: 3.8',
-        'Programming Language :: Python :: 3.9',
-        'Programming Language :: Python :: 3.10',
-        'Programming Language :: Python :: 3.11',
-        'Programming Language :: Python :: 3.12',
-    ],
-    install_requires=['solders','abstract_solcatcher','abstract_utilities','solana'],
-    package_dir={"": "src"},
-    packages=setuptools.find_packages(where="src"),
-    python_requires=">=3.6",
-    # Add this line to include wheel format in your distribution
-    setup_requires=['wheel'],
-)
+from time import time
+import setuptools
+with open("README.md", "r", encoding="utf-8") as fh:
+    long_description = fh.read()
+setuptools.setup(
+    name='abstract_solana',
+    version='0.0.2.9',
+    author='putkoff',
+    author_email='partners@abstractendeavors.com',
+    description="",
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    url='https://github.com/AbstractEndeavors/abstract_solana',
+    classifiers=[
+        'Development Status :: 3 - Alpha',
+        'Intended Audience :: Developers',
+        'Topic :: Software Development :: Libraries',
+        'License :: OSI Approved :: MIT License',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
+    ],
+    install_requires=['solders','abstract_solcatcher','abstract_utilities','solana'],
+    package_dir={"": "src"},
+    packages=setuptools.find_packages(where="src"),
+    python_requires=">=3.6",
+    # Add this line to include wheel format in your distribution
+    setup_requires=['wheel'],
+)

{abstract_solana-0.0.2.6 → abstract_solana-0.0.2.9}/src/abstract_solana/abstract_rpcs/__init__.py

@@ -1,3 +1,4 @@
 from .get_body import *
 from .db_templates import *
-from .solana_rpc_client import get_rpc_dict,abstract_solana_rate_limited_call
+from .solana_rpc_client import get_rpc_dict,abstract_solana_rate_limited_call,make_call
+from .rate_limiter import *

abstract_solana-0.0.2.9/src/abstract_solana/abstract_rpcs/rate_limiter.py

@@ -0,0 +1,178 @@
+import time
+import os
+import json
+from datetime import datetime
+
+from abstract_utilities import *
+from abstract_security import get_env_value
+def getAbsFile():
+  return os.path.abspath(__file__)
+def getAbsDir():
+  return os.path.dirname(getAbsFile())
+def getAbsPath(path):
+  return os.path.join(getAbsDir(),path)
+def getSaveStatePath():
+  return getAbsPath('rate_limiter_state.json')
+def readSaveState(url1,url2,path=None):
+  path= path or getSaveStatePath()
+  if not os.path.isfile(path):
+    state = {'last_method':None,'rate_limits': {url1: [], url2: []},'last_mb': {url1: {}, url2: {}},'cooldown_times': {url1: {}, url2: {}},'last_url':url1}
+    safe_dump_to_file(data=state,file_path=path)
+  return safe_read_from_json(path)
+def is_time_interval(time_obj, interval):
+    return (time.time() - time_obj) < interval-1
+
+def get_mb(sum_list, limit, last_mb):
+    return (sum_list + last_mb) > limit
+
+def datasize(data):
+    if isinstance(data, str):
+        size = len(data.encode('utf-8'))
+    elif isinstance(data, (bytes, bytearray)):
+        size = len(data)
+    elif isinstance(data, list) or isinstance(data, dict):
+        size = len(json.dumps(data).encode('utf-8'))
+    else:
+        size = len(str(data).encode('utf-8'))
+    return size/1000
+class RateLimiter(metaclass=SingletonMeta):
+    def __init__(self, rpc_url=None, fallback_rpc_url=None, env_directory=None,save_state_path = None):
+        if not hasattr(self, 'initialized'):  # Prevent reinitialization
+            self.initialized = True
+            self.rpc_url = rpc_url or get_env_value(key="solana_primary_rpc_url", path=env_directory) or "http://api.mainnet-beta.solana.com"
+            self.fallback_rpc_url = fallback_rpc_url or get_env_value(key="solana_fallback_rpc_url", path=env_directory)
+            self.state_file = save_state_path or getSaveStatePath()
+            self.url1 = self.rpc_url
+            self.url2 = self.fallback_rpc_url
+            self.rate_limits = {self.url1: [], self.url2: []}  # Separate rate limits for each URL
+            self.last_mb = {self.url1: {}, self.url2: {}}
+            self.cooldown_times = {self.url1: {}, self.url2: {}}  # Separate cooldowns for each URL
+            self.last_url = None
+            self.last_method = None
+            self.load_state()
+
+    def save_state(self):
+        state = {
+            'last_method': self.last_method,
+            'rate_limits': self.rate_limits,
+            'last_mb': self.last_mb,
+            'cooldown_times': self.cooldown_times,
+            'last_url': self.last_url
+        }
+        safe_dump_to_file(data=state, file_path=self.state_file)
+
+    def load_state(self):
+        state = readSaveState(self.url1,self.url2)
+        self.last_method = state.get('last_method')
+        self.rate_limits = state.get('rate_limits', {self.url1: [], self.url2: []})
+        self.last_mb = state.get('last_mb', {self.url1: {}, self.url2: {}})
+        self.cooldown_times = state.get('cooldown_times', {self.url1: {}, self.url2: {}})
+
+    def set_cooldown(self, url, method=None, add=False):
+        if method:
+            if add:
+                self.cooldown_times[url][method] = time.time() + add
+            if method in self.cooldown_times[url] and time.time() > self.cooldown_times[url][method]:
+                del self.cooldown_times[url][method]
+            return method in self.cooldown_times[url]
+        return False
+
+    def get_last_rate_limit(self, url):
+        if self.rate_limits[url]:
+            return self.rate_limits[url][-1]
+        return {}
+
+    def is_all_limit(self, url, method):
+       if url == self.url1:
+         if method not in self.last_mb:
+            self.last_mb[method] = 0
+
+         if self.set_cooldown(url, method):
+            print(f'set_cooldown for method {method} in {url} hit')
+            return True
+
+        # Clean up expired queries for the current URL
+        self.rate_limits[url] = [
+            query for query in self.rate_limits[url] if is_time_interval(query.get('time') or 0, 30)
+        ]
+        last_rate_limit = self.get_last_rate_limit(url)
+
+        # Check data size limits
+        total_mb = sum(query.get('data', 0) for query in self.rate_limits[url])
+        mb = get_mb(total_mb, 100, self.last_mb[url][method])
+        if mb:
+            print(f'mb {total_mb} of limit 100 hit')
+            return True
+
+        # Check if the last request for the same method was within 10 seconds
+        time_rate = [
+            query for query in self.rate_limits[url] if is_time_interval(query.get('time') or 0, 10)
+        ]
+        if len(time_rate) > 100:
+            print(f'time_rate {time_rate} of timerate limit 100 hit')
+            return True
+
+        method_specific_time_rate = [
+            query for query in time_rate if query['method'] == method
+        ]
+        if len(method_specific_time_rate) > 40:
+            print(f'method_specific_time_rate {len(method_specific_time_rate)} of method_specific_time_rate limit 40 hit')
+            return True
+
+        return False
+
+    def log_response(self, method=None, response=None, retry_after=None):
+        method = method or 'default_method'
+        response = response or {}
+        data_size = datasize(response)
+        active_url = self.last_url
+
+        # Handle Retry-After logic
+        if retry_after:
+            try:
+                wait_time = int(retry_after)
+            except ValueError:
+                retry_after_date = datetime.strptime(retry_after, '%a, %d %b %Y %H:%M:%S GMT')
+                wait_time = (retry_after_date - datetime.utcnow()).total_seconds()
+            self.set_cooldown(active_url, method, add=max(wait_time, 0))
+
+        if active_url == self.url1:
+            self.rate_limits[active_url].append({'method': method, 'data': data_size, 'time': time.time()})
+
+        # Clean up expired entries for the current URL
+        self.rate_limits[active_url] = [
+            query for query in self.rate_limits[active_url] if is_time_interval(query['time'], 30)
+        ]
+        self.save_state()
+    def get_cooldown_for_method(self,url,method):
+        wait_time = 0
+        if self.set_cooldown(url,method):
+          wait_time = int(self.cooldown_times[url][method]) - time.time()
+          if wait_time <= 0:
+            del self.cooldown_times[url][method]
+           
+          else:
+            return wait_time
+        return False
+    def get_url(self, method=None):
+        method = method or 'default_method'
+        wait_time = self.get_cooldown_for_method(self.url1,method)
+        
+        if wait_time:
+          wait_time = int(self.cooldown_times[self.url1][method]) - time.time()
+          if wait_time > 0:
+            self.last_url = self.url2
+            #retry_after_date = datetime.strptime(str(int(self.cooldown_times[method])), '%a, %d %b %Y %H:%M:%S GMT')
+            print(f"{method} is on cooldown for {wait_time} more seconds")
+        if method == 'get_url2':
+          self.last_url = self.url2
+          return self.last_url
+        # If fallback URL is selected, skip all limits
+ 
+        is_limit = self.is_all_limit(self.url1, method)
+        if not is_limit:
+            self.last_method = method
+            self.last_url = self.url1
+        print([is_limit,self.last_url])
+        return self.last_url
+

{abstract_solana-0.0.2.6 → abstract_solana-0.0.2.9}/src/abstract_solana/abstract_rpcs/solana_rpc_client.py

@@ -1,11 +1,11 @@
 from solana.rpc.core import _ClientCore
 from typing import Dict, List, Optional, Sequence, Union
 from solana.rpc.commitment import Commitment, Finalized
-from ..abstract_utils.pubkey_utils import get_pubkey,get_sigkey
+from .rate_limiter import RateLimiter
+from ..abstract_utils.pubkey_utils import *
 import inspect,asyncio,json,requests
-from abstract_apis import get_url,make_endpoint,get_headers,asyncPostRequest,get_async_response,get_headers
+from abstract_apis import *
 from abstract_utilities import is_number
-from .rate_limiter import RateLimiter
 rate_limiter = RateLimiter()
 def convert_to_lower(string_obj):
     return ''.join(f"_{char.lower()}" if char.isupper() else char for char in str(string_obj))
@@ -85,10 +85,17 @@ def get_rpc_dict(endpoint,*args,**kwargs):
     kwargs = get_conversions(variables,*args,**kwargs)
     kwargs = json.loads(str(call_function(function,**kwargs)))
     return kwargs
-def abstract_solana_rate_limited_call(endpoint,*args,**kwargs):
-    rpc_dict = get_rpc_dict(endpoint,*args,**kwargs)
-    url = rate_limiter.get_url(rpc_dict.get('method'))
-    if isinstance(url,dict):
-        return url
-    response = get_async_response(asyncPostRequest,url=url,endpoint=endpoint,data=rpc_dict)
+def make_call(url,body):
+    return postRpcRequest(url=url,**body,retry_after =True,status_code=True, headers=get_headers())
+def abstract_solana_rate_limited_call(method, *args, **kwargs):
+    # Build the request body
+    body = get_rpc_dict(method, *args, **kwargs)
+    body_method = body.get('method')
+    url = rate_limiter.get_url(body_method)
+    response,status_code,retry_after = make_call(url,body)   
+    rate_limiter.log_response(body_method, response, retry_after)
+    if url == rate_limiter.url1 and status_code == 429:
+        url = rate_limiter.get_url('get_url2')
+        response,status_code,retry_after = make_call(url,body)
+        rate_limiter.log_response(body_method, response)
     return response

{abstract_solana-0.0.2.6 → abstract_solana-0.0.2.9}/src/abstract_solana.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: abstract_solana
-Version: 0.0.2.6
+Version: 0.0.2.9
 Home-page: https://github.com/AbstractEndeavors/abstract_solana
 Author: putkoff
 Author-email: partners@abstractendeavors.com

{abstract_solana-0.0.2.6 → abstract_solana-0.0.2.9}/src/abstract_solana.egg-info/SOURCES.txt

@@ -1,4 +1,5 @@
 README.md
+pyproject.toml
 setup.py
 src/abstract_solana/__init__.py
 src/abstract_solana.egg-info/PKG-INFO

abstract_solana-0.0.2.6/src/abstract_solana/abstract_rpcs/rate_limiter.py

@@ -1,141 +0,0 @@
-import time,os,json
-from abstract_utilities import *
-from abstract_security import *
-def getAbsFile():
-  return os.path.abspath(__file__)
-def getAbsDir():
-  return os.path.dirname(getAbsFile())
-def getAbsPath(path):
-  return os.path.join(getAbsDir(),path)
-def getSaveStatePath():
-  return getAbsPath('rate_limiter_state.json')
-def readSaveState():
-  path= getSaveStatePath()
-  if not os.path.isfile(path):
-    state = {'last_method':None,'rate_limit': [],'last_mb': {},'cooldown_time': False,'last_url':None}
-    safe_dump_to_file(file_path=path,data=state)
-  return safe_read_from_json(getSaveStatePath())
-def is_time_interval(time_obj, interval):
-    print([time.time() - time_obj,interval-1])
-    return (time.time() - time_obj) < interval-1
-
-def get_mb(sum_list, limit, last_mb):
-    return (sum_list + last_mb) > limit
-
-def datasize(data):
-    if isinstance(data, str):
-        return len(data.encode('utf-8'))
-    elif isinstance(data, (bytes, bytearray)):
-        return len(data)
-    elif isinstance(data, list) or isinstance(data, dict):
-        return len(json.dumps(data).encode('utf-8'))
-    else:
-        return len(str(data).encode('utf-8'))
-
-class RateLimiter(metaclass=SingletonMeta):
-    def __init__(self,rpc_url = None,fallback_rpc_url=None,env_directory=None):
-        if not hasattr(self, 'initialized'):  # Prevent reinitialization
-          self.initialized = True
-          self.rpc_url = rpc_url or get_env_value(key="solana_primary_rpc_url",path=env_directory) or "https://api.mainnet-beta.solana.com"
-          self.fallback_rpc_url = fallback_rpc_url or get_env_value(key="solana_fallback_rpc_url",path=env_directory)
-          self.initialized = True
-          self.rate_limit = []
-          self.last_mb = {}
-          self.cooldown_time = False
-          self.url1 = self.rpc_url
-          self.url2 = self.fallback_rpc_url
-          self.state_file = getSaveStatePath()
-          self.last_url = None
-          self.last_method=None
-          self.load_state()
-    def get_url_2(self):
-        return self.url2
-
-    def save_state(self):
-        state = {
-            'last_method':self.last_method,
-            'rate_limit': self.rate_limit,
-            'last_mb': self.last_mb,
-            'cooldown_time': self.cooldown_time,
-            'last_url': self.last_url
-        }
-        safe_dump_to_file(self.state_file, state)
-
-    def load_state(self):
-        state = readSaveState()
-        self.last_method = state.get('last_method')
-        self.rate_limit = state.get('rate_limit', [])
-        self.last_mb = state.get('last_mb', {})
-        self.last_url = state.get('last_url')
-        self.cooldown_time = state.get('cooldown_time', False)
-
-    def set_cooldown(self, add=False):
-        if add:
-            self.cooldown_time = time.time() + add
-        if self.cooldown_time and (time.time() > self.cooldown_time):
-            self.cooldown_time = False
-        return bool(self.cooldown_time)
-
-    def get_last_rate_limit(self):
-        if self.rate_limit:
-            return self.rate_limit[-1]
-        return {}
-
-    def is_all_limit(self, method):
-        if method not in self.last_mb:
-            self.last_mb[method] = 0
-
-        if self.set_cooldown():
-            return True
-
-        self.rate_limit = [query for query in self.rate_limit if is_time_interval(query.get('time') or 0, 30)]
-        last_rate_limit = self.get_last_rate_limit()
-
-        # Check if data size exceeds limit
-        if get_mb(sum(query.get('data', 0) for query in self.rate_limit), 100, self.last_mb[method]):
-            return True
-
-        # Check if the last request for the same method was within 10 seconds
-        if self.last_method == method and is_time_interval(last_rate_limit.get('time') or 0, 10):
-            return True
-
-        # Check if more than 100 requests in the last 10 seconds
-        time_rate = [query for query in self.rate_limit if is_time_interval(query.get('time') or 0, 10)]
-        if len(time_rate) > 100:
-            return True
-
-        # Check if more than 40 requests for the same method in the last 10 seconds
-        method_specific_time_rate = [query for query in time_rate if query['method'] == method]
-        if len(method_specific_time_rate) > 40:
-            return True
-
-        return False
-
-    def log_response(self, method=None, response_data=None):
-        method = method or 'default_method'
-        response_data = response_data or {}
-        data_size = datasize(response_data)
-        self.last_mb[method] = data_size
-
-        if self.last_url == self.url1:
-            self.rate_limit.append({'method': method, 'data': data_size, 'time': time.time()})
-
-        self.rate_limit = [query for query in self.rate_limit if is_time_interval(query['time'], 30)]
-        self.save_state()
-
-    def get_url(self, method=None):
-        method = method or 'default_method'
-        if self.url2 and method == 'get_url_2':
-            self.last_url = self.url2
-            return self.url2
-        
-        if not self.is_all_limit(method):
-            self.last_method = method
-            
-            self.last_url = self.url1
-        elif self.url2:
-            self.last_url = self.url2
-        else:
-          return {"rate_limited":"limit has been reached"}
-        return self.last_url
-