itd-sdk 2.5.1__tar.gz → 2.5.2__tar.gz

{itd_sdk-2.5.1 → itd_sdk-2.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: itd-sdk
-Version: 2.5.1
+Version: 2.5.2
 Summary: ITD client for python
 Keywords: itd,itdsocial
 Author: fi-res

{itd_sdk-2.5.1 → itd_sdk-2.5.2}/itd/__init__.py

@@ -32,9 +32,11 @@ iiii    iii     ttt    dd           dd
 """
 
 from importlib.metadata import version
+from time import sleep
 
 __version__ = version("itd-sdk")
 
+from itd._default import limiters
 from itd.clan import Clan, TopClans
 from itd.client import Client as ITDClient
 from itd.client import Config as ITDConfig
@@ -47,7 +49,19 @@ from itd.session import Sessions
 from itd.user import Me, User, Users, get_follow_status
 from itd.version import Apps, Changelog
 
+
+def acquire_limiters():
+    try:
+        sleep(max(limiter.get_delay() for limiter in limiters.values() if limiter.is_ran))
+    except ValueError:
+        pass
+
+    for limiter in limiters.values():
+        limiter.is_ran = False
+
+
 __all__ = [
+    '__version__',
     'Changelog',
     'Apps',
     'ITDClient',
@@ -68,5 +82,6 @@ __all__ = [
     'User',
     'Me',
     'Users',
-    'get_follow_status'
+    'get_follow_status',
+    'acquire_limiters'
 ]

{itd_sdk-2.5.1 → itd_sdk-2.5.2}/itd/_default.py

@@ -8,7 +8,8 @@ if TYPE_CHECKING:
     from itd.client import Client
 
 _default_client: Client | None = None
-limiters: dict[str, RateLimiter] = {}
+limiters: dict[int, RateLimiter] = {}
+limits: dict[str, int] = {}
 ip_limiter = IPRateLimiter()
 
 

itd_sdk-2.5.2/itd/_limiter.py

@@ -0,0 +1,97 @@
+from time import monotonic, sleep
+
+from itd.logger import get_logger
+
+l = get_logger('limiter')
+
+
+class RateLimiter:
+    window: int = 65
+
+    def __init__(self, capacity: int):
+        l.debug(r'\[%s] create rate limiter', capacity)
+        self.capacity = capacity
+        self.is_ran = True
+
+    def sync(self, remaining: int): ...
+    def acquire(self): ...
+    def on_limit(self): ...
+    def get_delay(self): ...
+
+
+class SafeRateLimiter(RateLimiter):
+    def __init__(self, capacity: int):
+        super().__init__(capacity)
+        self.delay = 60 / capacity
+        self.last_request = monotonic()
+        self.requests = 0
+        self.is_ran = False
+
+    def get_delay(self):
+        self.requests += 1
+        self.last_request = monotonic()
+        return max(0.1, self.delay - (monotonic() - self.last_request))
+
+    def acquire(self):
+        l.debug(r'\[%s] acquire limiter delay=%s', self.capacity, round(self.delay, 2))
+        sleep(self.get_delay())
+
+    def sync(self, remaining: int):
+        l.info(r'\[%s] sync limiter remaining=%s', self.capacity, remaining)
+        if remaining < self.capacity * 0.2:
+            self.delay *= 1 + 1 / (remaining or 0.5)
+            l.info(r'\[%s] increase delay=%s', self.capacity, round(self.delay, 2))
+        if remaining > self.capacity * 0.9:
+            self.delay *= 0.7
+            l.info(r'\[%s] decrease delay=%s', self.capacity, round(self.delay, 2))
+        elif 10 < self.requests < 500 and remaining > self.capacity * 0.5:
+            self.delay -= 1 / self.requests
+        self.delay = max(0.1, min(self.delay, 30))
+        self.is_ran = True
+
+    def on_limit(self):
+        self.delay *= 2
+        self.delay = min(self.delay, 30)
+
+
+class BurstRateLimiter(RateLimiter):
+    def __init__(self, capacity: int):
+        super().__init__(capacity)
+        self.requests = []
+
+    def sync(self, remaining: int):
+        l.debug(r'\[%s] sync limiter remaining=%s', self.capacity, remaining)
+
+        used = self.capacity - remaining
+        if used > 0:
+            self.requests = self.requests[-used:]  # ai
+        while len(self.requests) < used:
+            self.requests.append(monotonic())
+
+    def acquire(self):
+        l.debug(r'\[%s] acquire limiter', self.capacity)
+
+        self.requests = [request for request in self.requests if request > monotonic() - self.window]
+        while len(self.requests) >= self.capacity:
+            self.requests = [request for request in self.requests if request > monotonic() - self.window]
+            if self.requests:
+                sleep(max(self.requests[0] + self.window - monotonic(), 0.1))
+
+
+class IPRateLimiter:
+    window: int = 60
+    max_requests: int = 90
+
+    def __init__(self):
+        self.requests = []
+
+    def acquire(self):
+        self.requests.append(monotonic())
+        self.requests = [request for request in self.requests if request > monotonic() - self.window]
+        while len(self.requests) >= self.max_requests:
+            self.requests = [request for request in self.requests if request > monotonic() - self.window]
+            if self.requests:
+                sleep(self.get_delay())
+
+    def get_delay(self):
+        return max(self.requests[0] + self.window - monotonic(), 0.1)

{itd_sdk-2.5.1 → itd_sdk-2.5.2}/itd/base.py

@@ -9,7 +9,7 @@ from pydantic import BaseModel
 from requests import Response
 from requests.exceptions import JSONDecodeError
 
-from itd._default import get_default_client, ip_limiter, limiters
+from itd._default import get_default_client, ip_limiter, limiters, limits
 from itd._limiter import BurstRateLimiter, SafeRateLimiter
 from itd.enums import ALL, BATCH, All, Batch, DebugResponseMode, LoadStatus
 from itd.exceptions import DEFAULT_ERRORS, AccessTokenExpiredError, AccountDeletedError, ITDException, RateLimitError, UnauthorizedError, ValidationError
@@ -100,10 +100,10 @@ class ITDBaseModel:
 
             try:
                 value = object.__getattribute__(self, name)
-                exists = True
-            except AttributeError:
+                exc = None
+            except AttributeError as e:
                 value = None
-                exists = False
+                exc = e
 
             if (
                 _getattr(self, '_refreshable')
@@ -119,8 +119,8 @@ class ITDBaseModel:
                 self.refresh()
                 return object.__getattribute__(self, name)
 
-            if not exists:
-                raise AttributeError
+            if exc is not None:
+                raise exc
             return value
 
 
@@ -317,13 +317,12 @@ def api_wrapper(*exceptions: ITDException):
     def decorator(func):
         @wraps(func)
         def wrapper(client: Client, *args, **kwargs) -> Response | None:
+            name = func.__name__
+
             def exec():
-                name = func.__name__
-                if name in ('get_followers', 'get_following'):
-                    name = 'get_follow'
                 l.info('exec %s %s %s', func.__name__, _filter_bytes(args), kwargs)
-                if name in limiters:
-                    limiters[name].acquire()
+                if client.config.auto_acquire and name in limits:
+                    limiters[limits[name]].acquire()
                 ip_limiter.acquire()
 
                 res: Response = func(client, *args, **kwargs)
@@ -336,14 +335,15 @@ def api_wrapper(*exceptions: ITDException):
 
                 remaining = int(res.headers.get('x-ratelimit-remaining', 0))
                 limit = int(res.headers.get('x-ratelimit-limit', 0))
+                limits[name] = limit
 
                 if client.config._anti_rate_limit:
-                    if client.config._burst_requests and name not in limiters:
-                        limiters[name] = BurstRateLimiter(name, limit)
+                    if client.config._burst_requests and limit not in limiters:
+                        limiters[limit] = BurstRateLimiter(limit)
 
-                    elif name not in limiters:
-                        limiters[name] = SafeRateLimiter(name, limit)
-                    limiters[name].sync(remaining)
+                    elif limit not in limiters:
+                        limiters[limit] = SafeRateLimiter(limit)
+                    limiters[limit].sync(remaining)
 
                 if client.config.debug_response == DebugResponseMode.BEFORE:
                     l.debug('response (raw): %s', res.text)
@@ -409,6 +409,7 @@ def api_wrapper(*exceptions: ITDException):
                     retry_after = getattr(e, 'retry_after') or client.config.retry_delay
                     l.warning('%s on %s: wait %ss', e.__class__.__name__, func.__name__, retry_after)
                     sleep(retry_after)
+                    limiters[name].on_limit()
 
         return wrapper
 

{itd_sdk-2.5.1 → itd_sdk-2.5.2}/itd/client.py

@@ -17,7 +17,7 @@ from itd.api.auth import change_password, logout, refresh_token
 from itd.api.posts import get_stats
 from itd.api.search import search
 from itd.enums import BATCH, All, AuthLevel, Batch, DebugResponseMode, ParseMode, RateLimitMode, Role, UserAgent
-from itd.exceptions import InsufficientAuthLevelError, InternalError, NotFoundError, RateLimitError
+from itd.exceptions import InsufficientAuthLevelError, NotFoundError, RateLimitError
 from itd.hashtag import Hashtag, Hashtags
 from itd.logger import get_logger
 from itd.post import DwellTracker, Post
@@ -80,6 +80,7 @@ class Config:
     burst_requests: bool | None = None
     anti_ip_ban: bool = True
     limit_coefficient: float | None = None
+    auto_acquire: bool = True
 
     # enable_logging: bool | None = None
     # logging_level = 'DEBUG'
@@ -101,8 +102,8 @@ class Config:
     timeout_file: float | None = None
     timeout_file_download: float | None = None
 
-    url: str = 'xn--d1ah4a.com'
-    url_api: str | None = None
+    url: str = 'https://xn--d1ah4a.com/api'
+    url_api: str | None = None  # deprecated
     user_agent: UserAgent | str = UserAgent.BROWSER
     solve_challenge: bool = True
 
@@ -134,10 +135,12 @@ class Config:
 
     on_exceptions: dict[type[Exception], Callable[[Exception], None]] = field(default_factory=dict)
     batch_sizes: BatchSizes = field(default_factory=BatchSizes)
+    refresh_token_cookie_name: str = 'refresh_token'
 
     def __post_init__(self):
-        self._url_api = self.url_api if self.url_api else f'https://{self.url}/api'
-        self.url = self.url.split('https://')[0].split('http://')[0]
+        if self.url_api is not None:
+            l.warning('config.url_api is deprecated and will be removed in 2.7.0. Please use config.url.')
+            self.url = self.url_api
 
         match self.user_agent:
             case UserAgent.DEFAULT:
@@ -206,14 +209,14 @@ class Config:
 
 
 class AccessToken(BaseModel):
-    roles: list[Role]
+    roles: list[Role] = [Role.USER]
     session_id: UUID = Field(alias='sid')
-    is_active: bool = Field(alias='isActive')
+    is_active: bool = Field(True, alias='isActive')
     subject_id: UUID = Field(alias='sub')
     issued_at: datetime = Field(alias='iat')
-    issuer: str = Field(alias='iss')  # "auth-service"
+    issuer: str | None = Field(None, alias='iss')  # "auth-service"
     expired_at: datetime = Field(alias='exp')
-    jwt_id: UUID = Field(alias='jti')
+    jwt_id: UUID | None = Field(None, alias='jti')
 
     @field_validator('issued_at', 'expired_at', mode='plain')
     @classmethod
@@ -242,9 +245,9 @@ class Client:
             self.access_token = access.replace('Bearer ', '')
 
         if refresh:
+            self.session.cookies.set(config.refresh_token_cookie_name, refresh, path='/')
             self.auth_level = AuthLevel.REFRESH
             self.refresh_token = refresh
-            self.session.cookies.set('refresh_token', refresh, path='/', domain=self.config.url)
             # if access is None:
             #     self.refresh_auth()
 
@@ -352,7 +355,7 @@ class Client:
         res = refresh_token(self)
         res.raise_for_status()
 
-        self.access_token = res.json()['accessToken']
+        self.access_token = res.json().get('accessToken') or res.json()['token']
         self.access_token_data = AccessToken.model_validate(decode_jwt_payload(self.access_token))
 
         assert self.access_token

{itd_sdk-2.5.1 → itd_sdk-2.5.2}/itd/comment.py

@@ -41,9 +41,9 @@ class Comment(ITDBaseModel):
     _base_comment: Comment | None = None
 
     @classmethod
-    def from_dict(cls, data: dict, post: Post | None = None, base_comment: Comment | None = None, *, client: Client | None = None) -> Comment:  # ty: ignore[invalid-method-override]
+    def from_dict(cls, data: dict, post: Post | None = None, base_comment: Comment | None = None, *, client: Client | None = None) -> Comment:
         instance = super().from_dict(data, client=client)
-        if post:
+        if post is not None:
             instance._post = post
         instance._base_comment = base_comment
         instance.replies._post = post

{itd_sdk-2.5.1 → itd_sdk-2.5.2}/itd/exceptions.py

@@ -97,6 +97,11 @@ class InvalidAccessTokenError(AuthError):
     json_check = staticmethod(lambda json: json.get('error') == 'invalid signature')
 
 
+class JWTAlgorithmUnsupportedError(AuthError):
+    json_check = staticmethod(lambda json: json.get('error') == 'Unsupported token algorithm')
+    text = 'Access token JWT algorithm unsupported'
+
+
 class SessionRevokedError(AuthError):
     code = 'SESSION_REVOKED'
     text = 'Session revoked (logged out)'
@@ -104,7 +109,7 @@ class SessionRevokedError(AuthError):
 
 class AccessTokenExpiredError(AuthError):
     text = 'Token expired'
-    json_check = staticmethod(lambda json: json.get('error') == 'token expired')
+    json_check = staticmethod(lambda json: json.get('error') == 'token expired' or json.get('message') == 'Invalid or expired token')
 
 
 class LoginError(AuthError):
@@ -345,6 +350,7 @@ DEFAULT_ERRORS = (
     RateLimitError(),
     InvalidAccessTokenError(),
     UnauthorizedError(),
+    JWTAlgorithmUnsupportedError(),
     AccessTokenExpiredError(),
     AccountBannedError(),
     AccountTemporarilyBannedError(),

{itd_sdk-2.5.1 → itd_sdk-2.5.2}/itd/notification.py

@@ -168,12 +168,12 @@ class Notifications(ITDList[Notification]):
 
         l.info('stop stream')
 
-    def stream_bg(self) -> Thread:
+    def stream_bg(self, daemon: bool = False) -> Thread:
         def _stream():
             for _ in self.stream():
                 continue
 
-        thread = Thread(target=_stream)
+        thread = Thread(target=_stream, daemon=daemon)
         thread.start()
         return thread
 

{itd_sdk-2.5.1 → itd_sdk-2.5.2}/itd/request.py

@@ -29,7 +29,7 @@ def _get_jhash(b: int) -> int:
     return k
 
 
-def _solve_ddos_guard(session: Session, response: Response, domain: str = 'xn--d1ah4a.com', user_agent: str = '') -> bool:
+def _solve_ddos_guard(session: Session, response: Response, user_agent: str = '') -> bool:
     """Solve DDoS-Guard JS challenge. Returns True if solved (duplicate request required)."""
     if '<html>' not in response.text[:500] or 'get_jhash' not in response.text:
         return False
@@ -45,8 +45,8 @@ def _solve_ddos_guard(session: Session, response: Response, domain: str = 'xn--d
     jhash = _get_jhash(code)
     l.info('solved jhash=%s', jhash)
 
-    session.cookies.set('__jhash_', str(jhash), domain=domain, path='/')
-    session.cookies.set('__jua_', quote(user_agent, safe=''), domain=domain, path='/')
+    session.cookies.set('__jhash_', str(jhash), path='/')
+    session.cookies.set('__jua_', quote(user_agent, safe=''), path='/')
 
     return True
 
@@ -97,11 +97,11 @@ def fetch(client: 'Client', method: str, url: str, params: dict = {}, files: dic
     def _do_request():
         m = method.lower()
         if m == "get":
-            return client.session.get(f'{client.config._url_api}/{url}', timeout=client.config.timeout, params=params, headers=headers)
+            return client.session.get(f'{client.config.url}/{url}', timeout=client.config.timeout, params=params, headers=headers)
 
         return client.session.request(
             m.upper(),
-            f'{client.config._url_api}/{url}',
+            f'{client.config.url}/{url}',
             timeout=client.config.timeout_file if files else client.config.timeout,
             json=params,
             headers=headers,
@@ -111,7 +111,7 @@ def fetch(client: 'Client', method: str, url: str, params: dict = {}, files: dic
     res = _do_request()
     if client.config.solve_challenge:
         for _ in range(3):
-            if not _solve_ddos_guard(client.session, res, client.config.url, client.config._user_agent):
+            if not _solve_ddos_guard(client.session, res, client.config._user_agent):
                 break
             l.debug('ddos-guard cookies: %s', {c.name: c.value for c in client.session.cookies if c.name.startswith('__')})
             res = _do_request()

{itd_sdk-2.5.1 → itd_sdk-2.5.2}/pyproject.toml

@@ -14,7 +14,7 @@ version_provider = "uv"
 
 [project]
 name = "itd-sdk"
-version = "2.5.1"
+version = "2.5.2"
 description = "ITD client for python"
 readme = "README.md"
 authors = [

itd_sdk-2.5.1/itd/_limiter.py

@@ -1,107 +0,0 @@
-from time import monotonic, sleep
-
-from itd.logger import get_logger
-
-l = get_logger('limiter')
-
-
-class RateLimiter:
-    window: int = 65
-
-    def __init__(self, name: str, capacity: int):
-        l.debug(r'\[%s] create rate limiter limit=%s', name, capacity)
-        self.name = name
-        self.capacity = capacity
-
-    def sync(self, remaining: int): ...
-    def acquire(self): ...
-
-
-class SafeRateLimiter(RateLimiter):
-    def __init__(self, name: str, capacity: int):
-        super().__init__(name, capacity)
-        self.refill_rate = 1.5
-        self.requests = 0
-        self.remainings = []
-
-    def acquire(self):
-        l.debug(r'\[%s] acquire limiter delay=%s', self.name, round(self.refill_rate, 2))
-        sleep(self.refill_rate)
-        self.requests += 1
-
-    def sync(self, remaining: int):
-        l.debug(r'\[%s] sync limiter remaining=%s', self.name, remaining)
-        if self.requests > 50:
-            return
-        if self.requests == 50:
-            l.info(r'\[%s] stop learning final=%s', self.name, round(self.refill_rate, 2))
-            return
-
-        self.remainings.append(remaining or -5)
-        if len(self.remainings) > 5:
-            if self.remainings[-5] > remaining + 2:
-                value = self.refill_rate * (1 + (self.remainings[-5] - remaining) / 8)
-                l.info(
-                    r'\[%s] increase refill rate %s -> %s (remaining %s -> %s)',
-                    self.name,
-                    round(self.refill_rate, 2),
-                    round(value, 2),
-                    self.remainings[-5],
-                    remaining
-                )
-                self.refill_rate = value
-                self.remainings.clear()
-            elif self.remainings[-5] < remaining - 2:
-                value = self.refill_rate * (1 - (self.remainings[-5] - remaining) / 8)
-                l.info(
-                    r'\[%s] decrease refill rate %s -> %s (remaining %s -> %s)',
-                    self.name,
-                    round(self.refill_rate, 2),
-                    round(value, 2),
-                    self.remainings[-5],
-                    remaining
-                )
-                self.refill_rate = value
-                self.remainings.clear()
-            else:
-                l.debug(r'\[%s] refill rate is stable', self.name)
-
-
-class BurstRateLimiter(RateLimiter):
-    def __init__(self, name: str, capacity: int):
-        super().__init__(name, capacity)
-        self.requests = []
-
-    def sync(self, remaining: int):
-        l.debug(r'\[%s] sync limiter remaining=%s', self.name, remaining)
-
-        used = self.capacity - remaining
-        if used > 0:
-            self.requests = self.requests[-used:]  # ai
-        while len(self.requests) < used:
-            self.requests.append(monotonic())
-
-    def acquire(self):
-        l.debug(r'\[%s] acquire limiter', self.name)
-
-        self.requests = [request for request in self.requests if request > monotonic() - self.window]
-        while len(self.requests) >= self.capacity:
-            self.requests = [request for request in self.requests if request > monotonic() - self.window]
-            if self.requests:
-                sleep(max(self.requests[0] + self.window - monotonic(), 0.1))
-
-
-class IPRateLimiter:
-    window: int = 60
-    max_requests: int = 100
-
-    def __init__(self):
-        self.requests = []
-
-    def acquire(self):
-        self.requests.append(monotonic())
-        self.requests = [request for request in self.requests if request > monotonic() - self.window]
-        while len(self.requests) >= self.max_requests:
-            self.requests = [request for request in self.requests if request > monotonic() - self.window]
-            if self.requests:
-                sleep(max(self.requests[0] + self.window - monotonic(), 0.1))