jesse 3.2.2__tar.gz → 3.2.4__tar.gz

{jesse-3.2.2/jesse.egg-info → jesse-3.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: jesse
-Version: 3.2.2
+Version: 3.2.4
 Summary: A trading framework for cryptocurrencies
 Home-page: https://jesse.trade
 Author: Saleh Mir

{jesse-3.2.2 → jesse-3.2.4}/jesse/modes/import_candles_mode/__init__.py

@@ -8,6 +8,7 @@ import pydash
 
 import jesse.helpers as jh
 from jesse.exceptions import CandleNotFoundInExchange
+from jesse.enums import live_session_modes
 from jesse.models.Candle import Candle
 from jesse.modes.import_candles_mode.drivers import (
     build_historical_provider_registry,
@@ -204,6 +205,35 @@ def _report_import_progress(
         _print_import_progressbar(exchange, symbol, percent, remaining_seconds, reached_date)
 
 
+def _missing_live_candle_ranges(
+    exchange: str, symbol: str, start: int, end: int, max_candles: int,
+) -> list[tuple[int, int]]:
+    """Cover absent live minutes with provider-sized pages, preserving sparse observations."""
+    ranges = []
+    cursor = start
+    for candle in candle_repository.fetch_candles_from_db(exchange, symbol, '1m', start, end - 60_000):
+        timestamp = int(candle[0])
+        if timestamp > cursor:
+            ranges.append((cursor, timestamp))
+        cursor = timestamp + 60_000
+    if cursor < end:
+        ranges.append((cursor, end))
+
+    # Thousands of isolated no-trade minutes must not cause thousands of requests
+    # on every restart. Re-reading stored rows is safe because inserts retain them.
+    page_span = max_candles * 60_000
+    pages = []
+    for missing_start, missing_end in ranges:
+        if pages and missing_start <= pages[-1][1]:
+            page_start, page_end = pages.pop()
+            missing_end = max(missing_end, page_end)
+        else:
+            page_start = missing_start
+        page_count = (missing_end - page_start + page_span - 1) // page_span
+        pages.append((page_start, min(end, page_start + page_count * page_span)))
+    return pages
+
+
 def run(
         client_id: str,
         exchange: str,
@@ -270,8 +300,10 @@ def _run(
     from jesse.services.db import database
     database.open_connection()
 
-    end_timestamp = arrow.utcnow().floor('day').int_timestamp * 1000
     provider = build_historical_provider_registry((exchange,)).get(exchange)
+    # Crypto startup needs today's closed bars. Provider ranges are half-open, so the
+    # current minute is excluded; traditional historical providers retain their daily cutoff.
+    end_timestamp = arrow.utcnow().floor('minute' if isinstance(provider, CandleExchange) else 'day').int_timestamp * 1000
     max_candles = provider.capabilities.max_candles_per_request
     if max_candles is None:
         raise ValueError(f'Historical provider {exchange!r} does not declare a request range limit')
@@ -310,6 +342,14 @@ def _run(
         latest_timestamp,
         interval,
     )
+    if isinstance(provider, CandleExchange) and mode in (
+        live_session_modes.LIVETRADE, live_session_modes.PAPERTRADE,
+    ):
+        # Stored endpoints do not prove live warm-up coverage: WebSocket writes may
+        # bracket a downtime gap. Historical imports still preserve intentional sparse gaps.
+        import_ranges = _missing_live_candle_ranges(
+            exchange, symbol, effective_start_timestamp, end_timestamp, max_candles,
+        )
     total_span = end_timestamp - effective_start_timestamp
     work_span = sum(range_end - range_start for range_start, range_end in import_ranges)
     completed_span = total_span - work_span

{jesse-3.2.2 → jesse-3.2.4}/jesse/modes/import_candles_mode/drivers/Bybit/BybitMain.py

@@ -1,10 +1,16 @@
 import requests
+import time
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 import jesse.helpers as jh
 from jesse.modes.import_candles_mode.drivers.interface import CandleExchange
 from typing import Union
 from jesse import exceptions
+from jesse.services.historical_data.errors import (
+    ProviderRateLimitError,
+    ProviderRequestError,
+    ProviderUnavailableError,
+)
 from .bybit_utils import timeframe_to_interval
 
 
@@ -27,6 +33,50 @@ class BybitMain(CandleExchange):
         )
         self.session.mount('https://', HTTPAdapter(max_retries=retries, pool_maxsize=100))
 
+    def _get_kline_data(self, payload: dict) -> list:
+        """Retry transient JSON failures that HTTP-level retries cannot see."""
+        # Match the HTTP adapter's three-retry budget. Bybit's 10006 limit uses a
+        # rolling one-second window; exponential waits avoid immediately hitting it again.
+        for attempt in range(4):
+            response = self.session.get(self.endpoint + '/v5/market/kline', params=payload, timeout=10)
+            self.validate_response(response)
+            data = response.json()
+            code = data.get('retCode', 0)
+            message = f"{payload['symbol']} on {self.name}: {data['retMsg']} (retCode={code})"
+            # Bybit documents 10000 as a server timeout and 10016 as a server error.
+            if code in (10006, 10000, 10016):
+                error_type = ProviderRateLimitError if code == 10006 else ProviderUnavailableError
+                if attempt == 3:
+                    raise error_type(message)
+                delay = float(2 ** attempt)
+                reset = response.headers.get('X-Bapi-Limit-Reset-Timestamp') if code == 10006 else None
+                if reset:
+                    try:
+                        delay = max(delay, float(reset) / 1000 - time.time())
+                    except ValueError:
+                        pass  # A malformed reset header must not disable the bounded fallback.
+                # Do not block startup indefinitely on a bad or unusually distant reset timestamp.
+                if delay > 30:
+                    raise ProviderRateLimitError(
+                        f"Bybit rate limit for {payload['symbol']} on {self.name}; retry after its limit resets."
+                    )
+                time.sleep(delay)
+                continue
+            if code != 0 or data['retMsg'] != 'OK':
+                # 10001 also covers unrelated parameter errors; only symbol-specific
+                # messages (or the explicit invalid-symbol code) mean a missing market.
+                invalid_symbol_message = data['retMsg'].lower() in ('symbol invalid', 'invalid symbol')
+                if (
+                    code == 10029
+                    or (code == 10001 and 'symbol' in data['retMsg'].lower())
+                    # Retain explicit invalid-symbol errors even when the response omits its code.
+                    or ('retCode' not in data and invalid_symbol_message)
+                ):
+                    raise exceptions.SymbolNotFound(message)
+                raise ProviderRequestError(message)
+            return data['result']['list']
+        raise ProviderRateLimitError(f'Bybit rate limit on {self.name}')
+
     def get_starting_time(self, symbol: str) -> Union[int, None]:
         dashless_symbol = jh.dashless_symbol(symbol)
         # Bybit answers `start` in ascending order, so one 1m candle from 2018 is the exact
@@ -39,9 +89,7 @@ class BybitMain(CandleExchange):
             'start': 1514811660000
         }
 
-        response = self.session.get(self.endpoint + '/v5/market/kline', params=payload, timeout=10)
-        self.validate_response(response)
-        data = response.json()['result']['list']
+        data = self._get_kline_data(payload)
         if not data:
             return None
         return int(data[0][0])
@@ -57,11 +105,7 @@ class BybitMain(CandleExchange):
             'limit': self.count
         }
 
-        response = self.session.get(self.endpoint + '/v5/market/kline', params=payload, timeout=10)
-
-        if response.json()['retMsg'] != 'OK':
-            raise exceptions.SymbolNotFound(response.json()['retMsg'])
-        data = response.json()['result']['list']
+        data = self._get_kline_data(payload)
         # Reverse the data list
         data = data[::-1]
 

{jesse-3.2.2 → jesse-3.2.4}/jesse/modes/import_candles_mode/drivers/interface.py

@@ -164,6 +164,21 @@ class CandleExchange(HistoricalCandleProvider, ABC):
         if response.status_code == 404:
             raise ValueError(f'ERROR {response.status_code} {response.reason}. Check the symbol')
 
+        # 429: request-weight limit hit; 418: the IP kept sending after 429s and is temporarily banned
+        # (Binance and Binance-style APIs). Every process on the same IP counts toward the limit.
+        if response.status_code in (418, 429):
+            retry_after = (getattr(response, 'headers', None) or {}).get('Retry-After')
+            wait = f' The exchange asks to wait {retry_after} seconds.' if retry_after else ''
+            if response.status_code == 418:
+                explanation = ('the exchange has temporarily banned this IP address for sending too many requests '
+                               'after rate-limit warnings. The ban lifts on its own.')
+            else:
+                explanation = 'the request rate limit for this IP address was reached.'
+            raise ConnectionError(
+                f'ERROR {response.status_code} {response.reason}: {explanation}{wait} Reduce how often this IP '
+                f'calls the exchange - other bots or scripts on the same IP count too.'
+            )
+
         # if the response code is not in the 200-299, raise an exception
         if response.status_code // 100 != 2:
             raise ConnectionError(f'ERROR {response.status_code} {response.reason}')

{jesse-3.2.2 → jesse-3.2.4}/jesse/services/candle_service.py

@@ -574,6 +574,9 @@ def add_candle(
     # initial
     if last_index == -1:
         arr.append(candle)
+        # A session with no warm-up still needs its first observed forming bucket.
+        if with_generation and timeframe == '1m':
+            _generate_bigger_timeframes(candle, exchange, symbol, with_execution)
         return
 
     # read the last candle's timestamp once as a plain scalar instead of
@@ -690,43 +693,17 @@ def _generate_bigger_timeframes(candle: np.ndarray, exchange: str, symbol: str,
         if timeframe == '1m':
             continue
 
-        last_candle = get_current_candle(exchange, symbol, timeframe)
-        generate_from_count = int((candle[0] - last_candle[0]) / 60_000)
-        number_of_candles = len(get_candles(exchange, symbol, '1m'))
-        short_candles = get_candles(exchange, symbol, '1m')[-1 - generate_from_count:]
-
-        if generate_from_count == -1:
-            # it's receiving an slightly older candle than the last one. Ignore it
-            return
-
-        if generate_from_count < 0:
-            current_1m = get_current_candle(exchange, symbol, '1m')
-            raise ValueError(
-                f'generate_from_count cannot be negative! '
-                f'generate_from_count:{generate_from_count}, candle[0]:{candle[0]}, '
-                f'last_candle[0]:{last_candle[0]}, current_1m:{current_1m[0]}, number_of_candles:{number_of_candles}')
-
-        if len(short_candles) == 0:
-            raise ValueError(
-                f'No candles were passed. More info:'
-                f'\nexchange:{exchange}, symbol:{symbol}, timeframe:{timeframe}, generate_from_count:{generate_from_count}'
-                f'\nlast_candle\'s timestamp: {last_candle[0]}'
-                f'\ncurrent timestamp: {jh.now()}'
-            )
-
-        # update latest candle
-        generated_candle = generate_candle_from_one_minutes(
-            timeframe,
-            short_candles,
-            accept_forming_candles=True
-        )
-
-        # Fix: force the generated candle's timestamp to the correct period-aligned boundary.
-        # Without this, when a live session starts mid-period (e.g. at 10:36 on an hourly
-        # timeframe), the 1m store only has that one candle, so generate_candle_from_one_minutes
-        # stamps the resulting 1h candle as 10:36 instead of the correct period start 10:00.
+        # Elapsed minutes are not row counts when an exchange omits no-trade bars.
+        # Select only observations in this candle's UTC bucket, including forming rows.
         timeframe_ms = jh.timeframe_to_one_minutes(timeframe) * 60_000
-        generated_candle[0] = candle[0] - (candle[0] % timeframe_ms)
+        bucket_start = int(candle[0]) - int(candle[0]) % timeframe_ms
+        short_candles = get_candles(exchange, symbol, '1m')
+        start_index = int(np.searchsorted(short_candles[:, 0], bucket_start, side='left'))
+        end_index = int(np.searchsorted(short_candles[:, 0], bucket_start + timeframe_ms, side='left'))
+        source_rows = short_candles[start_index:end_index]
+        if not len(source_rows):
+            continue
+        generated_candle = generate_candle_from_observed_minutes(timeframe, source_rows)
 
         add_candle(
             generated_candle, exchange, symbol, timeframe, with_execution, with_generation=False
@@ -847,6 +824,13 @@ def get_current_candle(exchange: str, symbol: str, timeframe: str) -> np.ndarray
         else:
             return arr[-1]
 
+    if jh.is_live():
+        # Live stores already hold native exchange bars or clock-aligned locally
+        # generated bars. Reconstructing by 1m row count would corrupt sparse history
+        # and could override native bars when local generation is disabled.
+        arr = store.candles.get_storage(exchange, symbol, timeframe)
+        return arr[-1] if len(arr) else np.zeros((0, 6))
+
     if store.candles.uses_timestamp_buckets:
         candles = _get_timestamp_bucket_candles(exchange, symbol, timeframe)
         return candles[-1] if len(candles) else np.zeros((0, 6))

jesse-3.2.4/jesse/version.py

@@ -0,0 +1 @@
+__version__ = "3.2.4"

{jesse-3.2.2 → jesse-3.2.4/jesse.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: jesse
-Version: 3.2.2
+Version: 3.2.4
 Summary: A trading framework for cryptocurrencies
 Home-page: https://jesse.trade
 Author: Saleh Mir

{jesse-3.2.2 → jesse-3.2.4}/setup.py

@@ -1,12 +1,22 @@
+import re
 from pathlib import Path
 
 from setuptools import setup, find_packages
 
-# also change in version.py
-VERSION = "3.2.2"
 DESCRIPTION = "A trading framework for cryptocurrencies"
 BASE_DIR = Path(__file__).resolve().parent
 
+# The version lives only in jesse/version.py. It is read as text because importing the package
+# would import Jesse and its dependencies during the build.
+_version_match = re.search(
+    r"^__version__\s*=\s*['\"]([^'\"]+)['\"]",
+    (BASE_DIR / "jesse" / "version.py").read_text(encoding="utf-8"),
+    re.MULTILINE,
+)
+if not _version_match:
+    raise RuntimeError("Could not find __version__ in jesse/version.py")
+VERSION = _version_match.group(1)
+
 with open(BASE_DIR / "requirements.txt", "r", encoding="utf-8") as f:
     REQUIRED_PACKAGES = f.read().splitlines()
 

jesse-3.2.2/jesse/version.py

@@ -1 +0,0 @@
-__version__ = "3.2.2"