dccd 2.3.0__tar.gz → 2.3.1__tar.gz

{dccd-2.3.0 → dccd-2.3.1}/CHANGELOG.md

@@ -6,6 +6,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.3.1] - 2026-05-24
+
+### Fixed
+
+- `dccd/storage.py` — `DataStore.missing_intervals` now detects the gap **before** the first saved row when the requested `start` predates `file_min`; previously only the trailing gap (after `file_max`) was returned, causing `dccd backfill --start <early-date>` to silently skip all historical data before the first existing candle (#46)
+- `dccd/histo_dl/coinbase.py` — raise `RuntimeError` when Coinbase returns HTTP 200 with a JSON dict (e.g. `{"message": "..."}` for near-future windows) instead of silently iterating dict keys and crashing with `ValueError` (#45)
+- `dccd/histo_dl/coinbase.py` — additional guard: raise `RuntimeError` when Coinbase returns a JSON list whose first element is not itself a list/tuple (e.g. `["message"]`); previously caused `float("m")` `ValueError` (#45)
+- `dccd/histo_dl/exchange.py` — `_sort_data` no longer raises `KeyError: 'TS'` when the API returns empty data; returns early with an empty `self.df` so the backfill skips the window cleanly (#45)
+- `dccd/histo_dl/exchange.py` — `_sort_data` strips any candle at or beyond `self.end` before merging; exchanges with inclusive endpoint semantics (Coinbase) no longer cause `_advance` to overshoot by one span per window, preventing drift that accumulated into near-future requests (#45)
+- `dccd/histo_dl/okx.py` — raise `RuntimeError` when OKX response code is not `"0"`, letting the backfill retry/skip logic handle API-level errors (#45)
+- `dccd/histo_dl/okx.py` — switch `_import_data` from `/market/candles` to `/market/history-candles`; the former only serves the last ~24 h of 1-minute bars and silently returns empty data for older windows (#45)
+
 ## [2.3.0] - 2026-05-22
 
 ### Added

{dccd-2.3.0 → dccd-2.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dccd
-Version: 2.3.0
+Version: 2.3.1
 Summary: Download Crypto Currency Data from different exchanges.
 Author-email: Arthur Bernard <arthur.bernard.92@gmail.com>
 License: MIT

{dccd-2.3.0 → dccd-2.3.1}/dccd/histo_dl/coinbase.py

@@ -26,7 +26,6 @@ from typing import Any
 from dccd.histo_dl.exchange import ImportDataCryptoCurrencies
 
 # Import local packages
-from dccd.tools.date_time import TS_to_date
 
 __all__ = ['FromCoinbase']
 
@@ -116,8 +115,8 @@ class FromCoinbase(ImportDataCryptoCurrencies):
     def _import_data(self, start: int | str = 'last', end: int | str = 'now') -> list[dict[str, Any]]:
         self.start, self.end = self._set_time(start, end)
         param = {
-            'start': TS_to_date(self.start - self.span),
-            'end': TS_to_date(self.end),
+            'start': datetime.fromtimestamp(self.start, tz=timezone.utc).isoformat(),
+            'end':   datetime.fromtimestamp(self.end,   tz=timezone.utc).isoformat(),
             'granularity': self.span,
         }
         r = self._fetch(
@@ -127,6 +126,10 @@ class FromCoinbase(ImportDataCryptoCurrencies):
             param,
         )
         text = r.json()
+        if not isinstance(text, list):
+            raise RuntimeError(f"Coinbase API error: {text!r:.200}")
+        if text and not isinstance(text[0], (list, tuple)):
+            raise RuntimeError(f"Coinbase unexpected response: {text[:3]!r:.200}")
         data = [{
             'date': float(e[0]),
             'open': float(e[3]),

{dccd-2.3.0 → dccd-2.3.1}/dccd/histo_dl/exchange.py

@@ -128,10 +128,9 @@ class ImportDataCryptoCurrencies(ABC):
            wait=wait_exponential(multiplier=1, min=1, max=60),
            stop=stop_after_attempt(5))
     def _fetch(self, url: str, params: dict[str, Any]) -> requests.Response:
-        """ Fetch URL with automatic retry on HTTP 429. """
+        """ Fetch URL and raise on any HTTP error. """
         r = requests.get(url, params)
-        if r.status_code == 429:
-            r.raise_for_status()
+        r.raise_for_status()
         return r
 
     def _get_last_date(self) -> int:
@@ -228,6 +227,17 @@ class ImportDataCryptoCurrencies(ABC):
         """
         data = [OHLCBar(**d).model_dump(exclude_none=False) for d in data]
         df = pd.DataFrame(data).rename(columns={'date': 'TS'})
+        if df.empty or 'TS' not in df.columns:
+            self.df = df
+            return self
+        # Discard any candle at or beyond self.end: those belong to the next
+        # window.  Without this filter, exchanges that return the endpoint
+        # candle (inclusive API boundary) would cause _advance to overshoot
+        # by one span, and the drift compounds over many windows.
+        # Guard: only filter when self.end is set (> 0); direct callers that
+        # bypass _set_time leave self.end = 0, so we skip the filter.
+        if self.end > 0:
+            df = df[df['TS'] < self.end]
         # Use self.end as the exclusive grid boundary so the full window is
         # covered even when the last trade arrives >span seconds before the
         # window end.  Callers must set self.end to the correct window
@@ -236,6 +246,9 @@ class ImportDataCryptoCurrencies(ABC):
             list(range(self.start, self.end, self.span)),
             columns=['TS']
         )
+        if df.empty or 'TS' not in df.columns:
+            self.df = df
+            return self
         df = (df.merge(TS, on='TS', how='outer', sort=False)
               .sort_values('TS')
               .reset_index(drop=True)

{dccd-2.3.0 → dccd-2.3.1}/dccd/histo_dl/okx.py

@@ -145,8 +145,11 @@ class FromOKX(ImportDataCryptoCurrencies):
             'limit': 300,
         }
 
-        r = self._fetch('https://www.okx.com/api/v5/market/candles', param)
-        text = r.json()['data']
+        r = self._fetch('https://www.okx.com/api/v5/market/history-candles', param)
+        body = r.json()
+        if body.get('code', '0') != '0':
+            raise RuntimeError(f"OKX API error {body['code']}: {body.get('msg', body)}")
+        text = body['data']
         text.reverse()
 
         data = [{

{dccd-2.3.0 → dccd-2.3.1}/dccd/storage.py

@@ -319,18 +319,23 @@ class DataStore:
 
             file_path = self.directory / f'{year}.parquet'
 
-            if year < current_year and file_path.exists():
-                if self.is_period_complete(year):
+            if file_path.exists():
+                if year < current_year and self.is_period_complete(year):
                     continue  # full year already on disk — skip
-                # incomplete past year: resume from last saved row
-                df = pd.read_parquet(file_path, columns=['TS'])
-                if not df.empty:
-                    ivl_start = int(df['TS'].max()) + self.span
-            elif file_path.exists():
-                # current year: always extend from the last saved row
                 df = pd.read_parquet(file_path, columns=['TS'])
                 if not df.empty:
-                    ivl_start = int(df['TS'].max()) + self.span
+                    file_min = int(df['TS'].min())
+                    file_max = int(df['TS'].max())
+                    # Gap before the first saved row (e.g. backfill requested
+                    # from an earlier date than the file's first candle)
+                    if ivl_start < file_min:
+                        intervals.append((ivl_start, file_min))
+                    # Trailing gap after the last saved row
+                    trailing = file_max + self.span
+                    if trailing < ivl_end:
+                        intervals.append((trailing, ivl_end))
+                    continue
+                # file exists but is empty: fall through to full-interval append
 
             if ivl_start < ivl_end:
                 intervals.append((ivl_start, ivl_end))

{dccd-2.3.0 → dccd-2.3.1}/dccd/tests/test_coinbase.py

@@ -47,7 +47,7 @@ def test_malformed_response_raises(loader, monkeypatch):
     m.status_code = 200
     m.json.return_value = {"error": "bad"}
     monkeypatch.setattr("requests.get", lambda *a, **kw: m)
-    with pytest.raises((TypeError, ValueError)):
+    with pytest.raises((TypeError, ValueError, RuntimeError)):
         loader._import_data(start=0)
 
 

{dccd-2.3.0 → dccd-2.3.1}/dccd/tests/test_storage.py

@@ -336,6 +336,52 @@ def test_missing_intervals_already_up_to_date(tmp_path):
     assert intervals == []
 
 
+def test_missing_intervals_start_before_file_min_current_year(tmp_path):
+    """File starts mid-year; requested start is before file_min → beginning gap + trailing gap."""
+    from datetime import datetime, timezone
+    span = 3600
+    store = DataStore(str(tmp_path), 'binance', 'BTC/USDT', span, 'ohlc')
+    current_year = datetime.now(tz=timezone.utc).year
+    year_start = int(datetime(current_year, 1, 1, tzinfo=timezone.utc).timestamp())
+    # File covers mid-May onwards (simulates the user's real situation)
+    file_start = year_start + 100 * span
+    file_end   = year_start + 110 * span
+    store.save(_ohlc_df(list(range(file_start, file_end + span, span))))
+    end_ts = file_end + 50 * span
+
+    intervals = store.missing_intervals(year_start, end_ts)
+
+    # Expect two intervals: [year_start, file_start) and [file_end+span, end_ts)
+    assert len(intervals) == 2
+    assert intervals[0] == (year_start, file_start)
+    assert intervals[1] == (file_end + span, end_ts)
+
+
+def test_missing_intervals_start_before_file_min_past_year(tmp_path):
+    """Past incomplete year where file starts after requested start → beginning gap returned."""
+    span = 3600
+    store = DataStore(str(tmp_path), 'binance', 'BTC/USDT', span, 'ohlc')
+    file_start = _Y2023 + 100 * span
+    file_end   = _Y2023 + 200 * span
+    store.save(_ohlc_df(list(range(file_start, file_end + span, span))))
+
+    intervals = store.missing_intervals(_Y2023, _Y2024)
+
+    assert len(intervals) == 2
+    assert intervals[0] == (_Y2023, file_start)
+    assert intervals[1] == (file_end + span, _Y2024)
+
+
+def test_missing_intervals_file_already_covers_full_request(tmp_path):
+    """File already covers [start, end] exactly → no intervals."""
+    span = 3600
+    store = DataStore(str(tmp_path), 'binance', 'BTC/USDT', span, 'ohlc')
+    store.save(_ohlc_df(list(range(_Y2023, _Y2023 + 10 * span + span, span))))
+    end_ts = _Y2023 + 5 * span
+    intervals = store.missing_intervals(_Y2023, end_ts)
+    assert intervals == []
+
+
 def test_missing_intervals_non_ohlc_simple_resume(tmp_path):
     store = DataStore(str(tmp_path), 'binance', 'BTC/USDT', None, 'trades')
     store.save(_trades_df([1672531200]))

{dccd-2.3.0 → dccd-2.3.1}/dccd.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dccd
-Version: 2.3.0
+Version: 2.3.1
 Summary: Download Crypto Currency Data from different exchanges.
 Author-email: Arthur Bernard <arthur.bernard.92@gmail.com>
 License: MIT

{dccd-2.3.0 → dccd-2.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dccd"
-version = "2.3.0"
+version = "2.3.1"
 description = "Download Crypto Currency Data from different exchanges."
 readme = "README.rst"
 license = { text = "MIT" }