opencode-pyneruntime 6.6.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- opencode_pyneruntime-6.6.4.dist-info/METADATA +281 -0
- opencode_pyneruntime-6.6.4.dist-info/RECORD +261 -0
- opencode_pyneruntime-6.6.4.dist-info/WHEEL +5 -0
- opencode_pyneruntime-6.6.4.dist-info/entry_points.txt +6 -0
- opencode_pyneruntime-6.6.4.dist-info/licenses/LICENSE +201 -0
- opencode_pyneruntime-6.6.4.dist-info/licenses/NOTICE +21 -0
- opencode_pyneruntime-6.6.4.dist-info/top_level.txt +1 -0
- pynecore/__init__.py +6 -0
- pynecore/cli/__init__.py +2 -0
- pynecore/cli/app.py +238 -0
- pynecore/cli/commands/__init__.py +343 -0
- pynecore/cli/commands/benchmark.py +186 -0
- pynecore/cli/commands/compile.py +198 -0
- pynecore/cli/commands/data.py +857 -0
- pynecore/cli/commands/debug.py +63 -0
- pynecore/cli/commands/optimize.py +956 -0
- pynecore/cli/commands/plugin.py +242 -0
- pynecore/cli/commands/run.py +2006 -0
- pynecore/cli/pluggable.py +132 -0
- pynecore/cli/utils/__init__.py +0 -0
- pynecore/cli/utils/api_error_handler.py +168 -0
- pynecore/cli/utils/broker_picker.py +330 -0
- pynecore/cli/utils/error_hook.py +28 -0
- pynecore/cli/utils/keyreader.py +178 -0
- pynecore/cli/utils/provider_picker.py +19 -0
- pynecore/cli/utils/symbol_browser.py +1149 -0
- pynecore/core/__init__.py +0 -0
- pynecore/core/aggregator.py +257 -0
- pynecore/core/bar_magnifier.py +168 -0
- pynecore/core/broker/__init__.py +64 -0
- pynecore/core/broker/defaults.py +113 -0
- pynecore/core/broker/disappearance.py +927 -0
- pynecore/core/broker/emulator.py +345 -0
- pynecore/core/broker/exceptions.py +346 -0
- pynecore/core/broker/idempotency.py +401 -0
- pynecore/core/broker/intent_builder.py +334 -0
- pynecore/core/broker/journal.py +1785 -0
- pynecore/core/broker/models.py +1600 -0
- pynecore/core/broker/native_failsafe_manager.py +1436 -0
- pynecore/core/broker/one_way_emulator.py +1128 -0
- pynecore/core/broker/position.py +787 -0
- pynecore/core/broker/run_identity.py +126 -0
- pynecore/core/broker/software_entry_stop_engine.py +351 -0
- pynecore/core/broker/software_partial_bracket_engine.py +1379 -0
- pynecore/core/broker/spot_inventory.py +1327 -0
- pynecore/core/broker/storage.py +2655 -0
- pynecore/core/broker/store_helpers.py +2161 -0
- pynecore/core/broker/sync_engine.py +16070 -0
- pynecore/core/broker/validation.py +382 -0
- pynecore/core/class_property.py +7 -0
- pynecore/core/config.py +392 -0
- pynecore/core/csv_file.py +547 -0
- pynecore/core/currency.py +262 -0
- pynecore/core/data_converter.py +1002 -0
- pynecore/core/datetime.py +296 -0
- pynecore/core/download_info.py +71 -0
- pynecore/core/download_runner.py +274 -0
- pynecore/core/htf_aggregator.py +181 -0
- pynecore/core/import_hook.py +358 -0
- pynecore/core/instance_state.py +494 -0
- pynecore/core/live_ltf_collector.py +442 -0
- pynecore/core/live_ltf_window.py +189 -0
- pynecore/core/live_runner.py +1347 -0
- pynecore/core/module_property.py +26 -0
- pynecore/core/ohlcv_file.py +1888 -0
- pynecore/core/overload.py +371 -0
- pynecore/core/pine_cast.py +113 -0
- pynecore/core/pine_export.py +95 -0
- pynecore/core/pine_method.py +244 -0
- pynecore/core/pine_range.py +86 -0
- pynecore/core/pine_udt.py +69 -0
- pynecore/core/plugin/__init__.py +394 -0
- pynecore/core/plugin/broker.py +781 -0
- pynecore/core/plugin/cli.py +96 -0
- pynecore/core/plugin/live_provider.py +208 -0
- pynecore/core/plugin/provider.py +331 -0
- pynecore/core/provider_string.py +148 -0
- pynecore/core/random.py +40 -0
- pynecore/core/resampler.py +686 -0
- pynecore/core/safe_convert.py +64 -0
- pynecore/core/script.py +1011 -0
- pynecore/core/script_runner.py +3202 -0
- pynecore/core/security.py +1749 -0
- pynecore/core/security_process.py +1253 -0
- pynecore/core/security_shm.py +456 -0
- pynecore/core/series.py +417 -0
- pynecore/core/strategy_stats.py +669 -0
- pynecore/core/symbol_map.py +134 -0
- pynecore/core/syminfo.py +505 -0
- pynecore/core/viz.py +591 -0
- pynecore/lib/__init__.py +1771 -0
- pynecore/lib/_fixnan.py +32 -0
- pynecore/lib/_math_stateful.py +202 -0
- pynecore/lib/_timeframe_change.py +101 -0
- pynecore/lib/adjustment.py +6 -0
- pynecore/lib/alert.py +39 -0
- pynecore/lib/alert.pyi +14 -0
- pynecore/lib/array.py +1051 -0
- pynecore/lib/barmerge.py +60 -0
- pynecore/lib/barstate.py +30 -0
- pynecore/lib/box.py +415 -0
- pynecore/lib/chart.py +128 -0
- pynecore/lib/color.py +152 -0
- pynecore/lib/color.pyi +50 -0
- pynecore/lib/currency.py +62 -0
- pynecore/lib/dayofweek.py +36 -0
- pynecore/lib/dayofweek.pyi +18 -0
- pynecore/lib/display.py +8 -0
- pynecore/lib/dividends.py +9 -0
- pynecore/lib/earnings.py +11 -0
- pynecore/lib/extend.py +6 -0
- pynecore/lib/font.py +5 -0
- pynecore/lib/footprint.py +79 -0
- pynecore/lib/format.py +11 -0
- pynecore/lib/hline.py +67 -0
- pynecore/lib/hline.pyi +24 -0
- pynecore/lib/label.py +409 -0
- pynecore/lib/line.py +433 -0
- pynecore/lib/linefill.py +93 -0
- pynecore/lib/location.py +11 -0
- pynecore/lib/log.py +362 -0
- pynecore/lib/map.py +150 -0
- pynecore/lib/math.py +385 -0
- pynecore/lib/matrix.py +708 -0
- pynecore/lib/order.py +8 -0
- pynecore/lib/pivotpointtype.py +8 -0
- pynecore/lib/plot.py +95 -0
- pynecore/lib/plot.pyi +33 -0
- pynecore/lib/polyline.py +91 -0
- pynecore/lib/position.py +15 -0
- pynecore/lib/request.py +281 -0
- pynecore/lib/runtime.py +5 -0
- pynecore/lib/scale.py +9 -0
- pynecore/lib/session.py +267 -0
- pynecore/lib/session.pyi +12 -0
- pynecore/lib/shape.py +18 -0
- pynecore/lib/size.py +12 -0
- pynecore/lib/splits.py +4 -0
- pynecore/lib/strategy/__init__.py +4778 -0
- pynecore/lib/strategy/closedtrades.py +347 -0
- pynecore/lib/strategy/closedtrades.pyi +53 -0
- pynecore/lib/strategy/commission.py +9 -0
- pynecore/lib/strategy/direction.py +9 -0
- pynecore/lib/strategy/oca.py +13 -0
- pynecore/lib/strategy/opentrades.py +281 -0
- pynecore/lib/strategy/opentrades.pyi +49 -0
- pynecore/lib/strategy/risk.py +109 -0
- pynecore/lib/string.py +649 -0
- pynecore/lib/syminfo.py +84 -0
- pynecore/lib/ta.py +2230 -0
- pynecore/lib/table.py +290 -0
- pynecore/lib/text.py +17 -0
- pynecore/lib/ticker.py +207 -0
- pynecore/lib/timeframe.py +293 -0
- pynecore/lib/volume_row.py +67 -0
- pynecore/lib/xloc.py +4 -0
- pynecore/lib/yloc.py +5 -0
- pynecore/providers/__init__.py +0 -0
- pynecore/providers/ccxt.py +664 -0
- pynecore/providers/replay.py +187 -0
- pynecore/pynesys/__init__.py +0 -0
- pynecore/pynesys/api.py +498 -0
- pynecore/pynesys/compiler.py +112 -0
- pynecore/standalone.py +99 -0
- pynecore/testing/__init__.py +1 -0
- pynecore/testing/broker_lab/__init__.py +41 -0
- pynecore/testing/broker_lab/__main__.py +5 -0
- pynecore/testing/broker_lab/cli.py +87 -0
- pynecore/testing/broker_lab/generate.py +47 -0
- pynecore/testing/broker_lab/model.py +84 -0
- pynecore/testing/broker_lab/reference.py +645 -0
- pynecore/testing/broker_lab/runner.py +372 -0
- pynecore/testing/broker_lab/scheduler.py +50 -0
- pynecore/testing/broker_lab/subprocess.py +73 -0
- pynecore/transformers/__init__.py +0 -0
- pynecore/transformers/builtin_shadow.py +136 -0
- pynecore/transformers/closure_arguments_transformer.py +428 -0
- pynecore/transformers/display_rewrite.py +140 -0
- pynecore/transformers/dynamic_default.py +147 -0
- pynecore/transformers/function_isolation.py +757 -0
- pynecore/transformers/import_lifter.py +61 -0
- pynecore/transformers/import_normalizer.py +328 -0
- pynecore/transformers/inline_series_hoist.py +178 -0
- pynecore/transformers/input_transformer.py +175 -0
- pynecore/transformers/lib_series.py +201 -0
- pynecore/transformers/locations.py +70 -0
- pynecore/transformers/module_properties.json +3387 -0
- pynecore/transformers/module_property.py +221 -0
- pynecore/transformers/ne_guard.py +70 -0
- pynecore/transformers/persistent.py +320 -0
- pynecore/transformers/persistent_series.py +76 -0
- pynecore/transformers/safe_convert_transformer.py +97 -0
- pynecore/transformers/safe_division_transformer.py +95 -0
- pynecore/transformers/script_requirements.py +308 -0
- pynecore/transformers/security.py +752 -0
- pynecore/transformers/security_instantiation.py +274 -0
- pynecore/transformers/series.py +275 -0
- pynecore/transformers/slot_layout.py +381 -0
- pynecore/transformers/type_checking_stripper.py +25 -0
- pynecore/transformers/unused_series_detector.py +267 -0
- pynecore/types/__init__.py +21 -0
- pynecore/types/alert.py +5 -0
- pynecore/types/barmerge.py +5 -0
- pynecore/types/base.py +39 -0
- pynecore/types/box.py +37 -0
- pynecore/types/chart.py +17 -0
- pynecore/types/color.py +107 -0
- pynecore/types/currency.py +5 -0
- pynecore/types/datetime.py +6 -0
- pynecore/types/display.py +5 -0
- pynecore/types/dividends.py +5 -0
- pynecore/types/earnings.py +5 -0
- pynecore/types/extend.py +5 -0
- pynecore/types/font.py +5 -0
- pynecore/types/footprint.py +41 -0
- pynecore/types/format.py +5 -0
- pynecore/types/hline.py +24 -0
- pynecore/types/ib_persistent.py +8 -0
- pynecore/types/ib_persistent.pyi +10 -0
- pynecore/types/label.py +35 -0
- pynecore/types/line.py +32 -0
- pynecore/types/linefill.py +13 -0
- pynecore/types/location.py +5 -0
- pynecore/types/matrix.py +999 -0
- pynecore/types/na.py +237 -0
- pynecore/types/na.pyi +83 -0
- pynecore/types/ohlcv.py +12 -0
- pynecore/types/order.py +5 -0
- pynecore/types/persistent.py +8 -0
- pynecore/types/persistent.pyi +13 -0
- pynecore/types/pine_types.py +11 -0
- pynecore/types/pine_types.pyi +15 -0
- pynecore/types/pivotpointtype.py +5 -0
- pynecore/types/plot.py +12 -0
- pynecore/types/plot_meta.py +60 -0
- pynecore/types/polyline.py +40 -0
- pynecore/types/position.py +5 -0
- pynecore/types/scale.py +5 -0
- pynecore/types/script_type.py +15 -0
- pynecore/types/series.py +23 -0
- pynecore/types/series.pyi +19 -0
- pynecore/types/session.py +35 -0
- pynecore/types/shape.py +5 -0
- pynecore/types/size.py +5 -0
- pynecore/types/source.py +33 -0
- pynecore/types/splits.py +5 -0
- pynecore/types/strategy.py +45 -0
- pynecore/types/table.py +87 -0
- pynecore/types/text.py +13 -0
- pynecore/types/type_checker.py +7 -0
- pynecore/types/type_checker.pyi +48 -0
- pynecore/types/volume_row.py +36 -0
- pynecore/types/weekdays.py +11 -0
- pynecore/types/xloc.py +5 -0
- pynecore/types/yloc.py +5 -0
- pynecore/utils/__init__.py +0 -0
- pynecore/utils/file_utils.py +50 -0
- pynecore/utils/rich/__init__.py +0 -0
- pynecore/utils/rich/date_column.py +25 -0
- pynecore/utils/sequence_view.py +92 -0
- pynecore/utils/stdlib_checker.py +17 -0
|
@@ -0,0 +1,1347 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Async/sync bridge for live data streaming.
|
|
3
|
+
|
|
4
|
+
Runs a LiveProviderPlugin's async watch_ohlcv() in a background thread
|
|
5
|
+
and yields OHLCV objects to the synchronous ScriptRunner via queue.Queue.
|
|
6
|
+
|
|
7
|
+
Serial startup: ``live_ohlcv_generator()`` starts the background thread
|
|
8
|
+
**and blocks until ``provider.connect()`` succeeds** before returning.
|
|
9
|
+
This guarantees that by the time the caller starts consuming the warmup
|
|
10
|
+
(local OHLCV file), the WS subscription is already active — any bar that
|
|
11
|
+
closes while warmup is running goes into ``bar_queue`` and cannot be
|
|
12
|
+
lost. A parallel-start design is appealing on paper but in practice
|
|
13
|
+
``provider.connect()`` (REST session + activity-cursor recovery + WS
|
|
14
|
+
handshake + subscribe) is slow enough that warmup of a local file
|
|
15
|
+
finishes first; the resulting gap means the very next bar close on the
|
|
16
|
+
exchange happens with no listener, and that bar is gone forever.
|
|
17
|
+
|
|
18
|
+
Catch-up: when the consumer first pulls from the live iterator (right
|
|
19
|
+
after the file iterator has been exhausted), any closed bars already
|
|
20
|
+
sitting in the queue are drained as additional warmup (the script_runner
|
|
21
|
+
historical loop processes them with ``barstate.ishistory=True`` and the
|
|
22
|
+
strategy still suppressed). Intra-bar updates queued during warmup are
|
|
23
|
+
dropped — they would otherwise inflate ``bar_index`` against a still-open
|
|
24
|
+
bar that the historical loop is not equipped to dedup. Once the queue
|
|
25
|
+
empties, ``LIVE_TRANSITION`` is yielded inline; the script_runner flips
|
|
26
|
+
to live mode and every subsequent bar runs against an unsuppressed
|
|
27
|
+
strategy.
|
|
28
|
+
|
|
29
|
+
Dedup: the duplicate-filter for ``last_historical_timestamp`` uses
|
|
30
|
+
**strict** less-than for both closed bars and intra-bar updates. The
|
|
31
|
+
plugin contract is that ``download_ohlcv`` returns fully-closed bars
|
|
32
|
+
only (the Capital.com plugin, for instance, drops the still-forming
|
|
33
|
+
last bar in its REST response), so under normal operation the
|
|
34
|
+
``ts == last_historical`` case does not arise. The strict-less filter
|
|
35
|
+
is kept as defense-in-depth: if a provider violates the contract or a
|
|
36
|
+
race serves an in-progress bar from REST, the script_runner live loop
|
|
37
|
+
treats the same-timestamp first live update as a continuation of the
|
|
38
|
+
last warmup bar — it seeds ``last_bar_timestamp`` from
|
|
39
|
+
``last_warmup_timestamp``, so ``bar_index`` does not double-bump and
|
|
40
|
+
the bar simply gets one more execution with the refined OHLC values.
|
|
41
|
+
"""
|
|
42
|
+
import asyncio
|
|
43
|
+
import logging
|
|
44
|
+
import time
|
|
45
|
+
import threading
|
|
46
|
+
from collections.abc import Coroutine, Generator
|
|
47
|
+
from datetime import datetime
|
|
48
|
+
from queue import Queue, Empty, Full
|
|
49
|
+
from typing import Any
|
|
50
|
+
from zoneinfo import ZoneInfo
|
|
51
|
+
|
|
52
|
+
from pynecore.core.syminfo import SymInfo
|
|
53
|
+
from pynecore.types.ohlcv import OHLCV
|
|
54
|
+
from pynecore.core.plugin import is_retryable_provider_error
|
|
55
|
+
from pynecore.core.plugin.live_provider import LiveProviderPlugin
|
|
56
|
+
from pynecore.core.script_runner import LIVE_TRANSITION
|
|
57
|
+
from pynecore.lib.log import broker_info, broker_warning
|
|
58
|
+
from pynecore.lib.session import _is_in_session, _is_point_in_session
|
|
59
|
+
from pynecore.lib.timeframe import in_seconds
|
|
60
|
+
|
|
61
|
+
__all__ = ['live_ohlcv_generator', 'download_warmup_in_memory', 'LiveBarStreamer']
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class LiveBarStreamer:
|
|
65
|
+
"""Non-blocking, thread-safe closed-bar source for security subprocesses.
|
|
66
|
+
|
|
67
|
+
Wraps :func:`live_ohlcv_generator` in a background thread that drains its
|
|
68
|
+
output into an internal queue. The security subprocess polls
|
|
69
|
+
:meth:`pop_new_closed_bars` once per chart advance — receiving zero or
|
|
70
|
+
more freshly-closed bars without ever blocking the chart-driven flow.
|
|
71
|
+
|
|
72
|
+
Closed bars go to the queue; the latest still-forming (intra-bar) bar is
|
|
73
|
+
kept in a single slot read via :meth:`peek_developing_bar` (overwritten in
|
|
74
|
+
place, never queued). Cross-symbol :func:`request.security` ignores the
|
|
75
|
+
forming slot — it exposes closed bars only — while live
|
|
76
|
+
:func:`request.security_lower_tf` carries the forming bar as the developing
|
|
77
|
+
last element of its intrabar window. The warmup→live transition sentinel is
|
|
78
|
+
dropped (the subprocess does not switch modes mid-run).
|
|
79
|
+
"""
|
|
80
|
+
|
|
81
|
+
def __init__(self, provider: LiveProviderPlugin, symbol: str, timeframe: str,
|
|
82
|
+
*, syminfo: SymInfo | None = None,
|
|
83
|
+
last_historical_timestamp: int | None = None):
|
|
84
|
+
self._provider = provider
|
|
85
|
+
self._symbol = symbol
|
|
86
|
+
self._timeframe = timeframe
|
|
87
|
+
self._syminfo = syminfo
|
|
88
|
+
self._last_historical_timestamp = last_historical_timestamp
|
|
89
|
+
self._queue: Queue[OHLCV] = Queue()
|
|
90
|
+
# Latest still-forming bar, overwritten in place by the drain thread and
|
|
91
|
+
# read by the subprocess via ``peek_developing_bar``. A single reference
|
|
92
|
+
# assignment/read is atomic under the GIL, so no lock is needed.
|
|
93
|
+
self._developing: OHLCV | None = None
|
|
94
|
+
self._stopped = threading.Event()
|
|
95
|
+
self._gen: Generator[OHLCV, None, None] | None = None
|
|
96
|
+
self._thread: threading.Thread | None = None
|
|
97
|
+
# Captures an exception raised by the upstream generator so the
|
|
98
|
+
# next ``pop_new_closed_bars()`` call can surface it to the
|
|
99
|
+
# security subprocess. Without this, a dead WS would just stop
|
|
100
|
+
# delivering bars and the security loop would silently emit ``na``
|
|
101
|
+
# forever while the chart-side liveness check never notices.
|
|
102
|
+
self._drain_error: BaseException | None = None
|
|
103
|
+
|
|
104
|
+
def start(self) -> None:
|
|
105
|
+
"""Start the background drain thread."""
|
|
106
|
+
if self._thread is not None:
|
|
107
|
+
return
|
|
108
|
+
self._gen = live_ohlcv_generator(
|
|
109
|
+
provider=self._provider,
|
|
110
|
+
symbol=self._symbol,
|
|
111
|
+
timeframe=self._timeframe,
|
|
112
|
+
syminfo=self._syminfo,
|
|
113
|
+
last_historical_timestamp=self._last_historical_timestamp,
|
|
114
|
+
)
|
|
115
|
+
self._thread = threading.Thread(
|
|
116
|
+
target=self._drain, daemon=True, name=f"sec-stream-{self._symbol}",
|
|
117
|
+
)
|
|
118
|
+
self._thread.start()
|
|
119
|
+
|
|
120
|
+
def _drain(self) -> None:
|
|
121
|
+
assert self._gen is not None
|
|
122
|
+
try:
|
|
123
|
+
for bar in self._gen:
|
|
124
|
+
if self._stopped.is_set():
|
|
125
|
+
break
|
|
126
|
+
if bar is LIVE_TRANSITION:
|
|
127
|
+
continue
|
|
128
|
+
if getattr(bar, 'is_closed', True):
|
|
129
|
+
# A close supersedes the forming snapshot for its slot, but
|
|
130
|
+
# only when it is not older than the tracked forming bar.
|
|
131
|
+
# Providers that close a slot on the next slot's timestamp
|
|
132
|
+
# (cTrader) — or simple stream reordering — can deliver the
|
|
133
|
+
# forming tick of slot N+1 ahead of the late close of slot N;
|
|
134
|
+
# an unconditional clear would erase the newer forming
|
|
135
|
+
# snapshot and make ``peek_developing_bar`` read ``None``
|
|
136
|
+
# until the next tick. Timestamp-guarded exactly like the
|
|
137
|
+
# generator's ``last_forming_bar`` handling.
|
|
138
|
+
dev = self._developing
|
|
139
|
+
if dev is None or bar.timestamp >= dev.timestamp:
|
|
140
|
+
self._developing = None
|
|
141
|
+
self._queue.put(bar)
|
|
142
|
+
else:
|
|
143
|
+
# Latest forming (intra-bar) snapshot for the open slot.
|
|
144
|
+
self._developing = bar
|
|
145
|
+
except Exception as exc: # noqa: BLE001
|
|
146
|
+
logger.exception("LiveBarStreamer drain raised")
|
|
147
|
+
if not self._stopped.is_set():
|
|
148
|
+
self._drain_error = exc
|
|
149
|
+
|
|
150
|
+
def pop_new_closed_bars(self) -> list[OHLCV]:
|
|
151
|
+
"""Drain all currently-available closed bars (non-blocking).
|
|
152
|
+
|
|
153
|
+
Re-raises any exception captured by the upstream drain thread once
|
|
154
|
+
the queue is empty, so the security subprocess can fail loudly and
|
|
155
|
+
the chart-side liveness check (``proc.is_alive()`` polling) catches
|
|
156
|
+
the dead process instead of silently turning every bar into ``na``.
|
|
157
|
+
"""
|
|
158
|
+
out: list[OHLCV] = []
|
|
159
|
+
while True:
|
|
160
|
+
try:
|
|
161
|
+
out.append(self._queue.get_nowait())
|
|
162
|
+
except Empty:
|
|
163
|
+
break
|
|
164
|
+
if not out and self._drain_error is not None:
|
|
165
|
+
err = self._drain_error
|
|
166
|
+
# Single-shot raise so a re-tried call (e.g. after restart)
|
|
167
|
+
# would not keep raising forever; ``_drain_error`` stays set
|
|
168
|
+
# only as a flag for ``stop()`` cleanup.
|
|
169
|
+
self._drain_error = None
|
|
170
|
+
raise err
|
|
171
|
+
return out
|
|
172
|
+
|
|
173
|
+
def peek_developing_bar(self) -> 'OHLCV | None':
|
|
174
|
+
"""Return the latest still-forming bar without consuming it.
|
|
175
|
+
|
|
176
|
+
The forming bar is never queued — the drain thread overwrites this slot
|
|
177
|
+
in place as new intra-bar ticks arrive and clears it when the bar
|
|
178
|
+
closes. Live ``request.security_lower_tf`` reads it each chart tick to
|
|
179
|
+
carry the developing intrabar as the live last element of its window;
|
|
180
|
+
the eventual closed bar (delivered via :meth:`pop_new_closed_bars`)
|
|
181
|
+
finalizes that tail.
|
|
182
|
+
"""
|
|
183
|
+
return self._developing
|
|
184
|
+
|
|
185
|
+
def wait_for_bars(self, timeout: float) -> list[OHLCV]:
|
|
186
|
+
"""Block up to ``timeout`` seconds for at least one closed bar.
|
|
187
|
+
|
|
188
|
+
Used by the cross-symbol live HTF security loop when the chart
|
|
189
|
+
process has signaled a confirmed period whose bar has not yet been
|
|
190
|
+
published by the upstream WS feed (different exchange / network
|
|
191
|
+
delay). Blocking briefly on the streamer queue avoids advancing the
|
|
192
|
+
chart's ``last_confirmed`` watermark past the security's actual bar
|
|
193
|
+
and emitting ``na`` for that period.
|
|
194
|
+
|
|
195
|
+
Drains any further bars that have already accumulated after the
|
|
196
|
+
first one arrives so a single call still returns the full batch.
|
|
197
|
+
Re-raises a captured drain error using the same single-shot
|
|
198
|
+
semantics as :meth:`pop_new_closed_bars`.
|
|
199
|
+
"""
|
|
200
|
+
out: list[OHLCV] = []
|
|
201
|
+
try:
|
|
202
|
+
first = self._queue.get(timeout=max(0.0, timeout))
|
|
203
|
+
except Empty:
|
|
204
|
+
if self._drain_error is not None:
|
|
205
|
+
err = self._drain_error
|
|
206
|
+
self._drain_error = None
|
|
207
|
+
raise err
|
|
208
|
+
return out
|
|
209
|
+
out.append(first)
|
|
210
|
+
while True:
|
|
211
|
+
try:
|
|
212
|
+
out.append(self._queue.get_nowait())
|
|
213
|
+
except Empty:
|
|
214
|
+
break
|
|
215
|
+
return out
|
|
216
|
+
|
|
217
|
+
def stop(self) -> None:
|
|
218
|
+
"""Signal the drain thread to exit and close the upstream generator.
|
|
219
|
+
|
|
220
|
+
The drain thread is typically parked inside ``next(self._gen)`` at
|
|
221
|
+
this point. Calling ``self._gen.close()`` from a *different* thread
|
|
222
|
+
while the generator is suspended on a ``yield`` is supported, but
|
|
223
|
+
if the runtime considers the generator to be executing on the drain
|
|
224
|
+
side (race with the moment a bar is being yielded), CPython raises
|
|
225
|
+
``ValueError: generator already executing``. The drain thread will
|
|
226
|
+
eventually return on its own once ``self._stopped`` is observed, so
|
|
227
|
+
swallow the race here and rely on the ``join`` below to finish
|
|
228
|
+
cleanup.
|
|
229
|
+
"""
|
|
230
|
+
self._stopped.set()
|
|
231
|
+
if self._gen is not None:
|
|
232
|
+
try:
|
|
233
|
+
self._gen.close()
|
|
234
|
+
except (RuntimeError, ValueError, GeneratorExit):
|
|
235
|
+
pass
|
|
236
|
+
if self._thread is not None:
|
|
237
|
+
self._thread.join(timeout=5.0)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def download_warmup_in_memory(
|
|
241
|
+
provider: LiveProviderPlugin,
|
|
242
|
+
time_from: datetime,
|
|
243
|
+
time_to: datetime,
|
|
244
|
+
) -> list[OHLCV]:
|
|
245
|
+
"""
|
|
246
|
+
Download historical OHLCV data into an in-memory list (no file written).
|
|
247
|
+
|
|
248
|
+
Used by the security subprocess to fetch warmup bars without creating
|
|
249
|
+
any ``.ohlcv`` file. Captures the records by temporarily redirecting
|
|
250
|
+
:meth:`ProviderPlugin.save_ohlcv_data` to an internal list — the
|
|
251
|
+
plugin's own ``download_ohlcv`` implementation is otherwise unchanged.
|
|
252
|
+
|
|
253
|
+
:param provider: A live provider instance (``ohlcv_dir`` may be ``None``).
|
|
254
|
+
:param time_from: Naive UTC start of the warmup window.
|
|
255
|
+
:param time_to: Naive UTC end of the warmup window.
|
|
256
|
+
:return: Fully-closed warmup bars in chronological order.
|
|
257
|
+
"""
|
|
258
|
+
captured: list[OHLCV] = []
|
|
259
|
+
|
|
260
|
+
original_save = provider.save_ohlcv_data
|
|
261
|
+
|
|
262
|
+
def _capture(data):
|
|
263
|
+
if isinstance(data, OHLCV):
|
|
264
|
+
captured.append(data)
|
|
265
|
+
else:
|
|
266
|
+
captured.extend(data)
|
|
267
|
+
|
|
268
|
+
provider.save_ohlcv_data = _capture # type: ignore[method-assign]
|
|
269
|
+
try:
|
|
270
|
+
tf = time_from.replace(tzinfo=None) if time_from.tzinfo else time_from
|
|
271
|
+
tt = time_to.replace(tzinfo=None) if time_to.tzinfo else time_to
|
|
272
|
+
provider.download_ohlcv(tf, tt)
|
|
273
|
+
finally:
|
|
274
|
+
provider.save_ohlcv_data = original_save # type: ignore[method-assign]
|
|
275
|
+
|
|
276
|
+
return captured
|
|
277
|
+
|
|
278
|
+
logger = logging.getLogger(__name__)
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
class _Sentinel(BaseException):
|
|
282
|
+
"""Marker signaling end of the live stream."""
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
_SENTINEL = _Sentinel()
|
|
286
|
+
|
|
287
|
+
# Soft cap for intra-bar updates queued ahead of the consumer. Closed
|
|
288
|
+
# bars are never dropped (they go through the blocking ``put`` path),
|
|
289
|
+
# but intra-bar updates are advisory and must not accumulate without
|
|
290
|
+
# bound when the consumer falls behind — otherwise stale ticks would
|
|
291
|
+
# sit ahead of newer closed bars and delay live transition.
|
|
292
|
+
_INTRA_BAR_SOFT_CAP = 32
|
|
293
|
+
|
|
294
|
+
# Sleep cadence during known-closed windows. Long enough to keep the
|
|
295
|
+
# TimeoutError handler from spinning at ``effective_timeout`` (~50ms)
|
|
296
|
+
# while the boundary deadline is stale, short enough that the next
|
|
297
|
+
# session open is noticed within ~30s. Exposed at module scope so tests
|
|
298
|
+
# can shrink it without spending the full 30s per gated timeout.
|
|
299
|
+
_CLOSED_WINDOW_SLEEP_S = 30.0
|
|
300
|
+
|
|
301
|
+
# Floor for the feed-staleness threshold (``feed_timeout_bars`` periods,
|
|
302
|
+
# but never less than this). Keeps tiny timeframes from flapping into a
|
|
303
|
+
# reconnect on a few quiet seconds. Exposed at module scope so tests can
|
|
304
|
+
# shrink it.
|
|
305
|
+
_FEED_STALE_FLOOR_S = 90.0
|
|
306
|
+
|
|
307
|
+
# Cadence of WARNING-level idle-synth reminders within one idle streak:
|
|
308
|
+
# the first synth of a streak warns, then every Nth; the rest are DEBUG.
|
|
309
|
+
_SYNTH_WARN_EVERY = 10
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def _warn_this_attempt(attempts: int) -> bool:
|
|
313
|
+
"""Reconnect-log rate policy: WARNING for the first attempts of an
|
|
314
|
+
outage, then one WARNING per ten attempts (the rest log at DEBUG).
|
|
315
|
+
With the backoff saturated at ``max_reconnect_delay`` (default 60 s)
|
|
316
|
+
this works out to roughly one console line every ten minutes during
|
|
317
|
+
a long outage instead of one per attempt.
|
|
318
|
+
"""
|
|
319
|
+
return attempts <= 3 or attempts % 10 == 0
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def _is_transient_connect_error(exc: BaseException) -> bool:
|
|
323
|
+
"""Whether a failed initial ``provider.connect()`` is worth retrying.
|
|
324
|
+
|
|
325
|
+
Extends the provider-error classification (:func:`is_retryable_provider_error`)
|
|
326
|
+
to the raw socket/TLS layer: a transient ``OSError``-derived fault — most
|
|
327
|
+
notably ``ConnectionResetError`` ([Errno 54]) raised straight out of
|
|
328
|
+
``asyncio.open_connection`` during the TLS handshake — is a network blip, not
|
|
329
|
+
a user-actionable misconfiguration, so the startup connect should ride it out
|
|
330
|
+
on the backoff path rather than die before the handshake. The ``__cause__`` /
|
|
331
|
+
``__context__`` chain is walked so a transient still classifies after being
|
|
332
|
+
re-wrapped. Permanent failures (bad symbol / credentials / account mode)
|
|
333
|
+
surface as a non-retryable :class:`ProviderError` and correctly return
|
|
334
|
+
``False`` here so they keep failing fast.
|
|
335
|
+
|
|
336
|
+
:param exc: The exception raised by ``provider.connect()``.
|
|
337
|
+
:return: ``True`` if a retry could plausibly succeed.
|
|
338
|
+
"""
|
|
339
|
+
if is_retryable_provider_error(exc):
|
|
340
|
+
return True
|
|
341
|
+
seen: set[int] = set()
|
|
342
|
+
current: BaseException | None = exc
|
|
343
|
+
while current is not None and id(current) not in seen:
|
|
344
|
+
seen.add(id(current))
|
|
345
|
+
if isinstance(current, OSError):
|
|
346
|
+
return True
|
|
347
|
+
current = current.__cause__ or current.__context__
|
|
348
|
+
return False
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def live_ohlcv_generator(
|
|
352
|
+
provider: LiveProviderPlugin,
|
|
353
|
+
symbol: str,
|
|
354
|
+
timeframe: str,
|
|
355
|
+
syminfo: SymInfo | None = None,
|
|
356
|
+
*,
|
|
357
|
+
last_historical_timestamp: int | None = None,
|
|
358
|
+
shutdown_timeout: float = 120.0,
|
|
359
|
+
event_loop: asyncio.AbstractEventLoop | None = None,
|
|
360
|
+
engine_event_stream: Coroutine[Any, Any, Any] | None = None,
|
|
361
|
+
raise_on_connect_failure: bool = False,
|
|
362
|
+
) -> Generator[OHLCV, None, None]:
|
|
363
|
+
"""
|
|
364
|
+
Bridge async watch_ohlcv() to a sync Generator[OHLCV, None, None].
|
|
365
|
+
|
|
366
|
+
Spawns a background thread running asyncio, collects OHLCV objects
|
|
367
|
+
via queue.Queue, and yields them including intra-bar updates.
|
|
368
|
+
|
|
369
|
+
The background thread is started **eagerly** at call time, not on
|
|
370
|
+
first ``next()`` — so the WS subscription is open during warmup and
|
|
371
|
+
no bar is lost in the gap between the REST historical download
|
|
372
|
+
finishing and the consumer reaching the first live update.
|
|
373
|
+
|
|
374
|
+
The first batch of bars yielded after the consumer starts pulling is
|
|
375
|
+
the catch-up: closed bars that landed in the queue while the local
|
|
376
|
+
warmup loop was running. ``LIVE_TRANSITION`` is yielded inline once
|
|
377
|
+
the queue empties, so the script_runner can flip to live mode at the
|
|
378
|
+
correct point regardless of how long warmup took.
|
|
379
|
+
|
|
380
|
+
:param provider: A LiveProviderPlugin instance (already configured).
|
|
381
|
+
:param symbol: Symbol in provider-specific format.
|
|
382
|
+
:param timeframe: Timeframe in TradingView format.
|
|
383
|
+
:param syminfo: Optional symbol metadata used to gate idle-bar
|
|
384
|
+
synthesis and reconnect attempts on the trading-
|
|
385
|
+
session calendar. ``None`` (the default) or an
|
|
386
|
+
empty ``opening_hours`` preserves the legacy 24/7
|
|
387
|
+
behaviour where idle synth and reconnect fire on
|
|
388
|
+
every timeout.
|
|
389
|
+
:param last_historical_timestamp: Timestamp of the last historical bar to avoid duplicates.
|
|
390
|
+
:param shutdown_timeout: Max seconds to wait for graceful shutdown. 0 = wait forever.
|
|
391
|
+
:param event_loop: Optional externally-owned event loop. When supplied, the background
|
|
392
|
+
thread runs the async loop on it via ``run_until_complete`` instead
|
|
393
|
+
of ``asyncio.run``. Required for broker mode so that the Order Sync
|
|
394
|
+
Engine can submit coroutines to the same loop.
|
|
395
|
+
:param engine_event_stream: Optional coroutine (typically
|
|
396
|
+
``OrderSyncEngine.run_event_stream()``) to run as a
|
|
397
|
+
long-lived task alongside the OHLCV watcher. The engine
|
|
398
|
+
receives its :class:`OrderEvent` stream this way.
|
|
399
|
+
:param raise_on_connect_failure: When True, a ``provider.connect()`` that
|
|
400
|
+
fails fast during warmup is re-raised here, from
|
|
401
|
+
the construction call, instead of being buffered
|
|
402
|
+
for the first bar pull. Broker mode sets this so
|
|
403
|
+
the real connect error surfaces before
|
|
404
|
+
``start_broker()`` can mask it with a generic
|
|
405
|
+
"live connection not established" reconcile
|
|
406
|
+
failure. Data-only callers (and the security
|
|
407
|
+
``LiveBarStreamer``) leave it False and keep the
|
|
408
|
+
surface-through-the-iterator behaviour.
|
|
409
|
+
:return: Iterator yielding OHLCV objects (both closed and intra-bar) interleaved
|
|
410
|
+
with a single ``LIVE_TRANSITION`` sentinel marking the warmup→live boundary.
|
|
411
|
+
"""
|
|
412
|
+
# Unbounded on the closed-bar path: a bounded queue would block
|
|
413
|
+
# ``bar_queue.put`` when the consumer (``script_runner`` live path)
|
|
414
|
+
# falls behind during heavy fill processing. Since ``_async_loop``
|
|
415
|
+
# runs as an asyncio task on the same event loop as ``_listen_loop``
|
|
416
|
+
# (broker mode pumps both via ``run_coroutine_threadsafe`` onto the
|
|
417
|
+
# main loop), a sync ``Queue.put`` parking on ``not_full`` would
|
|
418
|
+
# stall the entire event loop — listener stops draining the WS,
|
|
419
|
+
# quote ticks pile up in the kernel buffer, ``_tick_volume`` stops
|
|
420
|
+
# advancing, and the watchdog ends up synthesising V=0 bars on a
|
|
421
|
+
# market that is actually trading. Intra-bar updates are bounded
|
|
422
|
+
# by a soft ``qsize`` cap at the put site (see
|
|
423
|
+
# ``_INTRA_BAR_SOFT_CAP`` below) so advisory ticks cannot pile up
|
|
424
|
+
# ahead of closed bars when the consumer lags. The closed-bar
|
|
425
|
+
# ``broker_warning`` in the put path below flags any consumer lag.
|
|
426
|
+
bar_queue: Queue[OHLCV | BaseException] = Queue()
|
|
427
|
+
stop_event = threading.Event()
|
|
428
|
+
# Loop-side mirror of ``stop_event`` so an ``await`` running on the broker
|
|
429
|
+
# loop (the reconnect ``connect()`` / ``on_reconnect()`` handshake) can be
|
|
430
|
+
# abandoned the instant a shutdown is requested from the consumer thread.
|
|
431
|
+
# ``stop_event`` is a ``threading.Event`` — awaiting it from the loop would
|
|
432
|
+
# need a poll; the ``asyncio.Event`` here is *set* on the loop via
|
|
433
|
+
# ``call_soon_threadsafe`` (see ``_consumer``), keeping teardown event-driven
|
|
434
|
+
# and bounded even when a reconnect is in flight. Populated by ``_async_loop``
|
|
435
|
+
# with its running loop + event once it starts.
|
|
436
|
+
shutdown_signal: dict[str, Any] = {}
|
|
437
|
+
# Signalled by ``_async_loop`` once ``provider.connect()`` has either
|
|
438
|
+
# succeeded (WS subscribed, ready to receive bars) or failed (with
|
|
439
|
+
# the exception already pushed into ``bar_queue``). Used to make
|
|
440
|
+
# ``live_ohlcv_generator`` block until the WS is up before returning
|
|
441
|
+
# — see module docstring for why warmup must follow connect.
|
|
442
|
+
connected_event = threading.Event()
|
|
443
|
+
connect_timeout_seconds = 30.0
|
|
444
|
+
# Holds the exception from a fast-failing ``provider.connect()``. Appended
|
|
445
|
+
# before ``connected_event`` is set (so the construction-site read below
|
|
446
|
+
# synchronises on the event and observes it), letting the caller re-raise
|
|
447
|
+
# the real cause instead of returning a generator whose buffered error
|
|
448
|
+
# only surfaces on the first pull — too late, by which point
|
|
449
|
+
# ``start_broker()`` has masked it. See ``raise_on_connect_failure``.
|
|
450
|
+
connect_failure: list[BaseException] = []
|
|
451
|
+
|
|
452
|
+
async def _graceful_shutdown():
|
|
453
|
+
"""Poll can_shutdown(), then disconnect. Respects shutdown_timeout."""
|
|
454
|
+
logger.info("Graceful shutdown started, polling can_shutdown()...")
|
|
455
|
+
|
|
456
|
+
if shutdown_timeout > 0:
|
|
457
|
+
deadline = time.monotonic() + shutdown_timeout
|
|
458
|
+
else:
|
|
459
|
+
deadline = None
|
|
460
|
+
|
|
461
|
+
while True:
|
|
462
|
+
try:
|
|
463
|
+
if await provider.can_shutdown():
|
|
464
|
+
logger.info("Provider ready to shut down")
|
|
465
|
+
break
|
|
466
|
+
except Exception as e:
|
|
467
|
+
logger.warning("can_shutdown() raised: %s", e)
|
|
468
|
+
break
|
|
469
|
+
|
|
470
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
471
|
+
logger.warning("Shutdown timeout (%.0fs reached), forcing disconnect",
|
|
472
|
+
shutdown_timeout)
|
|
473
|
+
break
|
|
474
|
+
|
|
475
|
+
await asyncio.sleep(1.0)
|
|
476
|
+
|
|
477
|
+
try:
|
|
478
|
+
await provider.disconnect()
|
|
479
|
+
except (OSError, RuntimeError):
|
|
480
|
+
pass
|
|
481
|
+
|
|
482
|
+
# Idle-bar synthesis: providers whose WS feed only emits ohlc.event when
|
|
483
|
+
# a bar contained at least one tick (Capital.com is the documented
|
|
484
|
+
# case; Bybit/Binance/IB exhibit the same on illiquid moments) leave
|
|
485
|
+
# bar_index frozen across idle TF intervals. The REST history
|
|
486
|
+
# endpoint, however, returns those zero-volume bars on the same
|
|
487
|
+
# calendar boundaries — so on a future restart the historical replay
|
|
488
|
+
# would step through bars the live run never saw, and strategy
|
|
489
|
+
# decisions on the same minute diverge between live and replay.
|
|
490
|
+
# The watchdog below synthesises one zero-volume CLOSED bar
|
|
491
|
+
# (O=H=L=C=last close, V=0) per missed TF boundary so the live
|
|
492
|
+
# stream matches what REST will later return. This lives in the
|
|
493
|
+
# framework, not in each plugin, so providers stay free of bar-rhythm
|
|
494
|
+
# bookkeeping — they only emit when their feed pushes a real bar.
|
|
495
|
+
tf_seconds = max(1, int(in_seconds(timeframe)))
|
|
496
|
+
# Grace past close before declaring a bar missed. The minimum is set
|
|
497
|
+
# to 15s so plugins that recover dropped WS bars via REST have a
|
|
498
|
+
# realistic window to fetch and inject the missing bar before this
|
|
499
|
+
# synth fires — exchanges typically publish a closed bar to REST 5-10s
|
|
500
|
+
# past close, and the plugin watchdog needs a few extra seconds to
|
|
501
|
+
# detect, fetch and inject. The cap (30s) keeps longer TFs from
|
|
502
|
+
# sitting on idle gaps too long.
|
|
503
|
+
bar_grace = max(15.0, min(tf_seconds * 0.5, 30.0))
|
|
504
|
+
|
|
505
|
+
# Feed-liveness watchdog threshold: how long ``watch_ohlcv`` may stay
|
|
506
|
+
# silent during an open session before the feed is declared dead and a
|
|
507
|
+
# reconnect is forced even though ``is_connected`` still reports True
|
|
508
|
+
# (half-open socket, lost server-side subscription). ``None`` disables.
|
|
509
|
+
feed_stale_after: float | None = None
|
|
510
|
+
if provider.feed_timeout_bars:
|
|
511
|
+
feed_stale_after = max(
|
|
512
|
+
provider.feed_timeout_bars * tf_seconds, _FEED_STALE_FLOOR_S
|
|
513
|
+
)
|
|
514
|
+
|
|
515
|
+
# Resolve the symbol timezone once. ``syminfo.opening_hours`` times are
|
|
516
|
+
# expressed in this zone; epoch timestamps must be converted before
|
|
517
|
+
# session checks.
|
|
518
|
+
_sym_tz: ZoneInfo | None = None
|
|
519
|
+
if syminfo is not None and syminfo.opening_hours:
|
|
520
|
+
try:
|
|
521
|
+
_sym_tz = ZoneInfo(syminfo.timezone)
|
|
522
|
+
except Exception: # noqa: BLE001
|
|
523
|
+
logger.warning("Unknown syminfo.timezone=%r; session-gate disabled",
|
|
524
|
+
syminfo.timezone)
|
|
525
|
+
_has_calendar = (
|
|
526
|
+
syminfo is not None
|
|
527
|
+
and bool(syminfo.opening_hours)
|
|
528
|
+
and _sym_tz is not None
|
|
529
|
+
)
|
|
530
|
+
|
|
531
|
+
def _market_open_at(epoch_ts: float) -> bool:
|
|
532
|
+
"""Slot-aware "is this bar slot in-session?" check.
|
|
533
|
+
|
|
534
|
+
Returns True iff the candle ``[epoch_ts, epoch_ts+tf)`` overlaps
|
|
535
|
+
any ``opening_hours`` interval (or unconditionally True for the
|
|
536
|
+
24/7 fallback when the symbol has no calendar). Use this for
|
|
537
|
+
bar-synth decisions and any other slot-aware logic — for the
|
|
538
|
+
point-in-time "is the market open right now?" question use
|
|
539
|
+
:func:`_market_open_now` instead, which does not extend the
|
|
540
|
+
instant by one timeframe.
|
|
541
|
+
"""
|
|
542
|
+
if not _has_calendar:
|
|
543
|
+
return True
|
|
544
|
+
assert syminfo is not None and _sym_tz is not None
|
|
545
|
+
local_dt = datetime.fromtimestamp(epoch_ts, tz=_sym_tz)
|
|
546
|
+
return _is_in_session(syminfo.opening_hours, local_dt, tf_seconds)
|
|
547
|
+
|
|
548
|
+
def _market_open_now() -> bool:
|
|
549
|
+
"""Point-in-time "is the market open right now?" check.
|
|
550
|
+
|
|
551
|
+
Does not extend wall-clock by one timeframe, so it does not
|
|
552
|
+
report a session as open one timeframe before its real start.
|
|
553
|
+
Used by the reconnect gate so a long timeframe (e.g. 1h, 1D)
|
|
554
|
+
does not spend the final timeframe of a closed window churning
|
|
555
|
+
through pointless reconnect cycles.
|
|
556
|
+
"""
|
|
557
|
+
if not _has_calendar:
|
|
558
|
+
return True
|
|
559
|
+
assert syminfo is not None and _sym_tz is not None
|
|
560
|
+
local_dt = datetime.fromtimestamp(time.time(), tz=_sym_tz)
|
|
561
|
+
return _is_point_in_session(syminfo.opening_hours, local_dt)
|
|
562
|
+
|
|
563
|
+
async def _async_loop():
|
|
564
|
+
# Loop-side shutdown signal: ``_consumer`` sets it (cross-thread, via
|
|
565
|
+
# ``call_soon_threadsafe``) alongside ``stop_event`` so a reconnect
|
|
566
|
+
# handshake awaiting ``provider.connect()`` here is abandoned at once
|
|
567
|
+
# rather than blocking teardown for the whole connect timeout.
|
|
568
|
+
stop_aevent = asyncio.Event()
|
|
569
|
+
shutdown_signal['loop'] = asyncio.get_running_loop()
|
|
570
|
+
shutdown_signal['event'] = stop_aevent
|
|
571
|
+
|
|
572
|
+
async def _await_or_stop(awaitable) -> bool:
|
|
573
|
+
"""Await ``awaitable`` but abandon it if a shutdown is requested.
|
|
574
|
+
|
|
575
|
+
Races the awaitable against ``stop_aevent``. Returns ``True`` when
|
|
576
|
+
the shutdown signal won (the awaitable was cancelled — the caller
|
|
577
|
+
must stop reconnecting and let teardown proceed); ``False`` when the
|
|
578
|
+
awaitable finished on its own. Any exception it raised is
|
|
579
|
+
re-propagated so a genuine connect / reconnect failure still drives
|
|
580
|
+
the backoff-retry path.
|
|
581
|
+
"""
|
|
582
|
+
task = asyncio.ensure_future(awaitable)
|
|
583
|
+
stop_waiter = asyncio.ensure_future(stop_aevent.wait())
|
|
584
|
+
try:
|
|
585
|
+
await asyncio.wait(
|
|
586
|
+
{task, stop_waiter}, return_when=asyncio.FIRST_COMPLETED)
|
|
587
|
+
finally:
|
|
588
|
+
stop_waiter.cancel()
|
|
589
|
+
if not stop_waiter.done():
|
|
590
|
+
try:
|
|
591
|
+
await stop_waiter
|
|
592
|
+
except asyncio.CancelledError:
|
|
593
|
+
pass
|
|
594
|
+
if task.done():
|
|
595
|
+
task.result() # re-raise a connect/reconnect failure
|
|
596
|
+
return False
|
|
597
|
+
task.cancel()
|
|
598
|
+
try:
|
|
599
|
+
await task
|
|
600
|
+
except asyncio.CancelledError:
|
|
601
|
+
pass
|
|
602
|
+
except Exception: # noqa: BLE001 - teardown must not surface it
|
|
603
|
+
pass
|
|
604
|
+
return True
|
|
605
|
+
|
|
606
|
+
# Declared before ``try`` so the ``finally`` branch can reference
|
|
607
|
+
# it even when ``provider.connect()`` raises before assignment.
|
|
608
|
+
engine_task: asyncio.Task | None = None
|
|
609
|
+
# Last CLOSED bar seen (real or synthesised). The boundary
|
|
610
|
+
# watchdog only arms after the first real bar so we never
|
|
611
|
+
# fabricate state without a baseline.
|
|
612
|
+
last_closed_bar: OHLCV | None = None
|
|
613
|
+
# Latest FORMING (intra-bar) update for the currently open slot.
|
|
614
|
+
# Tracked independently of the intra-bar queue soft-cap so the
|
|
615
|
+
# boundary watchdog can finalise it with its real accumulated
|
|
616
|
+
# OHLCV when no close event arrives — providers that close a bar
|
|
617
|
+
# only on the next bar's timestamp (cTrader) never emit a closing
|
|
618
|
+
# event across a session boundary, so without this the real last
|
|
619
|
+
# bar of a session would be lost and replaced by a frozen V=0
|
|
620
|
+
# synth. Cleared when a closed bar supersedes its slot or once it
|
|
621
|
+
# has been finalised.
|
|
622
|
+
last_forming_bar: OHLCV | None = None
|
|
623
|
+
# Tracks whether the last observed market state was open. Flips to
|
|
624
|
+
# False on the first synth-skip in a closed period (emits a single
|
|
625
|
+
# INFO log) and back to True when a real bar arrives.
|
|
626
|
+
market_open_state = True
|
|
627
|
+
# Wall-clock of the last REAL ``watch_ohlcv`` update (closed bar or
|
|
628
|
+
# intra-bar tick). The feed-liveness watchdog measures staleness
|
|
629
|
+
# against this — ``last_closed_bar`` is unusable for that because
|
|
630
|
+
# idle-bar synthesis keeps rolling it forward on a dead feed.
|
|
631
|
+
# First stamped once ``connect()`` succeeds, then rebased on every
|
|
632
|
+
# reconnect and while the market is known-closed, so the staleness
|
|
633
|
+
# clock only runs against a live, in-session feed.
|
|
634
|
+
last_real_update: float
|
|
635
|
+
# Wall-clock when the current outage began (first failed attempt
|
|
636
|
+
# of a reconnect streak); ``None`` while connected. Lets the
|
|
637
|
+
# rate-limited reconnect logs and the recovery line report how
|
|
638
|
+
# long the feed was actually down.
|
|
639
|
+
outage_started: float | None = None
|
|
640
|
+
# Consecutive idle-synth bars since the last real closed bar.
|
|
641
|
+
# Drives the rate-limited synth warning: first of a streak warns,
|
|
642
|
+
# then every ``_SYNTH_WARN_EVERY``th, the rest log at DEBUG.
|
|
643
|
+
synth_streak = 0
|
|
644
|
+
async def _connect_with_initial_backoff() -> None:
|
|
645
|
+
"""Open the first provider connection, riding out transient faults.
|
|
646
|
+
|
|
647
|
+
A transient socket/TLS fault on the very first connect — a broker
|
|
648
|
+
edge reset (``ConnectionResetError``), a half-open network path —
|
|
649
|
+
must not kill the live run before the handshake. The same
|
|
650
|
+
classification the in-loop reconnect path applies to a mid-session
|
|
651
|
+
drop is extended here so the startup connect retries with capped
|
|
652
|
+
exponential backoff instead of propagating a raw traceback and
|
|
653
|
+
exiting. Retries are bounded by ``connect_timeout_seconds`` (the
|
|
654
|
+
window the constructing thread waits on ``connected_event``) so a
|
|
655
|
+
genuinely-down venue still surfaces a clean error inside that window
|
|
656
|
+
rather than looping forever, and a permanent misconfiguration
|
|
657
|
+
(non-retryable :class:`ProviderError`) is re-raised on the first
|
|
658
|
+
attempt so it keeps failing fast. The inter-attempt wait is
|
|
659
|
+
event-driven — raced against ``stop_aevent`` — so a shutdown request
|
|
660
|
+
abandons it at once without polling.
|
|
661
|
+
"""
|
|
662
|
+
deadline = time.monotonic() + connect_timeout_seconds
|
|
663
|
+
attempt = 0
|
|
664
|
+
delay = provider.reconnect_delay
|
|
665
|
+
while True:
|
|
666
|
+
try:
|
|
667
|
+
await provider.connect()
|
|
668
|
+
return
|
|
669
|
+
except BaseException as exc:
|
|
670
|
+
if not _is_transient_connect_error(exc):
|
|
671
|
+
raise
|
|
672
|
+
remaining = deadline - time.monotonic()
|
|
673
|
+
if remaining <= 0:
|
|
674
|
+
raise
|
|
675
|
+
attempt += 1
|
|
676
|
+
wait = min(delay, remaining)
|
|
677
|
+
log = logger.warning if _warn_this_attempt(attempt) else logger.debug
|
|
678
|
+
log("Initial connect failed (attempt %d): %s; "
|
|
679
|
+
"retrying in %.1fs", attempt, exc, wait)
|
|
680
|
+
try:
|
|
681
|
+
await asyncio.wait_for(stop_aevent.wait(), timeout=wait)
|
|
682
|
+
# Shutdown requested mid-backoff: surface the last
|
|
683
|
+
# connect error so teardown runs and the caller unblocks.
|
|
684
|
+
raise
|
|
685
|
+
except asyncio.TimeoutError:
|
|
686
|
+
pass
|
|
687
|
+
delay = min(delay * 2.0, provider.max_reconnect_delay)
|
|
688
|
+
|
|
689
|
+
try:
|
|
690
|
+
broker_info("WS connect starting (warmup blocks until subscribed)")
|
|
691
|
+
try:
|
|
692
|
+
await _connect_with_initial_backoff()
|
|
693
|
+
except BaseException as connect_exc:
|
|
694
|
+
# Record the real cause, then unblock the caller waiting on
|
|
695
|
+
# ``connected_event``. The append happens-before ``set()``,
|
|
696
|
+
# which the caller synchronises on via ``.wait()``, so the
|
|
697
|
+
# construction-site re-raise (when ``raise_on_connect_failure``)
|
|
698
|
+
# observes it. The outer ``except`` still pushes it onto
|
|
699
|
+
# ``bar_queue`` so a post-warmup failure surfaces through the
|
|
700
|
+
# iterator instead of hanging the wait for the full timeout.
|
|
701
|
+
connect_failure.append(connect_exc)
|
|
702
|
+
connected_event.set()
|
|
703
|
+
raise
|
|
704
|
+
watch_symbol = provider.normalize_symbol(symbol)
|
|
705
|
+
broker_info("WS connected and subscribed: %s %s@%s",
|
|
706
|
+
type(provider).__name__, symbol, timeframe)
|
|
707
|
+
connected_event.set()
|
|
708
|
+
last_real_update = time.time()
|
|
709
|
+
|
|
710
|
+
# Broker mode: attach the Order Sync Engine's event stream as
|
|
711
|
+
# a background task so OrderEvents land in its queue without
|
|
712
|
+
# blocking the OHLCV reader.
|
|
713
|
+
if engine_event_stream is not None:
|
|
714
|
+
engine_task = asyncio.create_task(engine_event_stream)
|
|
715
|
+
|
|
716
|
+
reconnect_attempts = 0
|
|
717
|
+
|
|
718
|
+
async def _handle_connection_error(
|
|
719
|
+
err: BaseException,
|
|
720
|
+
attempts: int,
|
|
721
|
+
) -> tuple[int, bool]:
|
|
722
|
+
"""Drive the closed-market wait / reconnect sequence.
|
|
723
|
+
|
|
724
|
+
Returns ``(attempts, should_break)``. Used by the
|
|
725
|
+
``except Exception`` branch below and by the
|
|
726
|
+
synth-skip session-gate when a dead WS is detected
|
|
727
|
+
inside the ``except asyncio.TimeoutError`` handler —
|
|
728
|
+
``raise`` from one ``except`` handler does not enter
|
|
729
|
+
its sibling handlers, so the original ``raise
|
|
730
|
+
ConnectionError`` pattern would escape past
|
|
731
|
+
``except Exception`` and kill the live iterator.
|
|
732
|
+
"""
|
|
733
|
+
nonlocal market_open_state, last_real_update, outage_started
|
|
734
|
+
# Session-gate: when the market is in a known-closed
|
|
735
|
+
# window (e.g. FX weekend), do not churn through
|
|
736
|
+
# reconnect cycles on a connection error. We sleep ~30s
|
|
737
|
+
# and re-enter the loop without incrementing
|
|
738
|
+
# ``reconnect_attempts`` so the backoff (and the
|
|
739
|
+
# rate-limited logging keyed on the attempt count) does
|
|
740
|
+
# not run away across a long closed window.
|
|
741
|
+
# Logs the connection error once on the closed→still-
|
|
742
|
+
# closed transition for post-mortem visibility.
|
|
743
|
+
# Uses the point-in-time helper (not the slot-aware
|
|
744
|
+
# ``_market_open_at``) so a long timeframe cannot report
|
|
745
|
+
# the market as already open one TF before the real
|
|
746
|
+
# session start.
|
|
747
|
+
if not _market_open_now():
|
|
748
|
+
if market_open_state:
|
|
749
|
+
broker_info(
|
|
750
|
+
"market closed: pausing reconnect attempts "
|
|
751
|
+
"until next session open "
|
|
752
|
+
"(last error: %s)",
|
|
753
|
+
err,
|
|
754
|
+
)
|
|
755
|
+
market_open_state = False
|
|
756
|
+
slept = 0.0
|
|
757
|
+
while slept < 30.0 and not stop_event.is_set():
|
|
758
|
+
await asyncio.sleep(min(1.0, 30.0 - slept))
|
|
759
|
+
slept += 1.0
|
|
760
|
+
return attempts, stop_event.is_set()
|
|
761
|
+
# No attempt limit: a live session must ride out an
|
|
762
|
+
# arbitrarily long outage (router restart, ISP drop,
|
|
763
|
+
# provider maintenance) and resume on its own. The
|
|
764
|
+
# exponential backoff saturates at
|
|
765
|
+
# ``provider.max_reconnect_delay`` and the per-attempt
|
|
766
|
+
# logging is rate-limited so a multi-hour outage costs
|
|
767
|
+
# a handful of log lines, not one per attempt.
|
|
768
|
+
attempts += 1
|
|
769
|
+
if attempts == 1:
|
|
770
|
+
outage_started = time.time()
|
|
771
|
+
offline_s = time.time() - (outage_started or time.time())
|
|
772
|
+
log = logger.warning if _warn_this_attempt(attempts) else logger.debug
|
|
773
|
+
log(
|
|
774
|
+
"Connection error (attempt %d, offline %.0fs): %s",
|
|
775
|
+
attempts, offline_s, err,
|
|
776
|
+
)
|
|
777
|
+
if await _await_or_stop(provider.on_disconnect()):
|
|
778
|
+
return attempts, True
|
|
779
|
+
# The exponent is clamped so the power stays a small int;
|
|
780
|
+
# the delay saturates at ``max_reconnect_delay`` anyway.
|
|
781
|
+
delay = min(
|
|
782
|
+
provider.reconnect_delay * (2 ** min(attempts - 1, 16)),
|
|
783
|
+
provider.max_reconnect_delay,
|
|
784
|
+
)
|
|
785
|
+
slept = 0.0
|
|
786
|
+
while slept < delay and not stop_event.is_set():
|
|
787
|
+
await asyncio.sleep(min(0.5, delay - slept))
|
|
788
|
+
slept += 0.5
|
|
789
|
+
if stop_event.is_set():
|
|
790
|
+
return attempts, True
|
|
791
|
+
try:
|
|
792
|
+
if await _await_or_stop(provider.disconnect()):
|
|
793
|
+
return attempts, True
|
|
794
|
+
except Exception as disc_err:
|
|
795
|
+
logger.debug(
|
|
796
|
+
"disconnect() before reconnect raised: %s",
|
|
797
|
+
disc_err,
|
|
798
|
+
)
|
|
799
|
+
try:
|
|
800
|
+
if await _await_or_stop(provider.connect()):
|
|
801
|
+
return attempts, True
|
|
802
|
+
if await _await_or_stop(provider.on_reconnect()):
|
|
803
|
+
return attempts, True
|
|
804
|
+
# Give the freshly (re)subscribed feed a full
|
|
805
|
+
# staleness window to deliver before the liveness
|
|
806
|
+
# watchdog may declare it dead again.
|
|
807
|
+
last_real_update = time.time()
|
|
808
|
+
logger.info("Reconnected successfully (attempt %d)", attempts)
|
|
809
|
+
except Exception as reconn_err:
|
|
810
|
+
log = (logger.warning if _warn_this_attempt(attempts)
|
|
811
|
+
else logger.debug)
|
|
812
|
+
log(
|
|
813
|
+
"Reconnect failed (attempt %d, offline %.0fs): %s",
|
|
814
|
+
attempts, time.time() - (outage_started or time.time()),
|
|
815
|
+
reconn_err,
|
|
816
|
+
)
|
|
817
|
+
return attempts, False
|
|
818
|
+
|
|
819
|
+
# Latched dead-WS signal from inside the
|
|
820
|
+
# ``except asyncio.TimeoutError`` handler: ``raise`` from one
|
|
821
|
+
# ``except`` does not enter sibling handlers of the same
|
|
822
|
+
# ``try``, so we cannot trigger ``_handle_connection_error``
|
|
823
|
+
# via ``raise``. The handler sets this flag instead and the
|
|
824
|
+
# top-of-loop dispatch at the next iteration consumes it.
|
|
825
|
+
pending_connection_error: BaseException | None = None
|
|
826
|
+
|
|
827
|
+
while not stop_event.is_set():
|
|
828
|
+
# Dispatch any deferred dead-WS signal from the previous
|
|
829
|
+
# iteration's ``except asyncio.TimeoutError`` handler
|
|
830
|
+
# (see ``pending_connection_error`` notes above).
|
|
831
|
+
if pending_connection_error is not None:
|
|
832
|
+
err = pending_connection_error
|
|
833
|
+
pending_connection_error = None
|
|
834
|
+
reconnect_attempts, should_break = (
|
|
835
|
+
await _handle_connection_error(err, reconnect_attempts)
|
|
836
|
+
)
|
|
837
|
+
if should_break:
|
|
838
|
+
break
|
|
839
|
+
continue
|
|
840
|
+
|
|
841
|
+
# Cap the per-iteration wait at 2 s for the existing
|
|
842
|
+
# is_connected healthcheck cadence; if a missed-bar
|
|
843
|
+
# deadline falls sooner, shorten the wait so synthesis
|
|
844
|
+
# fires promptly when the WS goes idle.
|
|
845
|
+
if last_closed_bar is not None:
|
|
846
|
+
boundary_deadline = (
|
|
847
|
+
last_closed_bar.timestamp + 2 * tf_seconds + bar_grace
|
|
848
|
+
)
|
|
849
|
+
boundary_remaining = boundary_deadline - time.time()
|
|
850
|
+
else:
|
|
851
|
+
boundary_remaining = float("inf")
|
|
852
|
+
effective_timeout = min(2.0, max(0.05, boundary_remaining))
|
|
853
|
+
|
|
854
|
+
try:
|
|
855
|
+
bar_update = await asyncio.wait_for(
|
|
856
|
+
provider.watch_ohlcv(watch_symbol, timeframe),
|
|
857
|
+
timeout=effective_timeout,
|
|
858
|
+
)
|
|
859
|
+
last_real_update = time.time()
|
|
860
|
+
if reconnect_attempts:
|
|
861
|
+
# Data-level recovery marker: ``Reconnected
|
|
862
|
+
# successfully`` above only proves the socket came
|
|
863
|
+
# back, not that data flows again (a reconnect can
|
|
864
|
+
# succeed onto a feed that stays silent). This is
|
|
865
|
+
# the line that closes an outage in the log, so it
|
|
866
|
+
# is WARNING like the failure lines it answers.
|
|
867
|
+
logger.warning(
|
|
868
|
+
"Live feed restored after %d reconnect attempt(s)"
|
|
869
|
+
" (offline %.0fs)",
|
|
870
|
+
reconnect_attempts,
|
|
871
|
+
time.time() - (outage_started or time.time()),
|
|
872
|
+
)
|
|
873
|
+
outage_started = None
|
|
874
|
+
reconnect_attempts = 0
|
|
875
|
+
|
|
876
|
+
# Filter duplicates from the historical phase. Strict
|
|
877
|
+
# ``<`` for closed bars too: see module docstring on
|
|
878
|
+
# the open-bar overlap (Capital.com et al.) — the
|
|
879
|
+
# equal-timestamp case must reach the script_runner
|
|
880
|
+
# so it can refine the partial last-warmup bar with
|
|
881
|
+
# the true close, not be silently swallowed here.
|
|
882
|
+
if last_historical_timestamp is not None:
|
|
883
|
+
ts = bar_update.timestamp
|
|
884
|
+
if ts < last_historical_timestamp:
|
|
885
|
+
continue
|
|
886
|
+
|
|
887
|
+
# In-stream dedup against the boundary watchdog:
|
|
888
|
+
# if a real ``ohlc.event`` arrives late (past
|
|
889
|
+
# ``bar_grace`` past close) for a slot we already
|
|
890
|
+
# synthesised, the synth is already in the queue
|
|
891
|
+
# and may have been consumed downstream — drop the
|
|
892
|
+
# late real bar so the consumer never sees two
|
|
893
|
+
# closed bars on the same TF boundary. Only applies
|
|
894
|
+
# to ``is_closed=True`` because providers' intra-bar
|
|
895
|
+
# updates legitimately reuse the last closed bar's
|
|
896
|
+
# timestamp until the next close arrives.
|
|
897
|
+
if (bar_update.is_closed
|
|
898
|
+
and last_closed_bar is not None
|
|
899
|
+
and bar_update.timestamp <= last_closed_bar.timestamp):
|
|
900
|
+
continue
|
|
901
|
+
|
|
902
|
+
if bar_update.is_closed:
|
|
903
|
+
last_closed_bar = bar_update
|
|
904
|
+
synth_streak = 0
|
|
905
|
+
# A real close for this slot (or a newer one)
|
|
906
|
+
# supersedes the tracked forming bar — drop it so
|
|
907
|
+
# the watchdog never re-finalises an already-closed
|
|
908
|
+
# slot. Timestamp-guarded so a late/duplicate older
|
|
909
|
+
# closed bar cannot wipe the current open slot's
|
|
910
|
+
# forming state.
|
|
911
|
+
if (last_forming_bar is not None
|
|
912
|
+
and bar_update.timestamp
|
|
913
|
+
>= last_forming_bar.timestamp):
|
|
914
|
+
last_forming_bar = None
|
|
915
|
+
if not market_open_state:
|
|
916
|
+
broker_info(
|
|
917
|
+
"market reopened: resuming live stream "
|
|
918
|
+
"(first real bar ts=%d)",
|
|
919
|
+
bar_update.timestamp,
|
|
920
|
+
)
|
|
921
|
+
market_open_state = True
|
|
922
|
+
# Backpressure probe: a non-trivial qsize or put
|
|
923
|
+
# latency here is the smoking gun for consumer-side
|
|
924
|
+
# lag stalling the asyncio loop. The queue is
|
|
925
|
+
# unbounded so ``put`` cannot actually block on
|
|
926
|
+
# capacity, but cross-thread handoff + GIL pressure
|
|
927
|
+
# can still take meaningful time when the consumer
|
|
928
|
+
# is busy. Warn so the live log preserves the
|
|
929
|
+
# evidence next time a synth-laden run happens.
|
|
930
|
+
qsize_before = bar_queue.qsize()
|
|
931
|
+
put_t0 = time.monotonic()
|
|
932
|
+
bar_queue.put(bar_update)
|
|
933
|
+
put_ms = (time.monotonic() - put_t0) * 1000.0
|
|
934
|
+
if qsize_before > 50 or put_ms > 100.0:
|
|
935
|
+
broker_warning(
|
|
936
|
+
"bar_queue backpressure: qsize_before=%d "
|
|
937
|
+
"put_ms=%.1f ts=%d",
|
|
938
|
+
qsize_before, put_ms, bar_update.timestamp,
|
|
939
|
+
)
|
|
940
|
+
else:
|
|
941
|
+
# Remember the latest forming bar BEFORE the queue
|
|
942
|
+
# soft-cap. Finalisation state must not depend on
|
|
943
|
+
# queue admission: when the consumer lags the cap
|
|
944
|
+
# drops the queued update, but the watchdog must
|
|
945
|
+
# still be able to close this slot with its real
|
|
946
|
+
# accumulated OHLCV at the session boundary.
|
|
947
|
+
last_forming_bar = bar_update
|
|
948
|
+
# Intra-bar updates are advisory — closed bars
|
|
949
|
+
# carry authoritative state. With an unbounded
|
|
950
|
+
# queue, ``put_nowait`` would never raise
|
|
951
|
+
# ``Full``, so a lagging consumer could let stale
|
|
952
|
+
# intra-bar items pile up ahead of newer closed
|
|
953
|
+
# bars and delay live-mode transition or fill
|
|
954
|
+
# processing. Apply a soft cap based on
|
|
955
|
+
# ``qsize`` so the closed-bar path retains its
|
|
956
|
+
# unbounded guarantee while intra-bar growth
|
|
957
|
+
# stays bounded.
|
|
958
|
+
if bar_queue.qsize() < _INTRA_BAR_SOFT_CAP:
|
|
959
|
+
try:
|
|
960
|
+
bar_queue.put_nowait(bar_update)
|
|
961
|
+
except Full:
|
|
962
|
+
pass
|
|
963
|
+
|
|
964
|
+
except asyncio.TimeoutError:
|
|
965
|
+
# Boundary watchdog: if the next-bar close has passed
|
|
966
|
+
# by more than ``bar_grace`` without a real WS bar,
|
|
967
|
+
# synthesise a zero-volume filler. Emits exactly one
|
|
968
|
+
# missed bar per timeout; the next iteration re-checks
|
|
969
|
+
# against ``time.time()`` and either fills the next
|
|
970
|
+
# gap or waits for a real push.
|
|
971
|
+
if (last_closed_bar is not None
|
|
972
|
+
and time.time()
|
|
973
|
+
>= last_closed_bar.timestamp
|
|
974
|
+
+ 2 * tf_seconds + bar_grace):
|
|
975
|
+
synth_ts = last_closed_bar.timestamp + tf_seconds
|
|
976
|
+
# Session-gate: never synthesise a bar for a slot
|
|
977
|
+
# that the symbol's opening_hours calendar marks
|
|
978
|
+
# as closed. Without this gate the framework would
|
|
979
|
+
# emit V=0 bars across the whole weekend on an FX
|
|
980
|
+
# symbol whose feed quite legitimately goes silent
|
|
981
|
+
# at session close. Pine remains paused;
|
|
982
|
+
# ``bar_index`` does not advance until a real bar
|
|
983
|
+
# arrives after the next session opens.
|
|
984
|
+
if not _market_open_at(synth_ts):
|
|
985
|
+
if market_open_state:
|
|
986
|
+
broker_info(
|
|
987
|
+
"market closed: pausing live stream "
|
|
988
|
+
"until next session open "
|
|
989
|
+
"(skipped synth ts=%d)",
|
|
990
|
+
synth_ts,
|
|
991
|
+
)
|
|
992
|
+
market_open_state = False
|
|
993
|
+
# Dead WS surfaces via the
|
|
994
|
+
# ``pending_connection_error`` flag, not by
|
|
995
|
+
# ``raise``: raising from inside this
|
|
996
|
+
# ``except asyncio.TimeoutError`` handler
|
|
997
|
+
# escapes past the sibling
|
|
998
|
+
# ``except Exception`` reconnect block and
|
|
999
|
+
# would kill the live iterator. WS alive
|
|
1000
|
+
# uses a coarse sleep instead of an
|
|
1001
|
+
# immediate ``continue`` so the loop does
|
|
1002
|
+
# not race ``wait_for`` at the 50ms floor
|
|
1003
|
+
# for the whole closed window
|
|
1004
|
+
# (``boundary_remaining`` is far past, so
|
|
1005
|
+
# ``effective_timeout`` would otherwise pin
|
|
1006
|
+
# to 0.05s → ~20 watch_ohlcv calls/s for
|
|
1007
|
+
# the entire weekend).
|
|
1008
|
+
if not provider.is_connected:
|
|
1009
|
+
pending_connection_error = ConnectionError(
|
|
1010
|
+
"Provider reports disconnected state"
|
|
1011
|
+
)
|
|
1012
|
+
elif not _market_open_now():
|
|
1013
|
+
# The staleness clock must not run while the
|
|
1014
|
+
# market is closed — a weekend of legitimate
|
|
1015
|
+
# feed silence would otherwise trip the
|
|
1016
|
+
# liveness watchdog right at session open.
|
|
1017
|
+
# Keyed on the CURRENT session state, not the
|
|
1018
|
+
# slot calendar: ``synth_ts`` never advances
|
|
1019
|
+
# in this branch, so after the session
|
|
1020
|
+
# reopens on a dead feed the slot stays
|
|
1021
|
+
# pinned at the pre-close boundary and an
|
|
1022
|
+
# unconditional rebase here would disarm the
|
|
1023
|
+
# watchdog forever.
|
|
1024
|
+
last_real_update = time.time()
|
|
1025
|
+
slept = 0.0
|
|
1026
|
+
while (slept < _CLOSED_WINDOW_SLEEP_S
|
|
1027
|
+
and not stop_event.is_set()):
|
|
1028
|
+
step = min(
|
|
1029
|
+
1.0,
|
|
1030
|
+
_CLOSED_WINDOW_SLEEP_S - slept,
|
|
1031
|
+
)
|
|
1032
|
+
await asyncio.sleep(step)
|
|
1033
|
+
slept += step
|
|
1034
|
+
if stop_event.is_set():
|
|
1035
|
+
break
|
|
1036
|
+
else:
|
|
1037
|
+
# Slot pinned in a closed window while the
|
|
1038
|
+
# session is open NOW: no real bar has
|
|
1039
|
+
# advanced the boundary since the close —
|
|
1040
|
+
# e.g. Monday morning on a feed that died
|
|
1041
|
+
# over the weekend. The staleness clock was
|
|
1042
|
+
# last rebased during the closed window, so
|
|
1043
|
+
# it measures in-session silence since the
|
|
1044
|
+
# reopen: trip the liveness watchdog once it
|
|
1045
|
+
# expires, otherwise wait coarsely for the
|
|
1046
|
+
# feed's first post-open bar.
|
|
1047
|
+
if (feed_stale_after is not None
|
|
1048
|
+
and time.time() - last_real_update
|
|
1049
|
+
>= feed_stale_after):
|
|
1050
|
+
pending_connection_error = ConnectionError(
|
|
1051
|
+
f"feed stale: no data from provider "
|
|
1052
|
+
f"for "
|
|
1053
|
+
f"{time.time() - last_real_update:.0f}s "
|
|
1054
|
+
f"during open session"
|
|
1055
|
+
)
|
|
1056
|
+
else:
|
|
1057
|
+
slept = 0.0
|
|
1058
|
+
while (slept < _CLOSED_WINDOW_SLEEP_S
|
|
1059
|
+
and not stop_event.is_set()):
|
|
1060
|
+
step = min(
|
|
1061
|
+
1.0,
|
|
1062
|
+
_CLOSED_WINDOW_SLEEP_S - slept,
|
|
1063
|
+
)
|
|
1064
|
+
await asyncio.sleep(step)
|
|
1065
|
+
slept += step
|
|
1066
|
+
if stop_event.is_set():
|
|
1067
|
+
break
|
|
1068
|
+
# Skip synth in all branches — we are waiting
|
|
1069
|
+
# out the closed window (WS alive), or we
|
|
1070
|
+
# deferred a connection / stale-feed error so
|
|
1071
|
+
# the next iteration's top-of-loop dispatch
|
|
1072
|
+
# drives the reconnect path.
|
|
1073
|
+
continue
|
|
1074
|
+
# Real-data finalisation: if the provider already
|
|
1075
|
+
# delivered a forming bar for exactly this slot,
|
|
1076
|
+
# close it with its accumulated OHLCV instead of
|
|
1077
|
+
# fabricating a frozen V=0 filler. Providers that
|
|
1078
|
+
# close a bar only when the next bar's timestamp
|
|
1079
|
+
# arrives (cTrader) never emit a closing event for
|
|
1080
|
+
# the last bar before a session boundary or a feed
|
|
1081
|
+
# gap — the close event simply never comes. Without
|
|
1082
|
+
# this the real last-session bar would be discarded
|
|
1083
|
+
# and replaced by a frozen synth (O=H=L=C=prev
|
|
1084
|
+
# close, V=0), shifting the strategy's view of the
|
|
1085
|
+
# close. The finalised bar is REAL data, so it ends
|
|
1086
|
+
# the idle streak rather than counting as synth, and
|
|
1087
|
+
# keeps its own ``extra_fields`` (ask/spread).
|
|
1088
|
+
if (last_forming_bar is not None
|
|
1089
|
+
and last_forming_bar.timestamp == synth_ts):
|
|
1090
|
+
finalized = last_forming_bar._replace(is_closed=True)
|
|
1091
|
+
last_closed_bar = finalized
|
|
1092
|
+
last_forming_bar = None
|
|
1093
|
+
synth_streak = 0
|
|
1094
|
+
broker_info(
|
|
1095
|
+
"idle-bar finalized forming bar: ts=%d "
|
|
1096
|
+
"close=%s vol=%s (real accumulated data; no "
|
|
1097
|
+
"close event arrived before boundary)",
|
|
1098
|
+
finalized.timestamp, finalized.close,
|
|
1099
|
+
finalized.volume,
|
|
1100
|
+
)
|
|
1101
|
+
bar_queue.put(finalized)
|
|
1102
|
+
continue
|
|
1103
|
+
# Dead feed during an open session: reconnect instead
|
|
1104
|
+
# of synthesising a frozen idle bar. Idle-bar synth is
|
|
1105
|
+
# for a live-but-quiet feed; manufacturing bars on a
|
|
1106
|
+
# dead socket would run the strategy on stale prices
|
|
1107
|
+
# while the real market keeps moving, and the steady
|
|
1108
|
+
# synth cadence keeps the boundary deadline perpetually
|
|
1109
|
+
# "just passed" so the dead-WS check below never fires.
|
|
1110
|
+
# ``is_connected`` only sees the transport, so the
|
|
1111
|
+
# feed-liveness watchdog additionally treats a healthy-
|
|
1112
|
+
# looking connection with no real ``watch_ohlcv`` data
|
|
1113
|
+
# for a whole staleness window as dead (half-open
|
|
1114
|
+
# socket, lost server-side subscription). The
|
|
1115
|
+
# ``_market_open_now()`` guard covers the slot-vs-now
|
|
1116
|
+
# mismatch: a backlog slot can still be in-session
|
|
1117
|
+
# right after the market closed, and the staleness
|
|
1118
|
+
# clock only counts in-session silence.
|
|
1119
|
+
# Defer via the flag — a ``raise`` here would escape the
|
|
1120
|
+
# sibling ``except Exception`` reconnect block (same
|
|
1121
|
+
# reason as the session-gated branch above).
|
|
1122
|
+
if not provider.is_connected:
|
|
1123
|
+
pending_connection_error = ConnectionError(
|
|
1124
|
+
"Provider reports disconnected state"
|
|
1125
|
+
)
|
|
1126
|
+
continue
|
|
1127
|
+
if (feed_stale_after is not None
|
|
1128
|
+
and time.time() - last_real_update
|
|
1129
|
+
>= feed_stale_after
|
|
1130
|
+
and _market_open_now()):
|
|
1131
|
+
pending_connection_error = ConnectionError(
|
|
1132
|
+
f"feed stale: no data from provider for "
|
|
1133
|
+
f"{time.time() - last_real_update:.0f}s "
|
|
1134
|
+
f"during open session"
|
|
1135
|
+
)
|
|
1136
|
+
continue
|
|
1137
|
+
last_close = last_closed_bar.close
|
|
1138
|
+
synth = OHLCV(
|
|
1139
|
+
timestamp=synth_ts,
|
|
1140
|
+
open=last_close,
|
|
1141
|
+
high=last_close,
|
|
1142
|
+
low=last_close,
|
|
1143
|
+
close=last_close,
|
|
1144
|
+
volume=0.0,
|
|
1145
|
+
extra_fields=last_closed_bar.extra_fields,
|
|
1146
|
+
is_closed=True,
|
|
1147
|
+
)
|
|
1148
|
+
last_closed_bar = synth
|
|
1149
|
+
# Explicit log marker so the V=0 in the next
|
|
1150
|
+
# OHLCV line is unambiguously framework synth,
|
|
1151
|
+
# not a provider-side closed bar whose
|
|
1152
|
+
# ``_tick_volume`` happened to be zero. Carries
|
|
1153
|
+
# the synth timestamp + frozen close so a
|
|
1154
|
+
# post-mortem can see exactly which TF slot the
|
|
1155
|
+
# watchdog filled and at what price. Rate-limited
|
|
1156
|
+
# within an idle streak — a legitimately quiet
|
|
1157
|
+
# market can idle for hours and one WARNING per
|
|
1158
|
+
# bar would flood the console; the first synth of
|
|
1159
|
+
# a streak and every ``_SYNTH_WARN_EVERY``th warn,
|
|
1160
|
+
# the rest log at DEBUG.
|
|
1161
|
+
synth_streak += 1
|
|
1162
|
+
synth_log = (
|
|
1163
|
+
broker_warning
|
|
1164
|
+
if (synth_streak == 1
|
|
1165
|
+
or synth_streak % _SYNTH_WARN_EVERY == 0)
|
|
1166
|
+
else logger.debug
|
|
1167
|
+
)
|
|
1168
|
+
synth_log(
|
|
1169
|
+
"idle-bar synth emitted: ts=%d close=%s "
|
|
1170
|
+
"(%d consecutive; no real ohlc.event for "
|
|
1171
|
+
">= 2*tf+grace)",
|
|
1172
|
+
synth_ts, last_close, synth_streak,
|
|
1173
|
+
)
|
|
1174
|
+
bar_queue.put(synth)
|
|
1175
|
+
continue
|
|
1176
|
+
if not provider.is_connected:
|
|
1177
|
+
# Defer to the top-of-loop dispatch rather than
|
|
1178
|
+
# raising here: a ``raise`` from inside this
|
|
1179
|
+
# ``except asyncio.TimeoutError`` handler escapes past
|
|
1180
|
+
# the sibling ``except Exception`` reconnect block and
|
|
1181
|
+
# would kill the live iterator (same reason as the
|
|
1182
|
+
# session-gated branch above).
|
|
1183
|
+
pending_connection_error = ConnectionError(
|
|
1184
|
+
"Provider reports disconnected state"
|
|
1185
|
+
)
|
|
1186
|
+
elif feed_stale_after is not None:
|
|
1187
|
+
# Feed-liveness watchdog on the regular polling
|
|
1188
|
+
# path (covers the pre-first-bar case too, where
|
|
1189
|
+
# the boundary watchdog is not armed yet). The
|
|
1190
|
+
# staleness clock pauses while the market is
|
|
1191
|
+
# closed: silence is legitimate there, and at
|
|
1192
|
+
# reopen the feed gets a fresh window.
|
|
1193
|
+
if not _market_open_now():
|
|
1194
|
+
last_real_update = time.time()
|
|
1195
|
+
elif (time.time() - last_real_update
|
|
1196
|
+
>= feed_stale_after):
|
|
1197
|
+
pending_connection_error = ConnectionError(
|
|
1198
|
+
f"feed stale: no data from provider for "
|
|
1199
|
+
f"{time.time() - last_real_update:.0f}s "
|
|
1200
|
+
f"during open session"
|
|
1201
|
+
)
|
|
1202
|
+
continue
|
|
1203
|
+
except asyncio.CancelledError:
|
|
1204
|
+
break
|
|
1205
|
+
except Exception as e:
|
|
1206
|
+
reconnect_attempts, should_break = (
|
|
1207
|
+
await _handle_connection_error(e, reconnect_attempts)
|
|
1208
|
+
)
|
|
1209
|
+
if should_break:
|
|
1210
|
+
break
|
|
1211
|
+
continue
|
|
1212
|
+
|
|
1213
|
+
except Exception as e:
|
|
1214
|
+
bar_queue.put(e)
|
|
1215
|
+
finally:
|
|
1216
|
+
if engine_task is not None and not engine_task.done():
|
|
1217
|
+
engine_task.cancel()
|
|
1218
|
+
try:
|
|
1219
|
+
await engine_task
|
|
1220
|
+
except (asyncio.CancelledError, Exception): # noqa: BLE001
|
|
1221
|
+
pass
|
|
1222
|
+
await _graceful_shutdown()
|
|
1223
|
+
bar_queue.put(_SENTINEL)
|
|
1224
|
+
|
|
1225
|
+
def _thread_target():
|
|
1226
|
+
if event_loop is not None:
|
|
1227
|
+
if event_loop.is_running():
|
|
1228
|
+
# Loop is already driven elsewhere (e.g. the CLI broker
|
|
1229
|
+
# event-loop pump). Submit our async worker onto it and
|
|
1230
|
+
# block this thread on the resulting future instead of
|
|
1231
|
+
# trying to start the loop a second time.
|
|
1232
|
+
future = asyncio.run_coroutine_threadsafe(_async_loop(), event_loop)
|
|
1233
|
+
future.result()
|
|
1234
|
+
else:
|
|
1235
|
+
asyncio.set_event_loop(event_loop)
|
|
1236
|
+
try:
|
|
1237
|
+
event_loop.run_until_complete(_async_loop())
|
|
1238
|
+
finally:
|
|
1239
|
+
# The caller owns the loop; don't close it here.
|
|
1240
|
+
pass
|
|
1241
|
+
else:
|
|
1242
|
+
asyncio.run(_async_loop())
|
|
1243
|
+
|
|
1244
|
+
# Start the provider thread, then block until ``provider.connect()``
|
|
1245
|
+
# has completed (or timed out / failed). The warmup that follows
|
|
1246
|
+
# therefore runs against an already-listening WS — bars that close
|
|
1247
|
+
# mid-warmup land in ``bar_queue`` and are drained as catch-up at
|
|
1248
|
+
# the warmup→live boundary. Without this barrier the eager-start
|
|
1249
|
+
# design is racy: a slow connect() can finish AFTER warmup, leaving
|
|
1250
|
+
# an unsubscribed window in which the next bar close is lost.
|
|
1251
|
+
thread = threading.Thread(target=_thread_target, daemon=True, name="live-provider")
|
|
1252
|
+
thread.start()
|
|
1253
|
+
if not connected_event.wait(timeout=connect_timeout_seconds):
|
|
1254
|
+
broker_info(
|
|
1255
|
+
"WS connect did not confirm within %.0fs — proceeding; "
|
|
1256
|
+
"any underlying error will surface on the first bar pull",
|
|
1257
|
+
connect_timeout_seconds,
|
|
1258
|
+
)
|
|
1259
|
+
elif connect_failure and raise_on_connect_failure:
|
|
1260
|
+
# connect() failed fast during warmup. Surface the REAL cause to the
|
|
1261
|
+
# caller now, before it reaches start_broker() (whose reconcile would
|
|
1262
|
+
# otherwise mask it). ``connected_event`` is set inside the connect
|
|
1263
|
+
# except BEFORE ``_async_loop``'s finally has run, so the producer
|
|
1264
|
+
# thread may still be draining ``_graceful_shutdown()`` (can_shutdown
|
|
1265
|
+
# poll + disconnect) on the shared broker loop. ``_consumer`` is never
|
|
1266
|
+
# created on this path, so nothing else will signal ``stop_event`` or
|
|
1267
|
+
# join the thread — and the caller's teardown closes the broker loop
|
|
1268
|
+
# right away, which would interrupt that pending shutdown and leak the
|
|
1269
|
+
# half-open connection. Signal and join here so the teardown finishes
|
|
1270
|
+
# while the loop is still alive.
|
|
1271
|
+
stop_event.set()
|
|
1272
|
+
join_timeout = (shutdown_timeout + 5.0) if shutdown_timeout > 0 else None
|
|
1273
|
+
thread.join(timeout=join_timeout)
|
|
1274
|
+
raise connect_failure[0]
|
|
1275
|
+
|
|
1276
|
+
def _consumer() -> Generator[OHLCV, None, None]:
|
|
1277
|
+
in_warmup_catchup = True
|
|
1278
|
+
|
|
1279
|
+
try:
|
|
1280
|
+
while True:
|
|
1281
|
+
if in_warmup_catchup:
|
|
1282
|
+
# Drain bars buffered during warmup without blocking.
|
|
1283
|
+
# An empty queue means warmup catch-up is over: emit
|
|
1284
|
+
# the transition sentinel and switch to live polling.
|
|
1285
|
+
try:
|
|
1286
|
+
item = bar_queue.get_nowait()
|
|
1287
|
+
except Empty:
|
|
1288
|
+
yield LIVE_TRANSITION
|
|
1289
|
+
in_warmup_catchup = False
|
|
1290
|
+
continue
|
|
1291
|
+
|
|
1292
|
+
if item is _SENTINEL:
|
|
1293
|
+
# Stream ended before any live bar arrived. Still
|
|
1294
|
+
# emit the transition so callers that gate on it
|
|
1295
|
+
# observe a clean warmup→live boundary.
|
|
1296
|
+
yield LIVE_TRANSITION
|
|
1297
|
+
break
|
|
1298
|
+
if isinstance(item, BaseException):
|
|
1299
|
+
raise item
|
|
1300
|
+
|
|
1301
|
+
# Intra-bar updates queued during warmup are dropped:
|
|
1302
|
+
# the historical loop in script_runner has no intra-bar
|
|
1303
|
+
# path and would treat each tick as a fresh bar, so the
|
|
1304
|
+
# bar_index would inflate against a still-open bar.
|
|
1305
|
+
# The latest tick state is preserved by the provider's
|
|
1306
|
+
# internal _last_bar_ohlcv; new ticks after transition
|
|
1307
|
+
# flow through the live path normally.
|
|
1308
|
+
if not item.is_closed:
|
|
1309
|
+
continue
|
|
1310
|
+
|
|
1311
|
+
yield item
|
|
1312
|
+
else:
|
|
1313
|
+
try:
|
|
1314
|
+
item = bar_queue.get(timeout=1.0)
|
|
1315
|
+
except Empty:
|
|
1316
|
+
if not thread.is_alive():
|
|
1317
|
+
break
|
|
1318
|
+
continue
|
|
1319
|
+
|
|
1320
|
+
if item is _SENTINEL:
|
|
1321
|
+
break
|
|
1322
|
+
if isinstance(item, BaseException):
|
|
1323
|
+
raise item
|
|
1324
|
+
|
|
1325
|
+
yield item
|
|
1326
|
+
|
|
1327
|
+
except KeyboardInterrupt:
|
|
1328
|
+
logger.info("Live streaming interrupted by user")
|
|
1329
|
+
finally:
|
|
1330
|
+
stop_event.set()
|
|
1331
|
+
# Wake a reconnect handshake blocked on ``provider.connect()`` on the
|
|
1332
|
+
# broker loop: setting the threading Event alone is invisible to an
|
|
1333
|
+
# in-flight ``await``, so mirror it onto the loop-side asyncio Event
|
|
1334
|
+
# (see ``_async_loop._await_or_stop``) or teardown waits out the whole
|
|
1335
|
+
# connect timeout before the producer thread can exit.
|
|
1336
|
+
_sd_loop = shutdown_signal.get('loop')
|
|
1337
|
+
_sd_event = shutdown_signal.get('event')
|
|
1338
|
+
if _sd_loop is not None and _sd_event is not None:
|
|
1339
|
+
try:
|
|
1340
|
+
_sd_loop.call_soon_threadsafe(_sd_event.set)
|
|
1341
|
+
except RuntimeError:
|
|
1342
|
+
# Loop already closed — nothing left to wake.
|
|
1343
|
+
pass
|
|
1344
|
+
join_timeout = (shutdown_timeout + 5.0) if shutdown_timeout > 0 else None
|
|
1345
|
+
thread.join(timeout=join_timeout)
|
|
1346
|
+
|
|
1347
|
+
return _consumer()
|