opencode-pyneruntime 6.6.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- opencode_pyneruntime-6.6.4.dist-info/METADATA +281 -0
- opencode_pyneruntime-6.6.4.dist-info/RECORD +261 -0
- opencode_pyneruntime-6.6.4.dist-info/WHEEL +5 -0
- opencode_pyneruntime-6.6.4.dist-info/entry_points.txt +6 -0
- opencode_pyneruntime-6.6.4.dist-info/licenses/LICENSE +201 -0
- opencode_pyneruntime-6.6.4.dist-info/licenses/NOTICE +21 -0
- opencode_pyneruntime-6.6.4.dist-info/top_level.txt +1 -0
- pynecore/__init__.py +6 -0
- pynecore/cli/__init__.py +2 -0
- pynecore/cli/app.py +238 -0
- pynecore/cli/commands/__init__.py +343 -0
- pynecore/cli/commands/benchmark.py +186 -0
- pynecore/cli/commands/compile.py +198 -0
- pynecore/cli/commands/data.py +857 -0
- pynecore/cli/commands/debug.py +63 -0
- pynecore/cli/commands/optimize.py +956 -0
- pynecore/cli/commands/plugin.py +242 -0
- pynecore/cli/commands/run.py +2006 -0
- pynecore/cli/pluggable.py +132 -0
- pynecore/cli/utils/__init__.py +0 -0
- pynecore/cli/utils/api_error_handler.py +168 -0
- pynecore/cli/utils/broker_picker.py +330 -0
- pynecore/cli/utils/error_hook.py +28 -0
- pynecore/cli/utils/keyreader.py +178 -0
- pynecore/cli/utils/provider_picker.py +19 -0
- pynecore/cli/utils/symbol_browser.py +1149 -0
- pynecore/core/__init__.py +0 -0
- pynecore/core/aggregator.py +257 -0
- pynecore/core/bar_magnifier.py +168 -0
- pynecore/core/broker/__init__.py +64 -0
- pynecore/core/broker/defaults.py +113 -0
- pynecore/core/broker/disappearance.py +927 -0
- pynecore/core/broker/emulator.py +345 -0
- pynecore/core/broker/exceptions.py +346 -0
- pynecore/core/broker/idempotency.py +401 -0
- pynecore/core/broker/intent_builder.py +334 -0
- pynecore/core/broker/journal.py +1785 -0
- pynecore/core/broker/models.py +1600 -0
- pynecore/core/broker/native_failsafe_manager.py +1436 -0
- pynecore/core/broker/one_way_emulator.py +1128 -0
- pynecore/core/broker/position.py +787 -0
- pynecore/core/broker/run_identity.py +126 -0
- pynecore/core/broker/software_entry_stop_engine.py +351 -0
- pynecore/core/broker/software_partial_bracket_engine.py +1379 -0
- pynecore/core/broker/spot_inventory.py +1327 -0
- pynecore/core/broker/storage.py +2655 -0
- pynecore/core/broker/store_helpers.py +2161 -0
- pynecore/core/broker/sync_engine.py +16070 -0
- pynecore/core/broker/validation.py +382 -0
- pynecore/core/class_property.py +7 -0
- pynecore/core/config.py +392 -0
- pynecore/core/csv_file.py +547 -0
- pynecore/core/currency.py +262 -0
- pynecore/core/data_converter.py +1002 -0
- pynecore/core/datetime.py +296 -0
- pynecore/core/download_info.py +71 -0
- pynecore/core/download_runner.py +274 -0
- pynecore/core/htf_aggregator.py +181 -0
- pynecore/core/import_hook.py +358 -0
- pynecore/core/instance_state.py +494 -0
- pynecore/core/live_ltf_collector.py +442 -0
- pynecore/core/live_ltf_window.py +189 -0
- pynecore/core/live_runner.py +1347 -0
- pynecore/core/module_property.py +26 -0
- pynecore/core/ohlcv_file.py +1888 -0
- pynecore/core/overload.py +371 -0
- pynecore/core/pine_cast.py +113 -0
- pynecore/core/pine_export.py +95 -0
- pynecore/core/pine_method.py +244 -0
- pynecore/core/pine_range.py +86 -0
- pynecore/core/pine_udt.py +69 -0
- pynecore/core/plugin/__init__.py +394 -0
- pynecore/core/plugin/broker.py +781 -0
- pynecore/core/plugin/cli.py +96 -0
- pynecore/core/plugin/live_provider.py +208 -0
- pynecore/core/plugin/provider.py +331 -0
- pynecore/core/provider_string.py +148 -0
- pynecore/core/random.py +40 -0
- pynecore/core/resampler.py +686 -0
- pynecore/core/safe_convert.py +64 -0
- pynecore/core/script.py +1011 -0
- pynecore/core/script_runner.py +3202 -0
- pynecore/core/security.py +1749 -0
- pynecore/core/security_process.py +1253 -0
- pynecore/core/security_shm.py +456 -0
- pynecore/core/series.py +417 -0
- pynecore/core/strategy_stats.py +669 -0
- pynecore/core/symbol_map.py +134 -0
- pynecore/core/syminfo.py +505 -0
- pynecore/core/viz.py +591 -0
- pynecore/lib/__init__.py +1771 -0
- pynecore/lib/_fixnan.py +32 -0
- pynecore/lib/_math_stateful.py +202 -0
- pynecore/lib/_timeframe_change.py +101 -0
- pynecore/lib/adjustment.py +6 -0
- pynecore/lib/alert.py +39 -0
- pynecore/lib/alert.pyi +14 -0
- pynecore/lib/array.py +1051 -0
- pynecore/lib/barmerge.py +60 -0
- pynecore/lib/barstate.py +30 -0
- pynecore/lib/box.py +415 -0
- pynecore/lib/chart.py +128 -0
- pynecore/lib/color.py +152 -0
- pynecore/lib/color.pyi +50 -0
- pynecore/lib/currency.py +62 -0
- pynecore/lib/dayofweek.py +36 -0
- pynecore/lib/dayofweek.pyi +18 -0
- pynecore/lib/display.py +8 -0
- pynecore/lib/dividends.py +9 -0
- pynecore/lib/earnings.py +11 -0
- pynecore/lib/extend.py +6 -0
- pynecore/lib/font.py +5 -0
- pynecore/lib/footprint.py +79 -0
- pynecore/lib/format.py +11 -0
- pynecore/lib/hline.py +67 -0
- pynecore/lib/hline.pyi +24 -0
- pynecore/lib/label.py +409 -0
- pynecore/lib/line.py +433 -0
- pynecore/lib/linefill.py +93 -0
- pynecore/lib/location.py +11 -0
- pynecore/lib/log.py +362 -0
- pynecore/lib/map.py +150 -0
- pynecore/lib/math.py +385 -0
- pynecore/lib/matrix.py +708 -0
- pynecore/lib/order.py +8 -0
- pynecore/lib/pivotpointtype.py +8 -0
- pynecore/lib/plot.py +95 -0
- pynecore/lib/plot.pyi +33 -0
- pynecore/lib/polyline.py +91 -0
- pynecore/lib/position.py +15 -0
- pynecore/lib/request.py +281 -0
- pynecore/lib/runtime.py +5 -0
- pynecore/lib/scale.py +9 -0
- pynecore/lib/session.py +267 -0
- pynecore/lib/session.pyi +12 -0
- pynecore/lib/shape.py +18 -0
- pynecore/lib/size.py +12 -0
- pynecore/lib/splits.py +4 -0
- pynecore/lib/strategy/__init__.py +4778 -0
- pynecore/lib/strategy/closedtrades.py +347 -0
- pynecore/lib/strategy/closedtrades.pyi +53 -0
- pynecore/lib/strategy/commission.py +9 -0
- pynecore/lib/strategy/direction.py +9 -0
- pynecore/lib/strategy/oca.py +13 -0
- pynecore/lib/strategy/opentrades.py +281 -0
- pynecore/lib/strategy/opentrades.pyi +49 -0
- pynecore/lib/strategy/risk.py +109 -0
- pynecore/lib/string.py +649 -0
- pynecore/lib/syminfo.py +84 -0
- pynecore/lib/ta.py +2230 -0
- pynecore/lib/table.py +290 -0
- pynecore/lib/text.py +17 -0
- pynecore/lib/ticker.py +207 -0
- pynecore/lib/timeframe.py +293 -0
- pynecore/lib/volume_row.py +67 -0
- pynecore/lib/xloc.py +4 -0
- pynecore/lib/yloc.py +5 -0
- pynecore/providers/__init__.py +0 -0
- pynecore/providers/ccxt.py +664 -0
- pynecore/providers/replay.py +187 -0
- pynecore/pynesys/__init__.py +0 -0
- pynecore/pynesys/api.py +498 -0
- pynecore/pynesys/compiler.py +112 -0
- pynecore/standalone.py +99 -0
- pynecore/testing/__init__.py +1 -0
- pynecore/testing/broker_lab/__init__.py +41 -0
- pynecore/testing/broker_lab/__main__.py +5 -0
- pynecore/testing/broker_lab/cli.py +87 -0
- pynecore/testing/broker_lab/generate.py +47 -0
- pynecore/testing/broker_lab/model.py +84 -0
- pynecore/testing/broker_lab/reference.py +645 -0
- pynecore/testing/broker_lab/runner.py +372 -0
- pynecore/testing/broker_lab/scheduler.py +50 -0
- pynecore/testing/broker_lab/subprocess.py +73 -0
- pynecore/transformers/__init__.py +0 -0
- pynecore/transformers/builtin_shadow.py +136 -0
- pynecore/transformers/closure_arguments_transformer.py +428 -0
- pynecore/transformers/display_rewrite.py +140 -0
- pynecore/transformers/dynamic_default.py +147 -0
- pynecore/transformers/function_isolation.py +757 -0
- pynecore/transformers/import_lifter.py +61 -0
- pynecore/transformers/import_normalizer.py +328 -0
- pynecore/transformers/inline_series_hoist.py +178 -0
- pynecore/transformers/input_transformer.py +175 -0
- pynecore/transformers/lib_series.py +201 -0
- pynecore/transformers/locations.py +70 -0
- pynecore/transformers/module_properties.json +3387 -0
- pynecore/transformers/module_property.py +221 -0
- pynecore/transformers/ne_guard.py +70 -0
- pynecore/transformers/persistent.py +320 -0
- pynecore/transformers/persistent_series.py +76 -0
- pynecore/transformers/safe_convert_transformer.py +97 -0
- pynecore/transformers/safe_division_transformer.py +95 -0
- pynecore/transformers/script_requirements.py +308 -0
- pynecore/transformers/security.py +752 -0
- pynecore/transformers/security_instantiation.py +274 -0
- pynecore/transformers/series.py +275 -0
- pynecore/transformers/slot_layout.py +381 -0
- pynecore/transformers/type_checking_stripper.py +25 -0
- pynecore/transformers/unused_series_detector.py +267 -0
- pynecore/types/__init__.py +21 -0
- pynecore/types/alert.py +5 -0
- pynecore/types/barmerge.py +5 -0
- pynecore/types/base.py +39 -0
- pynecore/types/box.py +37 -0
- pynecore/types/chart.py +17 -0
- pynecore/types/color.py +107 -0
- pynecore/types/currency.py +5 -0
- pynecore/types/datetime.py +6 -0
- pynecore/types/display.py +5 -0
- pynecore/types/dividends.py +5 -0
- pynecore/types/earnings.py +5 -0
- pynecore/types/extend.py +5 -0
- pynecore/types/font.py +5 -0
- pynecore/types/footprint.py +41 -0
- pynecore/types/format.py +5 -0
- pynecore/types/hline.py +24 -0
- pynecore/types/ib_persistent.py +8 -0
- pynecore/types/ib_persistent.pyi +10 -0
- pynecore/types/label.py +35 -0
- pynecore/types/line.py +32 -0
- pynecore/types/linefill.py +13 -0
- pynecore/types/location.py +5 -0
- pynecore/types/matrix.py +999 -0
- pynecore/types/na.py +237 -0
- pynecore/types/na.pyi +83 -0
- pynecore/types/ohlcv.py +12 -0
- pynecore/types/order.py +5 -0
- pynecore/types/persistent.py +8 -0
- pynecore/types/persistent.pyi +13 -0
- pynecore/types/pine_types.py +11 -0
- pynecore/types/pine_types.pyi +15 -0
- pynecore/types/pivotpointtype.py +5 -0
- pynecore/types/plot.py +12 -0
- pynecore/types/plot_meta.py +60 -0
- pynecore/types/polyline.py +40 -0
- pynecore/types/position.py +5 -0
- pynecore/types/scale.py +5 -0
- pynecore/types/script_type.py +15 -0
- pynecore/types/series.py +23 -0
- pynecore/types/series.pyi +19 -0
- pynecore/types/session.py +35 -0
- pynecore/types/shape.py +5 -0
- pynecore/types/size.py +5 -0
- pynecore/types/source.py +33 -0
- pynecore/types/splits.py +5 -0
- pynecore/types/strategy.py +45 -0
- pynecore/types/table.py +87 -0
- pynecore/types/text.py +13 -0
- pynecore/types/type_checker.py +7 -0
- pynecore/types/type_checker.pyi +48 -0
- pynecore/types/volume_row.py +36 -0
- pynecore/types/weekdays.py +11 -0
- pynecore/types/xloc.py +5 -0
- pynecore/types/yloc.py +5 -0
- pynecore/utils/__init__.py +0 -0
- pynecore/utils/file_utils.py +50 -0
- pynecore/utils/rich/__init__.py +0 -0
- pynecore/utils/rich/date_column.py +25 -0
- pynecore/utils/sequence_view.py +92 -0
- pynecore/utils/stdlib_checker.py +17 -0
|
@@ -0,0 +1,1253 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Security process loop — multiprocessing.Process target for request.security() contexts.
|
|
3
|
+
|
|
4
|
+
Each security context runs as a separate OS process with its own Python interpreter,
|
|
5
|
+
lib module, and Series state. The process loads its own OHLCV data, re-imports the
|
|
6
|
+
script module (triggering AST transformation), and runs the main() function per bar.
|
|
7
|
+
|
|
8
|
+
Communication with the chart process uses shared memory + Events (see security.py).
|
|
9
|
+
|
|
10
|
+
Two data-source modes (selected by the ``data_source`` arg type):
|
|
11
|
+
|
|
12
|
+
Backtest / file mode (``str`` path):
|
|
13
|
+
The subprocess opens a static ``.ohlcv`` file via :class:`OHLCVReader`
|
|
14
|
+
and steps through it indexed by ``current_bar``. Used by historical
|
|
15
|
+
runs and same-symbol HTF live runs (where the chart-side aggregator
|
|
16
|
+
ships closed bars over the SyncBlock).
|
|
17
|
+
|
|
18
|
+
Live cross-symbol mode (:class:`PluginSymbol`):
|
|
19
|
+
The subprocess instantiates the named :class:`LiveProviderPlugin`,
|
|
20
|
+
downloads warmup history in-memory via
|
|
21
|
+
:func:`download_warmup_in_memory`, then streams closed bars from the
|
|
22
|
+
provider's WS feed via :class:`LiveBarStreamer`. No ``.ohlcv`` file is
|
|
23
|
+
involved.
|
|
24
|
+
|
|
25
|
+
Three flavors of advance are supported, distinguished by SyncBlock flags:
|
|
26
|
+
|
|
27
|
+
(1) Historical closed bar — neither FLAG_IS_DEVELOPING nor FLAG_CLOSED_OVERRIDE
|
|
28
|
+
is set. The subprocess pulls the next available bar from its source and
|
|
29
|
+
runs ``main()`` with ``barstate.isconfirmed=True``. In file mode the
|
|
30
|
+
source is the local ``.ohlcv`` file; in live cross-symbol mode it is the
|
|
31
|
+
warmup buffer plus the WS stream.
|
|
32
|
+
|
|
33
|
+
(2) Live closed bar — FLAG_CLOSED_OVERRIDE is set, FLAG_IS_DEVELOPING is not.
|
|
34
|
+
The chart supplies the closed HTF OHLCV directly via the SyncBlock.
|
|
35
|
+
Used by same-symbol HTF contexts where the chart-side
|
|
36
|
+
:class:`HTFAggregator` builds the closed bar from the chart's own
|
|
37
|
+
OHLCV stream.
|
|
38
|
+
|
|
39
|
+
(3) Live developing bar — FLAG_IS_DEVELOPING is set. The chart supplies the
|
|
40
|
+
in-progress HTF OHLCV via the SyncBlock. The subprocess re-runs ``main()``
|
|
41
|
+
with ``barstate.isconfirmed=False`` against the same ``bar_index`` as the
|
|
42
|
+
first developing tick of this period (Series.add → set, no new bar push).
|
|
43
|
+
``RootVarSnapshot`` rolls the root vectors' var slots back to the period
|
|
44
|
+
baseline before each such re-execution; ``instance_state.reset()`` drops
|
|
45
|
+
the function instances.
|
|
46
|
+
"""
|
|
47
|
+
from __future__ import annotations
|
|
48
|
+
|
|
49
|
+
import logging
|
|
50
|
+
import os
|
|
51
|
+
import sys
|
|
52
|
+
import threading
|
|
53
|
+
from functools import partial
|
|
54
|
+
from pathlib import Path
|
|
55
|
+
from datetime import datetime, timedelta, UTC
|
|
56
|
+
from time import monotonic, sleep
|
|
57
|
+
from typing import TYPE_CHECKING, Callable, cast
|
|
58
|
+
|
|
59
|
+
from .security_shm import (
|
|
60
|
+
SyncBlock, ResultBlock, write_na,
|
|
61
|
+
FLAG_IS_DEVELOPING, FLAG_CLOSED_OVERRIDE,
|
|
62
|
+
is_ltf_window, is_ltf_chart_developing, is_ltf_live_phase,
|
|
63
|
+
)
|
|
64
|
+
from .security import (
|
|
65
|
+
create_security_protocol, inject_protocol,
|
|
66
|
+
)
|
|
67
|
+
from .live_ltf_collector import LiveLtfCollector
|
|
68
|
+
from .plugin.live_provider import PluginSymbol
|
|
69
|
+
|
|
70
|
+
logger = logging.getLogger(__name__)
|
|
71
|
+
|
|
72
|
+
if TYPE_CHECKING:
|
|
73
|
+
from multiprocessing.synchronize import Lock as LockType
|
|
74
|
+
from .live_runner import LiveBarStreamer
|
|
75
|
+
from ..types.ohlcv import OHLCV
|
|
76
|
+
|
|
77
|
+
# Seconds between parent-liveness checks in the orphan watchdog.
|
|
78
|
+
_ORPHAN_CHECK_INTERVAL = 2.0
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _start_parent_death_watchdog() -> None:
|
|
82
|
+
"""Hard-exit this security process if its parent dies without cleaning up.
|
|
83
|
+
|
|
84
|
+
A security context runs as a ``daemon=True`` :class:`multiprocessing.Process`.
|
|
85
|
+
A *clean* parent exit tears it down (daemon atexit + the runner's ``finally``
|
|
86
|
+
that sets ``stop_event`` and joins). But a *hard* kill of the parent — a
|
|
87
|
+
``SIGKILL``, or a ``subprocess`` timeout that kills only the direct child —
|
|
88
|
+
skips all of that: the child reparents to init, never receives ``stop_event``,
|
|
89
|
+
and on macOS (where timed Event waits fall back to select() polling) spins at
|
|
90
|
+
100% CPU while pinning its OHLCV and interpreter memory. macOS has no
|
|
91
|
+
``PR_SET_PDEATHSIG``, so the portable safety net is a watchdog thread that
|
|
92
|
+
notices the reparent and exits.
|
|
93
|
+
|
|
94
|
+
The captured parent PID is the spawning runner; when ``os.getppid()`` changes,
|
|
95
|
+
the parent is gone and this orphan exits via ``os._exit`` — skipping
|
|
96
|
+
atexit/``finally``, which could deadlock on shared memory the dead parent
|
|
97
|
+
still holds.
|
|
98
|
+
"""
|
|
99
|
+
parent_pid = os.getppid()
|
|
100
|
+
|
|
101
|
+
def _watch() -> None:
|
|
102
|
+
while True:
|
|
103
|
+
sleep(_ORPHAN_CHECK_INTERVAL) # watchdog tick, not a poll-retry
|
|
104
|
+
if os.getppid() != parent_pid:
|
|
105
|
+
logger.warning(
|
|
106
|
+
"Security process %d orphaned (parent %d gone); exiting.",
|
|
107
|
+
os.getpid(), parent_pid,
|
|
108
|
+
)
|
|
109
|
+
os._exit(1)
|
|
110
|
+
|
|
111
|
+
threading.Thread(target=_watch, daemon=True, name="sec-parent-watchdog").start()
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
# Warmup window for cross-symbol live security contexts. 500 bars matches the
|
|
115
|
+
# chart-side default in ``pyne run`` (``--from -500``); enough to seed any
|
|
116
|
+
# reasonable indicator without burning REST budget on excess history.
|
|
117
|
+
_DEFAULT_WARMUP_BARS = 500
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _run_rate_source_loop(
|
|
121
|
+
sec_id: str,
|
|
122
|
+
data_source: 'PluginSymbol',
|
|
123
|
+
sync_block: SyncBlock,
|
|
124
|
+
result_block: ResultBlock,
|
|
125
|
+
advance_event,
|
|
126
|
+
data_ready_event,
|
|
127
|
+
done_event,
|
|
128
|
+
stop_event,
|
|
129
|
+
result_locks: 'dict[str, LockType]',
|
|
130
|
+
) -> None:
|
|
131
|
+
"""Minimal close-only loop for auto-spawned rate-source contexts.
|
|
132
|
+
|
|
133
|
+
Skips Pine import and ``main()`` execution; every chart-side advance
|
|
134
|
+
pushes the next warmup/live ``close`` into the ``ResultBlock`` so the
|
|
135
|
+
:class:`CurrencyRateProvider` can read it as the live FX rate.
|
|
136
|
+
"""
|
|
137
|
+
from .plugin import load_plugin
|
|
138
|
+
from .plugin.live_provider import LiveProviderPlugin
|
|
139
|
+
from .live_runner import download_warmup_in_memory, LiveBarStreamer
|
|
140
|
+
from .security_shm import write_result, write_na
|
|
141
|
+
from pynecore.lib.timeframe import in_seconds
|
|
142
|
+
from pynecore.lib import _parse_timezone
|
|
143
|
+
|
|
144
|
+
provider_cls = load_plugin(data_source.provider_name)
|
|
145
|
+
if not issubclass(provider_cls, LiveProviderPlugin):
|
|
146
|
+
raise RuntimeError(
|
|
147
|
+
f"Plugin '{data_source.provider_name}' is not a live provider; "
|
|
148
|
+
f"cannot drive an auto-spawned rate-source context."
|
|
149
|
+
)
|
|
150
|
+
# ``ohlcv_dir`` is forwarded from the chart process so the child can
|
|
151
|
+
# resolve per-exchange config overrides under ``<workdir>/config/plugins``
|
|
152
|
+
# (e.g. the ``[binance]`` section of ``ccxt.toml``). No OHLCV file is
|
|
153
|
+
# written either way — the writer is just attached for symmetry; the
|
|
154
|
+
# rate-source loop reads bars directly into memory.
|
|
155
|
+
provider = provider_cls(
|
|
156
|
+
symbol=data_source.symbol,
|
|
157
|
+
timeframe=data_source.timeframe,
|
|
158
|
+
ohlcv_dir=data_source.ohlcv_dir,
|
|
159
|
+
config=data_source.config,
|
|
160
|
+
)
|
|
161
|
+
if data_source.syminfo is not None:
|
|
162
|
+
syminfo = data_source.syminfo
|
|
163
|
+
provider.syminfo = syminfo
|
|
164
|
+
else:
|
|
165
|
+
syminfo = provider.update_symbol_info()
|
|
166
|
+
|
|
167
|
+
tz = _parse_timezone(syminfo.timezone)
|
|
168
|
+
|
|
169
|
+
tf_seconds = in_seconds(data_source.timeframe)
|
|
170
|
+
time_to = datetime.now(UTC)
|
|
171
|
+
if data_source.time_from is not None:
|
|
172
|
+
time_from = data_source.time_from
|
|
173
|
+
if time_from.tzinfo is None:
|
|
174
|
+
time_from = time_from.replace(tzinfo=UTC)
|
|
175
|
+
else:
|
|
176
|
+
time_from = time_to - timedelta(seconds=tf_seconds * _DEFAULT_WARMUP_BARS)
|
|
177
|
+
|
|
178
|
+
# Start the WS streamer *before* the REST warmup download so the
|
|
179
|
+
# subscription is open while warmup is in flight. If we downloaded
|
|
180
|
+
# first, any bar that closes between the REST round-trip finishing
|
|
181
|
+
# and the WS handshake completing would be lost: the warmup window
|
|
182
|
+
# ends at ``time_to`` (captured above), and the not-yet-subscribed
|
|
183
|
+
# WS cannot replay closes from before its connect time.
|
|
184
|
+
# ``last_historical_timestamp=None`` because warmup hasn't run yet;
|
|
185
|
+
# we dedupe the post-warmup catch-up below against the warmup tail.
|
|
186
|
+
live_streamer = LiveBarStreamer(
|
|
187
|
+
provider, data_source.symbol, data_source.timeframe,
|
|
188
|
+
syminfo=syminfo,
|
|
189
|
+
last_historical_timestamp=None,
|
|
190
|
+
)
|
|
191
|
+
live_streamer.start()
|
|
192
|
+
bar_buffer = download_warmup_in_memory(provider, time_from, time_to)
|
|
193
|
+
last_warmup_ts = bar_buffer[-1].timestamp if bar_buffer else None
|
|
194
|
+
# Drain anything the WS already queued and dedupe against warmup.
|
|
195
|
+
# Streaming continues; later ``pop_new_closed_bars()`` calls only
|
|
196
|
+
# need a forward-timestamp check, which the buffer-walk already does.
|
|
197
|
+
# Equal-timestamp WS closes refine the warmup tail (providers whose
|
|
198
|
+
# REST window includes the currently forming candle deliver the true
|
|
199
|
+
# final close via WS at the same timestamp), so we overwrite rather
|
|
200
|
+
# than drop — mirroring the chart's live path which lets equal
|
|
201
|
+
# timestamps through for refinement.
|
|
202
|
+
if last_warmup_ts is not None:
|
|
203
|
+
for _bar in live_streamer.pop_new_closed_bars():
|
|
204
|
+
if _bar.timestamp == last_warmup_ts and bar_buffer:
|
|
205
|
+
bar_buffer[-1] = _bar
|
|
206
|
+
elif _bar.timestamp > last_warmup_ts:
|
|
207
|
+
bar_buffer.append(_bar)
|
|
208
|
+
else:
|
|
209
|
+
bar_buffer.extend(live_streamer.pop_new_closed_bars())
|
|
210
|
+
|
|
211
|
+
try:
|
|
212
|
+
current_bar = 0
|
|
213
|
+
# Track the most recently published close so multiple advance
|
|
214
|
+
# cycles within a single chart bar (e.g. ``calc_on_every_tick``
|
|
215
|
+
# strategies, or indicators that resignal the rate sources every
|
|
216
|
+
# script invocation) can re-publish the same value instead of
|
|
217
|
+
# falling back to ``na`` once ``bar_buffer`` is fully drained up
|
|
218
|
+
# to the current chart timestamp. ``CurrencyRateProvider`` requires
|
|
219
|
+
# a real rate for ``request.security(..., currency=…)`` conversion;
|
|
220
|
+
# writing NaN on the second call would stop the conversion mid-bar.
|
|
221
|
+
last_close: float | None = None
|
|
222
|
+
while True:
|
|
223
|
+
advance_event.wait()
|
|
224
|
+
advance_event.clear()
|
|
225
|
+
if stop_event.is_set():
|
|
226
|
+
break
|
|
227
|
+
|
|
228
|
+
target_time = sync_block.get_target_time(sec_id)
|
|
229
|
+
# Same dedupe applies forever: streamer was subscribed before
|
|
230
|
+
# warmup, so the first batch may overlap warmup; subsequent
|
|
231
|
+
# batches monotonically advance and the timestamp check is a
|
|
232
|
+
# no-op once ``last_warmup_ts`` is past. Equal-timestamp closes
|
|
233
|
+
# refine the partial warmup tail (see comment above the initial
|
|
234
|
+
# drain) — overwrite rather than drop.
|
|
235
|
+
for _bar in live_streamer.pop_new_closed_bars():
|
|
236
|
+
if last_warmup_ts is None:
|
|
237
|
+
bar_buffer.append(_bar)
|
|
238
|
+
elif _bar.timestamp == last_warmup_ts and bar_buffer:
|
|
239
|
+
bar_buffer[-1] = _bar
|
|
240
|
+
elif _bar.timestamp > last_warmup_ts:
|
|
241
|
+
bar_buffer.append(_bar)
|
|
242
|
+
|
|
243
|
+
bars_run = False
|
|
244
|
+
while current_bar < len(bar_buffer):
|
|
245
|
+
bar = bar_buffer[current_bar]
|
|
246
|
+
# A UTC->tz datetime roundtrip preserves the instant, so the
|
|
247
|
+
# raw timestamp is already the answer
|
|
248
|
+
bar_time_ms = int(bar.timestamp * 1000)
|
|
249
|
+
if bar_time_ms > target_time:
|
|
250
|
+
break
|
|
251
|
+
last_close = float(bar.close)
|
|
252
|
+
with result_locks[sec_id]:
|
|
253
|
+
write_result(result_block, sync_block, last_close)
|
|
254
|
+
current_bar += 1
|
|
255
|
+
bars_run = True
|
|
256
|
+
|
|
257
|
+
if not bars_run:
|
|
258
|
+
with result_locks[sec_id]:
|
|
259
|
+
if last_close is not None:
|
|
260
|
+
# Re-publish the last known close so a second
|
|
261
|
+
# advance within the same chart bar still sees a
|
|
262
|
+
# valid rate.
|
|
263
|
+
write_result(result_block, sync_block, last_close)
|
|
264
|
+
else:
|
|
265
|
+
# No bar has been consumed yet — the rate feed has
|
|
266
|
+
# nothing to publish; signal NaN so the consumer
|
|
267
|
+
# falls back to whatever default it has.
|
|
268
|
+
write_na(result_block, sync_block)
|
|
269
|
+
|
|
270
|
+
data_ready_event.set()
|
|
271
|
+
done_event.set()
|
|
272
|
+
finally:
|
|
273
|
+
live_streamer.stop()
|
|
274
|
+
result_block.close()
|
|
275
|
+
sync_block.close()
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def _collect_in_period_intrabars(
|
|
279
|
+
bar_buffer: 'list[OHLCV]',
|
|
280
|
+
next_idx: int,
|
|
281
|
+
period_end_exclusive: int,
|
|
282
|
+
ltf_span_ms: int,
|
|
283
|
+
chart_developing: bool,
|
|
284
|
+
grace_seconds: float,
|
|
285
|
+
wait_for_bars: 'Callable[[float], list[OHLCV]]',
|
|
286
|
+
ingest: 'Callable[[OHLCV], None]',
|
|
287
|
+
*,
|
|
288
|
+
now: 'Callable[[], float] | None' = None,
|
|
289
|
+
) -> 'tuple[list[OHLCV], int]':
|
|
290
|
+
"""Collect a chart period's closed intrabars, blocking (bounded) for a lagging
|
|
291
|
+
final intrabar on a confirmed chart bar.
|
|
292
|
+
|
|
293
|
+
Advances ``next_idx`` through ``bar_buffer`` collecting every closed intrabar
|
|
294
|
+
inside ``[period_start, period_end_exclusive)`` (future-period bars are left
|
|
295
|
+
for the next round). On a developing chart bar the partial collection is
|
|
296
|
+
returned at once. On a confirmed chart bar the published array must be the
|
|
297
|
+
FULL closed period, so when the final in-period intrabar
|
|
298
|
+
(``ts >= period_end_exclusive - ltf_span_ms``) is still missing the loop waits
|
|
299
|
+
up to ``grace_seconds`` for the streamer to deliver it, ingesting late bars as
|
|
300
|
+
they arrive — unless a later-period bar is already buffered, which means the
|
|
301
|
+
final intrabar is genuinely gone (a provider gap) and waiting would only burn
|
|
302
|
+
the grace. Warmup is a no-op here (its bars are already buffered).
|
|
303
|
+
|
|
304
|
+
The clock (``now``) and the streamer wait/ingest are injected so the loop is
|
|
305
|
+
deterministically testable without real time; ``now`` defaults to
|
|
306
|
+
:func:`time.monotonic`.
|
|
307
|
+
|
|
308
|
+
:param bar_buffer: The shared, append-only intrabar buffer; ``ingest`` mutates
|
|
309
|
+
this same list, so bars delivered by ``wait_for_bars`` become visible to
|
|
310
|
+
the next collect pass.
|
|
311
|
+
:param next_idx: Cursor into ``bar_buffer`` of the first not-yet-consumed bar.
|
|
312
|
+
:param period_end_exclusive: Exclusive end of the chart period, in ms.
|
|
313
|
+
:param ltf_span_ms: One LTF intrabar span, in ms (defines the final-intrabar
|
|
314
|
+
open as ``period_end_exclusive - ltf_span_ms``).
|
|
315
|
+
:param chart_developing: True for a forming chart bar (return the partial
|
|
316
|
+
collection without waiting), False for a confirmed one.
|
|
317
|
+
:param grace_seconds: Maximum seconds to block for the lagging final intrabar.
|
|
318
|
+
:param wait_for_bars: ``(timeout) -> list[OHLCV]`` blocking wait on the
|
|
319
|
+
streamer, up to ``timeout`` seconds.
|
|
320
|
+
:param ingest: ``(bar) -> None`` appends a streamer bar into ``bar_buffer``.
|
|
321
|
+
:param now: ``() -> float`` monotonic clock; defaults to ``time.monotonic``.
|
|
322
|
+
:return: ``(collected, next_idx)`` — the in-period intrabars and the advanced
|
|
323
|
+
cursor.
|
|
324
|
+
"""
|
|
325
|
+
if now is None:
|
|
326
|
+
now = monotonic
|
|
327
|
+
collected: 'list[OHLCV]' = []
|
|
328
|
+
final_open = period_end_exclusive - ltf_span_ms
|
|
329
|
+
deadline = now() + grace_seconds
|
|
330
|
+
while True:
|
|
331
|
+
while next_idx < len(bar_buffer):
|
|
332
|
+
_b = bar_buffer[next_idx]
|
|
333
|
+
if int(_b.timestamp * 1000) >= period_end_exclusive:
|
|
334
|
+
break
|
|
335
|
+
collected.append(_b)
|
|
336
|
+
next_idx += 1
|
|
337
|
+
if chart_developing:
|
|
338
|
+
break
|
|
339
|
+
if collected and int(collected[-1].timestamp * 1000) >= final_open:
|
|
340
|
+
break
|
|
341
|
+
# A bar beyond this period is already buffered: the stream has advanced
|
|
342
|
+
# past the period end, so the final in-period intrabar is genuinely
|
|
343
|
+
# missing (a gap) — stop waiting and publish what we have rather than
|
|
344
|
+
# burning the full grace.
|
|
345
|
+
if next_idx < len(bar_buffer):
|
|
346
|
+
logger.warning(
|
|
347
|
+
"Live LTF period ending %d ms missing its final intrabar "
|
|
348
|
+
"(stream advanced past it); publishing the gap as-is",
|
|
349
|
+
period_end_exclusive,
|
|
350
|
+
)
|
|
351
|
+
break
|
|
352
|
+
_remaining = deadline - now()
|
|
353
|
+
if _remaining <= 0:
|
|
354
|
+
logger.warning(
|
|
355
|
+
"Live LTF period ending %d ms incomplete after %.0fs grace; "
|
|
356
|
+
"publishing without the final intrabar",
|
|
357
|
+
period_end_exclusive, grace_seconds,
|
|
358
|
+
)
|
|
359
|
+
break
|
|
360
|
+
for _sb in wait_for_bars(_remaining):
|
|
361
|
+
ingest(_sb)
|
|
362
|
+
return collected, next_idx
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def _heikinashi_step(prev_open: float | None, prev_close: float | None,
|
|
366
|
+
o: float, h: float, lo: float, c: float
|
|
367
|
+
) -> tuple[float, float, float, float]:
|
|
368
|
+
"""One Heikin Ashi recurrence step over ordinary OHLC.
|
|
369
|
+
|
|
370
|
+
``prev_open``/``prev_close`` are the previous bar's Heikin Ashi open/close
|
|
371
|
+
(``None`` on the seed bar, where the open falls back to ``(o + c) / 2``).
|
|
372
|
+
|
|
373
|
+
:return: ``(ha_open, ha_high, ha_low, ha_close)``.
|
|
374
|
+
"""
|
|
375
|
+
ha_close = (o + h + lo + c) / 4.0
|
|
376
|
+
ha_open = (o + c) / 2.0 if (prev_open is None or prev_close is None) \
|
|
377
|
+
else (prev_open + prev_close) / 2.0
|
|
378
|
+
return ha_open, max(h, ha_open, ha_close), min(lo, ha_open, ha_close), ha_close
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
# noinspection PyProtectedMember
|
|
382
|
+
def security_process_main(
|
|
383
|
+
sec_id: str,
|
|
384
|
+
script_path: str,
|
|
385
|
+
data_source: 'str | PluginSymbol',
|
|
386
|
+
sync_block_name: str,
|
|
387
|
+
all_sec_ids: list[str],
|
|
388
|
+
# Events (multiprocessing.Event — picklable across spawn)
|
|
389
|
+
data_ready_event,
|
|
390
|
+
advance_event,
|
|
391
|
+
done_event,
|
|
392
|
+
stop_event,
|
|
393
|
+
is_ltf: bool = False,
|
|
394
|
+
result_locks: 'dict[str, LockType] | None' = None,
|
|
395
|
+
ohlcv_fields: 'list[str] | None' = None,
|
|
396
|
+
ohlcv_tuple: bool = False,
|
|
397
|
+
chart_type: 'str | None' = None,
|
|
398
|
+
chart_timeframe: 'str | None' = None,
|
|
399
|
+
plain_ltf: bool = False,
|
|
400
|
+
):
|
|
401
|
+
assert result_locks is not None, "result_locks must be provided by script_runner"
|
|
402
|
+
"""
|
|
403
|
+
Entry point for a security process (multiprocessing.Process target).
|
|
404
|
+
|
|
405
|
+
Re-registers import hooks (needed for spawn mode on macOS/Windows),
|
|
406
|
+
re-imports the script, and runs the bar loop.
|
|
407
|
+
|
|
408
|
+
:param sec_id: This security context's unique ID
|
|
409
|
+
:param script_path: Path to the script .py file
|
|
410
|
+
:param data_source: Either a path to an ``.ohlcv`` file (backtest / file
|
|
411
|
+
mode) or a :class:`PluginSymbol` describing the live
|
|
412
|
+
provider, symbol, timeframe and pre-loaded config.
|
|
413
|
+
:param sync_block_name: SharedMemory name of the SyncBlock
|
|
414
|
+
:param all_sec_ids: List of ALL security context IDs (for cross-reads)
|
|
415
|
+
:param data_ready_event: Event signaling data is available for reading
|
|
416
|
+
:param advance_event: Event signaling this process should advance
|
|
417
|
+
:param done_event: Event signaling this process finished its current round
|
|
418
|
+
:param stop_event: Event signaling this process should shut down
|
|
419
|
+
:param is_ltf: If True, accumulate expression values into array per round
|
|
420
|
+
:param ohlcv_fields: When set, the requested expression is only raw price
|
|
421
|
+
series (open/high/low/close/volume/hl2/hlc3/ohlc4/hlcc4); the per-bar
|
|
422
|
+
run skips main() and writes these fields straight from the bar.
|
|
423
|
+
:param ohlcv_tuple: True when ``ohlcv_fields`` came from a tuple/list
|
|
424
|
+
expression (write a tuple), False for a scalar expression.
|
|
425
|
+
:param chart_type: Synthetic chart type requested via ``ticker.heikinashi()``
|
|
426
|
+
etc. (currently only ``"heikinashi"``). When set, the child applies the
|
|
427
|
+
per-bar chart-type transform to every bar before the script reads it (so
|
|
428
|
+
backtest and live both work) and flips the matching ``chart.*`` builtin.
|
|
429
|
+
:param chart_timeframe: The chart's (main-series) timeframe. Stored in
|
|
430
|
+
``lib._main_timeframe`` so ``timeframe.main_period`` in this child reports
|
|
431
|
+
the chart TF instead of the context's own period (the child has no
|
|
432
|
+
``lib._script`` to carry it).
|
|
433
|
+
:param plain_ltf: Scalar ``request.security()`` with a timeframe finer than
|
|
434
|
+
the chart's. The chart signals every bar with the bar's own period as
|
|
435
|
+
the target; an empty round (no intrabar in the period — a feed gap)
|
|
436
|
+
must keep the previous value (TradingView ``gaps_off`` forward-fill)
|
|
437
|
+
instead of writing ``na``.
|
|
438
|
+
"""
|
|
439
|
+
# Safety net first: exit if the parent is hard-killed (see the watchdog docstring).
|
|
440
|
+
_start_parent_death_watchdog()
|
|
441
|
+
|
|
442
|
+
# Re-register import hooks (spawn mode starts a fresh Python process)
|
|
443
|
+
from . import import_hook # noqa
|
|
444
|
+
|
|
445
|
+
# Open shared memory blocks
|
|
446
|
+
sync_block = SyncBlock(all_sec_ids, create=False, name=sync_block_name)
|
|
447
|
+
result_block = ResultBlock(sec_id, create=False, version=0)
|
|
448
|
+
|
|
449
|
+
# Rate-source-only contexts skip the Pine machinery entirely: they only
|
|
450
|
+
# exist so the chart-side ``CurrencyRateProvider`` can read a fresh
|
|
451
|
+
# ``close`` from the result block. Dispatch before any script import.
|
|
452
|
+
if isinstance(data_source, PluginSymbol) and data_source.is_rate_source:
|
|
453
|
+
_run_rate_source_loop(
|
|
454
|
+
sec_id, data_source, sync_block, result_block,
|
|
455
|
+
advance_event, data_ready_event, done_event, stop_event,
|
|
456
|
+
result_locks,
|
|
457
|
+
)
|
|
458
|
+
return
|
|
459
|
+
|
|
460
|
+
# Create protocol functions for security context
|
|
461
|
+
(signal_fn, write_fn, read_fn, wait_fn, cleanup, flush_fn,
|
|
462
|
+
ltf_take_value, ltf_publish, ltf_buffer_len) = create_security_protocol(
|
|
463
|
+
sec_id, sync_block, result_block, all_sec_ids, result_locks, is_ltf=is_ltf,
|
|
464
|
+
)
|
|
465
|
+
|
|
466
|
+
# ── Source dispatch: file (backtest) or live provider (cross-symbol live) ──
|
|
467
|
+
from .syminfo import SymInfo
|
|
468
|
+
from ..types.ohlcv import OHLCV
|
|
469
|
+
live_provider = None
|
|
470
|
+
live_streamer: 'LiveBarStreamer | None' = None
|
|
471
|
+
bar_buffer: list[OHLCV] = []
|
|
472
|
+
# Watermark used by the live path to dedupe streamer bars against the
|
|
473
|
+
# warmup window — the streamer is subscribed *before* the REST warmup
|
|
474
|
+
# download, so the first batch from ``pop_new_closed_bars()`` may
|
|
475
|
+
# overlap warmup. ``None`` in file mode or when warmup yielded zero bars.
|
|
476
|
+
last_warmup_ts: int | None = None
|
|
477
|
+
# Grace window (seconds) the closed-only loop blocks on the streamer
|
|
478
|
+
# queue waiting for a bar at ``target_time`` that has not yet been
|
|
479
|
+
# published by the upstream WS feed. Cross-exchange symbol closes can
|
|
480
|
+
# land a few seconds after the chart symbol's close, and the chart
|
|
481
|
+
# process already advanced ``last_confirmed`` for this period — without
|
|
482
|
+
# blocking here the loop would emit ``na`` and never get re-signaled.
|
|
483
|
+
# Set when the live cross-symbol path is initialised below.
|
|
484
|
+
live_bar_grace_seconds: float = 0.0
|
|
485
|
+
# Warmup horizon (ms): the instant the REST warmup ended. The live LTF-window
|
|
486
|
+
# path uses it to drop a still-forming warmup tail bar. Set in the live branch.
|
|
487
|
+
warmup_horizon_ms: int = 0
|
|
488
|
+
|
|
489
|
+
if isinstance(data_source, PluginSymbol):
|
|
490
|
+
# Live cross-symbol path: own provider + warmup download + WS stream.
|
|
491
|
+
from .plugin import load_plugin
|
|
492
|
+
from .plugin.live_provider import LiveProviderPlugin
|
|
493
|
+
from .live_runner import download_warmup_in_memory, LiveBarStreamer
|
|
494
|
+
from pynecore.lib.timeframe import in_seconds
|
|
495
|
+
|
|
496
|
+
provider_cls = load_plugin(data_source.provider_name)
|
|
497
|
+
if not issubclass(provider_cls, LiveProviderPlugin):
|
|
498
|
+
raise RuntimeError(
|
|
499
|
+
f"Plugin '{data_source.provider_name}' is not a live provider; "
|
|
500
|
+
f"cannot drive cross-symbol live request.security."
|
|
501
|
+
)
|
|
502
|
+
# ``ohlcv_dir`` is forwarded from the chart process so the child can
|
|
503
|
+
# resolve per-exchange config overrides under
|
|
504
|
+
# ``<workdir>/config/plugins`` (e.g. the ``[binance]`` section of
|
|
505
|
+
# ``ccxt.toml``). The OHLCV writer is attached but no file is
|
|
506
|
+
# written — warmup goes through ``download_warmup_in_memory``.
|
|
507
|
+
live_provider = provider_cls(
|
|
508
|
+
symbol=data_source.symbol,
|
|
509
|
+
timeframe=data_source.timeframe,
|
|
510
|
+
ohlcv_dir=data_source.ohlcv_dir,
|
|
511
|
+
config=data_source.config,
|
|
512
|
+
)
|
|
513
|
+
# The chart may have pre-fetched syminfo to keep both sides in sync
|
|
514
|
+
# with a single REST round-trip; fall back to a local call if not.
|
|
515
|
+
if data_source.syminfo is not None:
|
|
516
|
+
syminfo = data_source.syminfo
|
|
517
|
+
live_provider.syminfo = syminfo
|
|
518
|
+
else:
|
|
519
|
+
syminfo = live_provider.update_symbol_info()
|
|
520
|
+
|
|
521
|
+
tf_seconds = in_seconds(data_source.timeframe)
|
|
522
|
+
# Mirror ``live_runner.bar_grace`` — 15..30s tracks how long an
|
|
523
|
+
# exchange typically lags between bar close and WS publish, scaled
|
|
524
|
+
# to the bar period (sub-15s windows would race against jitter).
|
|
525
|
+
live_bar_grace_seconds = max(15.0, min(tf_seconds * 0.5, 30.0))
|
|
526
|
+
time_to = datetime.now(UTC)
|
|
527
|
+
warmup_horizon_ms = int(time_to.timestamp() * 1000)
|
|
528
|
+
if data_source.time_from is not None:
|
|
529
|
+
time_from = data_source.time_from
|
|
530
|
+
if time_from.tzinfo is None:
|
|
531
|
+
time_from = time_from.replace(tzinfo=UTC)
|
|
532
|
+
else:
|
|
533
|
+
time_from = time_to - timedelta(seconds=tf_seconds * _DEFAULT_WARMUP_BARS)
|
|
534
|
+
|
|
535
|
+
# Start the WS streamer *before* the REST warmup so the subscription
|
|
536
|
+
# is open while warmup is in flight. If we downloaded first, any bar
|
|
537
|
+
# closing in the (REST-done → WS-connected) window would be lost.
|
|
538
|
+
# ``last_historical_timestamp=None`` because warmup hasn't run yet;
|
|
539
|
+
# the post-warmup catch-up below dedupes against the warmup tail.
|
|
540
|
+
live_streamer = LiveBarStreamer(
|
|
541
|
+
live_provider, data_source.symbol, data_source.timeframe,
|
|
542
|
+
syminfo=syminfo,
|
|
543
|
+
last_historical_timestamp=None,
|
|
544
|
+
)
|
|
545
|
+
live_streamer.start()
|
|
546
|
+
bar_buffer = download_warmup_in_memory(live_provider, time_from, time_to)
|
|
547
|
+
last_warmup_ts = bar_buffer[-1].timestamp if bar_buffer else None
|
|
548
|
+
# Equal-timestamp WS closes refine the warmup tail — providers whose
|
|
549
|
+
# REST window includes the currently forming candle deliver the true
|
|
550
|
+
# final close via WS at the same timestamp, so overwrite rather than
|
|
551
|
+
# drop. Mirrors the chart's live path (``live_ohlcv_generator``) which
|
|
552
|
+
# lets equal timestamps through for refinement. A tail refined this way
|
|
553
|
+
# is authoritative *and* already consumed from the streamer queue, so the
|
|
554
|
+
# LTF pop loop below must keep it (the stream will not redeliver it).
|
|
555
|
+
ws_refined_tail_ts: float | None = None
|
|
556
|
+
if last_warmup_ts is not None:
|
|
557
|
+
for _bar in live_streamer.pop_new_closed_bars():
|
|
558
|
+
if _bar.timestamp == last_warmup_ts and bar_buffer:
|
|
559
|
+
bar_buffer[-1] = _bar
|
|
560
|
+
ws_refined_tail_ts = _bar.timestamp
|
|
561
|
+
elif _bar.timestamp > last_warmup_ts:
|
|
562
|
+
bar_buffer.append(_bar)
|
|
563
|
+
else:
|
|
564
|
+
bar_buffer.extend(live_streamer.pop_new_closed_bars())
|
|
565
|
+
reader = None
|
|
566
|
+
ohlcv_path = None
|
|
567
|
+
else:
|
|
568
|
+
# File mode (backtest, same-symbol live with HTF aggregator).
|
|
569
|
+
ohlcv_path = data_source
|
|
570
|
+
from .ohlcv_file import OHLCVReader
|
|
571
|
+
reader = OHLCVReader(ohlcv_path)
|
|
572
|
+
reader.open()
|
|
573
|
+
|
|
574
|
+
ohlcv_base = Path(ohlcv_path)
|
|
575
|
+
toml_path = ohlcv_base.with_suffix('.toml')
|
|
576
|
+
syminfo = SymInfo.load_toml(toml_path)
|
|
577
|
+
|
|
578
|
+
# Gap-compacted bar view for HTF security contexts. ``OHLCVWriter``
|
|
579
|
+
# forward-fills non-trading session/calendar gaps with ``volume == -1`` flat
|
|
580
|
+
# bars, so a session-gapped intraday feed (e.g. a 720-minute HTF on Bursa palm
|
|
581
|
+
# oil) becomes a continuous 24h grid, and a weekday-only D/W/M feed grows
|
|
582
|
+
# synthetic weekend/holiday bars. The chart side drops these via
|
|
583
|
+
# ``read_from(skip_gaps=True)`` (and ``bar_opens`` rides the real opens only);
|
|
584
|
+
# the security child must too. Otherwise the child re-runs ``main()`` over the
|
|
585
|
+
# phantom bars: bar-count history reads (``ta.highest``/``ta.lowest``/``[n]``)
|
|
586
|
+
# span fewer real periods than TradingView, and stateful series like
|
|
587
|
+
# ``ta.sma(close, 3)`` accumulate the flat fill bars (a Friday->Monday daily
|
|
588
|
+
# gap would otherwise average two synthetic weekend closes). TradingView builds
|
|
589
|
+
# its HTF series from real bars only. LTF keeps the fills (its intrabar windows
|
|
590
|
+
# are intentionally continuous). Same-TF cross-symbol compacts too: a
|
|
591
|
+
# session-bounded feed on a 24/7 chart (TVC:US10Y on BINANCE:BTCUSDT) would
|
|
592
|
+
# otherwise run stateful indicators (``ta.ema``) over thousands of flat
|
|
593
|
+
# weekend/overnight fills, dragging their state toward the last close while
|
|
594
|
+
# TradingView's are frozen between real bars. The chart side pairs this with
|
|
595
|
+
# the ``bar_opens`` clamp in ``_get_confirmed_time``: in a gap nothing new is
|
|
596
|
+
# confirmed, so the prior real value forward-fills (TV ``gaps_off``) instead
|
|
597
|
+
# of the child advancing into an empty window and writing ``na``.
|
|
598
|
+
# ``None`` = no compaction (no gaps, or LTF).
|
|
599
|
+
real_index_map: list[int] | None = None
|
|
600
|
+
if reader is not None and not is_ltf:
|
|
601
|
+
from pynecore.lib.timeframe import in_seconds
|
|
602
|
+
if in_seconds(syminfo.period) > 0:
|
|
603
|
+
# Mirror ``read_from(skip_gaps=True)`` exactly: a gap is ``volume < 0``
|
|
604
|
+
# (the writer's -1 fill). ``>= 0`` would also drop NaN-volume real bars
|
|
605
|
+
# (no-volume instruments import as ``volume == na``), which the reader keeps.
|
|
606
|
+
rim = [i for i in range(reader.size) if not (reader.read(i).volume < 0)]
|
|
607
|
+
if len(rim) != reader.size:
|
|
608
|
+
real_index_map = rim
|
|
609
|
+
|
|
610
|
+
# Import the script module (triggers AST transformation)
|
|
611
|
+
from .script_runner import import_script, _set_lib_properties, _set_lib_syminfo_properties
|
|
612
|
+
from pynecore import lib
|
|
613
|
+
from pynecore.lib import barstate
|
|
614
|
+
from pynecore.core import instance_state, script as script_mod
|
|
615
|
+
from ..types.ohlcv import OHLCV
|
|
616
|
+
|
|
617
|
+
# Set syminfo BEFORE importing the script
|
|
618
|
+
_set_lib_syminfo_properties(syminfo)
|
|
619
|
+
|
|
620
|
+
# ``timeframe.main_period`` must report the chart TF, not this context's own
|
|
621
|
+
# period — the child has no ``lib._script`` to carry it, so propagate it here.
|
|
622
|
+
if chart_timeframe is not None:
|
|
623
|
+
lib._main_timeframe = chart_timeframe
|
|
624
|
+
|
|
625
|
+
# Chart-type context (``ticker.heikinashi()``): this child evaluates the
|
|
626
|
+
# script on Heikin Ashi candles, so the chart-type builtins must reflect the
|
|
627
|
+
# security context, not the parent's standard chart. The ``chart`` module is
|
|
628
|
+
# this process's own replica, so the parent's flags stay untouched.
|
|
629
|
+
if chart_type == 'heikinashi':
|
|
630
|
+
from pynecore.lib import chart
|
|
631
|
+
chart.is_heikinashi = True
|
|
632
|
+
chart.is_standard = False
|
|
633
|
+
|
|
634
|
+
# Parse timezone
|
|
635
|
+
from pynecore.lib import _parse_timezone
|
|
636
|
+
tz = _parse_timezone(syminfo.timezone)
|
|
637
|
+
|
|
638
|
+
# Mintick decimals for OHLC grid-snapping in ``_set_lib_properties``
|
|
639
|
+
# (``None`` when the symbol has no real mintick -> falls back to the
|
|
640
|
+
# significant-digit clean-up). Mirrors ``ScriptRunner._round_decimals``.
|
|
641
|
+
from .syminfo import mintick_decimals
|
|
642
|
+
_sec_mintick = getattr(syminfo, 'mintick', 0.0) or 0.0
|
|
643
|
+
round_decimals = mintick_decimals(_sec_mintick) if _sec_mintick > 0 else None
|
|
644
|
+
|
|
645
|
+
# A security child is a read-only replica of the user's script, not a place
|
|
646
|
+
# to persist config. ``script.indicator``/``strategy`` re-saves the script's
|
|
647
|
+
# ``.toml`` on import when ``pytest`` is absent (always true in a spawned
|
|
648
|
+
# child), so several contexts would race to rewrite — and can corrupt — the
|
|
649
|
+
# same user file. Disable the save for this process before importing.
|
|
650
|
+
os.environ['PYNE_SAVE_SCRIPT_TOML'] = '0'
|
|
651
|
+
|
|
652
|
+
# Import the script
|
|
653
|
+
script_module = import_script(Path(script_path))
|
|
654
|
+
|
|
655
|
+
# Inject security protocol into module globals. Imported library modules can
|
|
656
|
+
# contain request.security() calls too, so their transformed code references
|
|
657
|
+
# __active_security__ / __sec_read__ / … — every module whose main runs in
|
|
658
|
+
# this child needs the protocol, exactly like the chart side (see
|
|
659
|
+
# ScriptRunner's sec_modules). Injecting only the main module leaves a
|
|
660
|
+
# library main raising NameError on __active_security__.
|
|
661
|
+
sec_modules: list = [script_module]
|
|
662
|
+
for _reg_title, _reg_main in script_mod._registered_libraries:
|
|
663
|
+
_reg_mod = sys.modules.get(getattr(_reg_main, '__module__', ''))
|
|
664
|
+
if _reg_mod is not None and _reg_mod is not script_module:
|
|
665
|
+
sec_modules.append(_reg_mod)
|
|
666
|
+
for _sec_mod in sec_modules:
|
|
667
|
+
inject_protocol(_sec_mod, signal_fn, write_fn, read_fn, wait_fn,
|
|
668
|
+
active_security=sec_id)
|
|
669
|
+
|
|
670
|
+
# Fresh per-process state: drop anything inherited (fork start method) and
|
|
671
|
+
# create the root state vectors of the entry points this process drives —
|
|
672
|
+
# the script's main and the registered library mains (same binding rules
|
|
673
|
+
# as the runner: state-carrying entries get a root vector as hidden first
|
|
674
|
+
# argument, stateless ones are called as-is)
|
|
675
|
+
instance_state.reset()
|
|
676
|
+
main_func = script_module.main
|
|
677
|
+
root_keys: list[str] = []
|
|
678
|
+
seen_keys: set[str] = set()
|
|
679
|
+
bound_entries: dict[int, Callable] = {}
|
|
680
|
+
# noinspection PyProtectedMember
|
|
681
|
+
for entry_func in [main_func] + [f for _title, f in script_mod._registered_libraries]:
|
|
682
|
+
if id(entry_func) in bound_entries:
|
|
683
|
+
continue
|
|
684
|
+
entry_layout = getattr(entry_func, '__pyne_layout__', None)
|
|
685
|
+
if entry_layout is None:
|
|
686
|
+
bound_entries[id(entry_func)] = entry_func
|
|
687
|
+
continue
|
|
688
|
+
entry_root_key = f'{entry_func.__module__}.{entry_func.__qualname__}'
|
|
689
|
+
if entry_root_key in seen_keys:
|
|
690
|
+
entry_root_key = f'{entry_root_key}#{len(root_keys)}'
|
|
691
|
+
seen_keys.add(entry_root_key)
|
|
692
|
+
root_keys.append(entry_root_key)
|
|
693
|
+
bound_entries[id(entry_func)] = partial(
|
|
694
|
+
entry_func, instance_state.create_root(entry_root_key, entry_layout))
|
|
695
|
+
run_main = bound_entries[id(main_func)]
|
|
696
|
+
|
|
697
|
+
# Set lib semaphore to suppress plot/strategy/alert side effects
|
|
698
|
+
lib._lib_semaphore = True
|
|
699
|
+
|
|
700
|
+
# noinspection PyProtectedMember
|
|
701
|
+
def _run_script_main():
|
|
702
|
+
"""Mirror the chart's ``_run_libs_and_main``: registered library mains
|
|
703
|
+
initialize their exported-function proxies, so they must run before the
|
|
704
|
+
script's ``main()`` on every bar — otherwise a script that calls an
|
|
705
|
+
imported library function dies here with "Exported proxy has not been
|
|
706
|
+
initialized". ``lib._lib_semaphore`` stays True for both (every side
|
|
707
|
+
effect is suppressed in a security child)."""
|
|
708
|
+
for _title, _lib_main in script_mod._registered_libraries:
|
|
709
|
+
bound_entries.get(id(_lib_main), _lib_main)()
|
|
710
|
+
run_main()
|
|
711
|
+
|
|
712
|
+
# Plain-OHLCV fast path: the requested expression is only raw price series,
|
|
713
|
+
# all of which ``_set_lib_properties`` already wrote onto ``lib`` for the
|
|
714
|
+
# current bar (byte-identical to what main() would read). Replace the per-bar
|
|
715
|
+
# main() re-run with a direct write of those fields — every loop path
|
|
716
|
+
# (historical, live developing/closed, live LTF window) calls
|
|
717
|
+
# ``_run_script_main`` and so picks this up through the closure cell.
|
|
718
|
+
if ohlcv_fields:
|
|
719
|
+
if ohlcv_tuple:
|
|
720
|
+
_pt_fields = tuple(ohlcv_fields)
|
|
721
|
+
|
|
722
|
+
def _run_script_main():
|
|
723
|
+
write_fn(sec_id, tuple(getattr(lib, _f) for _f in _pt_fields))
|
|
724
|
+
else:
|
|
725
|
+
_pt_field = ohlcv_fields[0]
|
|
726
|
+
|
|
727
|
+
def _run_script_main():
|
|
728
|
+
write_fn(sec_id, getattr(lib, _pt_field))
|
|
729
|
+
|
|
730
|
+
# Chart-type (Heikin Ashi) per-bar transform. The two carried HA values ride
|
|
731
|
+
# a synthetic root vector whose var slots are captured/rolled back by the
|
|
732
|
+
# existing ``RootVarSnapshot`` (create_root + root_keys.append) — so a
|
|
733
|
+
# developing bar recomputes from the fixed prior-close baseline with no
|
|
734
|
+
# bespoke logic. ``_ha_apply`` reads the baseline and returns the HA bar;
|
|
735
|
+
# ``_ha_commit`` writes the new HA open/close AFTER ``_run_script_main()`` in
|
|
736
|
+
# each branch (before any ``snap.save()``), so ``save()`` captures the
|
|
737
|
+
# previous committed values and ``restore()`` rolls a re-tick back to them.
|
|
738
|
+
# Registering the HA root also makes ``RootVarSnapshot.has_vars`` True for an
|
|
739
|
+
# otherwise var-less plain-OHLCV context, activating the snapshot exactly
|
|
740
|
+
# where the recurrence needs it.
|
|
741
|
+
if chart_type == 'heikinashi':
|
|
742
|
+
_ha_key = f'__heikinashi__{sec_id}'
|
|
743
|
+
_ha_root = instance_state.create_root(_ha_key, {
|
|
744
|
+
'init': (None, None), 'series': (), 'varip': (), 'children': (),
|
|
745
|
+
'names': ('prevHaOpen', 'prevHaClose'),
|
|
746
|
+
})
|
|
747
|
+
root_keys.append(_ha_key)
|
|
748
|
+
_ha_pending: 'list[float | None]' = [None, None]
|
|
749
|
+
|
|
750
|
+
def _ha_apply(_b: OHLCV) -> OHLCV:
|
|
751
|
+
if _b.volume < 0:
|
|
752
|
+
# Gap-fill bar: forward-fill the last HA close flat and do NOT
|
|
753
|
+
# advance the recurrence (mirrors the removed feed transform's
|
|
754
|
+
# ``volume < 0`` skip + the writer's gap re-fill).
|
|
755
|
+
_fill = _ha_root[1] if _ha_root[1] is not None else _b.close
|
|
756
|
+
_ha_pending[0], _ha_pending[1] = _ha_root[0], _ha_root[1]
|
|
757
|
+
return _b._replace(open=_fill, high=_fill, low=_fill, close=_fill)
|
|
758
|
+
_ho, _hh, _hl, _hc = _heikinashi_step(
|
|
759
|
+
_ha_root[0], _ha_root[1], _b.open, _b.high, _b.low, _b.close)
|
|
760
|
+
_ha_pending[0], _ha_pending[1] = _ho, _hc
|
|
761
|
+
return _b._replace(open=_ho, high=_hh, low=_hl, close=_hc)
|
|
762
|
+
|
|
763
|
+
def _ha_commit() -> None:
|
|
764
|
+
_ha_root[0], _ha_root[1] = _ha_pending[0], _ha_pending[1]
|
|
765
|
+
else:
|
|
766
|
+
def _ha_apply(_b: OHLCV) -> OHLCV:
|
|
767
|
+
return _b
|
|
768
|
+
|
|
769
|
+
def _ha_commit() -> None:
|
|
770
|
+
pass
|
|
771
|
+
|
|
772
|
+
# Set up file-based logging if PYNE_SECURITY_LOG is set
|
|
773
|
+
security_log_path = os.environ.get("PYNE_SECURITY_LOG")
|
|
774
|
+
if security_log_path:
|
|
775
|
+
context_label = f"{syminfo.ticker} {syminfo.period}"
|
|
776
|
+
from pynecore.lib.log import setup_security_file_log
|
|
777
|
+
setup_security_file_log(security_log_path, context_label)
|
|
778
|
+
|
|
779
|
+
# Snapshot of this process's root vectors' var slots — the rollback
|
|
780
|
+
# baseline for live re-executions of the same HTF period.
|
|
781
|
+
var_snapshot: instance_state.RootVarSnapshot | None = None
|
|
782
|
+
|
|
783
|
+
# Companion series-slot snapshot, used ONLY by the live LTF-window path: a
|
|
784
|
+
# reordered feed can make the collector replay an earlier LTF ``bar_index``
|
|
785
|
+
# after a later one ran, which (unlike a same-index re-tick) would append to
|
|
786
|
+
# the root price/history series instead of overwriting. ``RootVarSnapshot``
|
|
787
|
+
# excludes series slots, so they get their own rollback. Captured only in the
|
|
788
|
+
# live phase (the warmup path is forward-only and never restores), seeded at
|
|
789
|
+
# the warmup->live transition so the first replay has a baseline.
|
|
790
|
+
series_snapshot: instance_state.RootSeriesSnapshot | None = None
|
|
791
|
+
|
|
792
|
+
# Tracks the last developing HTF period start (ms) the subprocess
|
|
793
|
+
# advanced into. Used to distinguish "new dev period" (allocate a new
|
|
794
|
+
# bar_index) from "another tick within the same dev period" (re-run
|
|
795
|
+
# against the saved baseline).
|
|
796
|
+
last_dev_period_start: int | None = None
|
|
797
|
+
|
|
798
|
+
# Set after the first live bar has been consumed. The historical loop
|
|
799
|
+
# leaves ``current_bar`` already pointing at the *next* unprocessed
|
|
800
|
+
# security index, so the very first live bar (developing or closed
|
|
801
|
+
# override) must reuse that slot. Only subsequent transitions between
|
|
802
|
+
# distinct HTF periods advance ``current_bar``.
|
|
803
|
+
seen_live_bar: bool = False
|
|
804
|
+
|
|
805
|
+
def _ensure_snapshot() -> instance_state.RootVarSnapshot | None:
|
|
806
|
+
nonlocal var_snapshot
|
|
807
|
+
if var_snapshot is None:
|
|
808
|
+
var_snapshot = instance_state.RootVarSnapshot(root_keys)
|
|
809
|
+
return var_snapshot if var_snapshot.has_vars else None
|
|
810
|
+
|
|
811
|
+
def _ensure_series_snapshot() -> instance_state.RootSeriesSnapshot | None:
|
|
812
|
+
nonlocal series_snapshot
|
|
813
|
+
if series_snapshot is None:
|
|
814
|
+
series_snapshot = instance_state.RootSeriesSnapshot(root_keys)
|
|
815
|
+
return series_snapshot if series_snapshot.has_series else None
|
|
816
|
+
|
|
817
|
+
# Append a streamer bar to ``bar_buffer``, deduped against ``last_warmup_ts``:
|
|
818
|
+
# the WS was subscribed before the REST warmup ran, so the initial batch may
|
|
819
|
+
# overlap the warmup tail (replace it) and anything older is a duplicate.
|
|
820
|
+
def _ingest_streamer_bar(_bar: 'OHLCV') -> None:
|
|
821
|
+
if last_warmup_ts is None:
|
|
822
|
+
bar_buffer.append(_bar)
|
|
823
|
+
elif _bar.timestamp == last_warmup_ts and bar_buffer:
|
|
824
|
+
bar_buffer[-1] = _bar
|
|
825
|
+
elif _bar.timestamp > last_warmup_ts:
|
|
826
|
+
bar_buffer.append(_bar)
|
|
827
|
+
|
|
828
|
+
# Polymorphic bar source: file-backed reader (random access on a static
|
|
829
|
+
# ``.ohlcv``) or in-memory buffer fed by a live WS streamer (append-only,
|
|
830
|
+
# grows over time as new closed bars arrive).
|
|
831
|
+
def _read_bar(idx: int) -> 'OHLCV | None':
|
|
832
|
+
if reader is not None:
|
|
833
|
+
if real_index_map is not None:
|
|
834
|
+
return reader.read(real_index_map[idx]) if idx < len(real_index_map) else None
|
|
835
|
+
if idx < reader.size:
|
|
836
|
+
return reader.read(idx)
|
|
837
|
+
return None
|
|
838
|
+
if live_streamer is not None:
|
|
839
|
+
for _sb in live_streamer.pop_new_closed_bars():
|
|
840
|
+
_ingest_streamer_bar(_sb)
|
|
841
|
+
return bar_buffer[idx] if idx < len(bar_buffer) else None
|
|
842
|
+
|
|
843
|
+
def _current_total() -> int:
|
|
844
|
+
if reader is not None:
|
|
845
|
+
return len(real_index_map) if real_index_map is not None else reader.size
|
|
846
|
+
return len(bar_buffer)
|
|
847
|
+
|
|
848
|
+
# ── Live LTF-window machinery (request.security_lower_tf on a streaming
|
|
849
|
+
# source) ──────────────────────────────────────────────────────────────
|
|
850
|
+
# A streaming LTF context accumulates each chart period's intrabars into a
|
|
851
|
+
# window (closed intrabars plus the developing intrabar as the live last
|
|
852
|
+
# element) via :class:`LiveLtfCollector`. Warmup replay (confirmed chart
|
|
853
|
+
# bars) and live both flow through this one path; the collector owns the
|
|
854
|
+
# monotonic LTF ``bar_index`` and the snapshot discipline, while ``_ltf_round``
|
|
855
|
+
# supplies bars from this process's own streamer and publishes the window.
|
|
856
|
+
ltf_next_idx = 0
|
|
857
|
+
ltf_span_ms = 0
|
|
858
|
+
ltf_phase_live = False
|
|
859
|
+
ltf_series_base_armed = False
|
|
860
|
+
_ltf_round: 'Callable[[int], None] | None' = None
|
|
861
|
+
if (is_ltf and live_streamer is not None
|
|
862
|
+
and isinstance(data_source, PluginSymbol)):
|
|
863
|
+
from pynecore.lib.timeframe import in_seconds
|
|
864
|
+
assert ltf_take_value is not None and ltf_publish is not None
|
|
865
|
+
_streamer = cast('LiveBarStreamer', live_streamer)
|
|
866
|
+
_take_value = ltf_take_value
|
|
867
|
+
_publish = ltf_publish
|
|
868
|
+
ltf_span_ms = int(in_seconds(data_source.timeframe) * 1000)
|
|
869
|
+
|
|
870
|
+
# The REST warmup may end on a still-forming tail bar. On the window
|
|
871
|
+
# path the developing intrabar must come ONLY from the live stream's
|
|
872
|
+
# developing slot (re-run per tick), never from the closed buffer — a
|
|
873
|
+
# forming tail left here would be consumed as a closed intrabar AND
|
|
874
|
+
# surface again via ``peek_developing_bar``. Drop any such tail; the live
|
|
875
|
+
# stream delivers it authoritatively (developing, then closed). Time
|
|
876
|
+
# eligibility (period not ended by the warmup horizon) is the primary,
|
|
877
|
+
# provider-agnostic test; the ``is_closed`` flag catches providers that
|
|
878
|
+
# stamp it explicitly. WS closes consumed during the catch-up drain are
|
|
879
|
+
# exempt: those bars were taken off the streamer queue and will not be
|
|
880
|
+
# redelivered, so popping them would permanently lose intrabars that
|
|
881
|
+
# closed during warmup. That covers both the equal-timestamp refinement
|
|
882
|
+
# of the REST tail (``ws_refined_tail_ts``) and any later closes appended
|
|
883
|
+
# past the REST tail (``timestamp > rest_warmup_tail_ts``) whose period
|
|
884
|
+
# end can fall after ``warmup_horizon_ms`` — the horizon was sampled
|
|
885
|
+
# before the warmup download, so a bar that closed mid-download would
|
|
886
|
+
# otherwise look future-dated and be dropped despite being authoritative.
|
|
887
|
+
rest_warmup_tail_ts = last_warmup_ts
|
|
888
|
+
while bar_buffer and bar_buffer[-1].timestamp != ws_refined_tail_ts and not (
|
|
889
|
+
rest_warmup_tail_ts is not None
|
|
890
|
+
and bar_buffer[-1].timestamp > rest_warmup_tail_ts) and (
|
|
891
|
+
not bar_buffer[-1].is_closed
|
|
892
|
+
or int(bar_buffer[-1].timestamp * 1000) + ltf_span_ms > warmup_horizon_ms):
|
|
893
|
+
bar_buffer.pop()
|
|
894
|
+
last_warmup_ts = bar_buffer[-1].timestamp if bar_buffer else None
|
|
895
|
+
|
|
896
|
+
def _run_ltf_intrabar(intrabar, bar_index, confirmed, is_new, islast):
|
|
897
|
+
_set_lib_properties(intrabar, bar_index, tz, lib, round_decimals)
|
|
898
|
+
lib.last_bar_index = bar_index
|
|
899
|
+
barstate.isfirst = (bar_index == 0)
|
|
900
|
+
barstate.islast = islast
|
|
901
|
+
barstate.isconfirmed = confirmed
|
|
902
|
+
barstate.ishistory = not ltf_phase_live
|
|
903
|
+
barstate.isrealtime = ltf_phase_live
|
|
904
|
+
barstate.islastconfirmedhistory = False
|
|
905
|
+
barstate.isnew = is_new
|
|
906
|
+
_run_script_main()
|
|
907
|
+
return _take_value()
|
|
908
|
+
|
|
909
|
+
def _save_ltf_baseline():
|
|
910
|
+
_snap = _ensure_snapshot()
|
|
911
|
+
if _snap is not None:
|
|
912
|
+
_snap.save()
|
|
913
|
+
# Series rollback is needed only in the live phase, where a reordered
|
|
914
|
+
# feed can trigger a backward replay; the warmup path is forward-only
|
|
915
|
+
# and never restores, so copying buffers there would be wasted work.
|
|
916
|
+
if ltf_phase_live:
|
|
917
|
+
_ss = _ensure_series_snapshot()
|
|
918
|
+
if _ss is not None:
|
|
919
|
+
_ss.save()
|
|
920
|
+
|
|
921
|
+
def _restore_ltf_baseline():
|
|
922
|
+
_snap = _ensure_snapshot()
|
|
923
|
+
if _snap is not None:
|
|
924
|
+
_snap.restore()
|
|
925
|
+
if series_snapshot is not None and series_snapshot.saved:
|
|
926
|
+
series_snapshot.restore()
|
|
927
|
+
instance_state.reset()
|
|
928
|
+
|
|
929
|
+
_ltf_collector = LiveLtfCollector(
|
|
930
|
+
_run_ltf_intrabar, _save_ltf_baseline, _restore_ltf_baseline,
|
|
931
|
+
)
|
|
932
|
+
|
|
933
|
+
def _ltf_round(slot_flags: int) -> None:
|
|
934
|
+
nonlocal ltf_next_idx, ltf_phase_live, ltf_series_base_armed
|
|
935
|
+
period_start = sync_block.get_target_time(sec_id)
|
|
936
|
+
period_end_exclusive = sync_block.get_ltf_period_end(sec_id)
|
|
937
|
+
chart_developing = is_ltf_chart_developing(slot_flags)
|
|
938
|
+
chart_confirmed = not chart_developing
|
|
939
|
+
ltf_phase_live = is_ltf_live_phase(slot_flags)
|
|
940
|
+
|
|
941
|
+
# Seed the series baseline once, on the warmup->live transition: the
|
|
942
|
+
# warmup ran forward-only so the series now hold exactly the confirmed
|
|
943
|
+
# prefix, and the first live replay's restore needs that baseline even
|
|
944
|
+
# if it precedes the first live confirmed save.
|
|
945
|
+
if ltf_phase_live and not ltf_series_base_armed:
|
|
946
|
+
_ss = _ensure_series_snapshot()
|
|
947
|
+
if _ss is not None:
|
|
948
|
+
_ss.save()
|
|
949
|
+
ltf_series_base_armed = True
|
|
950
|
+
|
|
951
|
+
for _sb in _streamer.pop_new_closed_bars():
|
|
952
|
+
_ingest_streamer_bar(_sb)
|
|
953
|
+
|
|
954
|
+
# Collect not-yet-consumed closed intrabars up to the period end
|
|
955
|
+
# (future-period bars are deferred). On a confirmed chart bar the
|
|
956
|
+
# published array must be the FULL closed period, so wait (bounded)
|
|
957
|
+
# for the final in-period intrabar if the streamer lags — a no-op
|
|
958
|
+
# during warmup, where the REST bars are already buffered.
|
|
959
|
+
collected, ltf_next_idx = _collect_in_period_intrabars(
|
|
960
|
+
bar_buffer, ltf_next_idx, period_end_exclusive, ltf_span_ms,
|
|
961
|
+
chart_developing, live_bar_grace_seconds,
|
|
962
|
+
_streamer.wait_for_bars, _ingest_streamer_bar,
|
|
963
|
+
)
|
|
964
|
+
|
|
965
|
+
developing_bar: 'OHLCV | None' = None
|
|
966
|
+
if chart_developing:
|
|
967
|
+
_dev = _streamer.peek_developing_bar()
|
|
968
|
+
# Only a forming bar strictly after the last collected closed
|
|
969
|
+
# intrabar is a live developing tail; a stale tick at/under it
|
|
970
|
+
# (e.g. a late forming update for an already-closed intrabar)
|
|
971
|
+
# would otherwise allocate a spurious new bar_index.
|
|
972
|
+
_last_collected_ms = (
|
|
973
|
+
int(collected[-1].timestamp * 1000) if collected else period_start - 1
|
|
974
|
+
)
|
|
975
|
+
if _dev is not None:
|
|
976
|
+
_dev_ms = int(_dev.timestamp * 1000)
|
|
977
|
+
if _last_collected_ms < _dev_ms < period_end_exclusive:
|
|
978
|
+
developing_bar = _dev
|
|
979
|
+
else:
|
|
980
|
+
logger.debug(
|
|
981
|
+
"LTF developing tick ts=%d ms outside (%d, %d) — skipped",
|
|
982
|
+
_dev_ms, _last_collected_ms, period_end_exclusive,
|
|
983
|
+
)
|
|
984
|
+
|
|
985
|
+
window_values = _ltf_collector.process_round(
|
|
986
|
+
period_start, period_end_exclusive, collected,
|
|
987
|
+
developing_bar, chart_confirmed=chart_confirmed,
|
|
988
|
+
)
|
|
989
|
+
_publish(window_values)
|
|
990
|
+
|
|
991
|
+
# Fixed ``last_bar_time`` anchor for file-backed (historical) runs: Pine
|
|
992
|
+
# fixes it to the security series' final bar, known up front from the
|
|
993
|
+
# static file. Scan back over the writer's gap-fill tail (``volume == -1``
|
|
994
|
+
# records; ``not (volume < 0)`` keeps NaN-volume real bars) — a no-op when
|
|
995
|
+
# ``real_index_map`` already compacted the gaps away. Streamer-fed (live)
|
|
996
|
+
# contexts keep the per-bar value from ``_set_lib_properties``: the
|
|
997
|
+
# realtime bar IS the last bar there.
|
|
998
|
+
file_last_bar_time_ms = 0
|
|
999
|
+
if reader is not None:
|
|
1000
|
+
for _pos in range(_current_total() - 1, -1, -1):
|
|
1001
|
+
_tail_bar = _read_bar(_pos)
|
|
1002
|
+
if _tail_bar is not None and not (_tail_bar.volume < 0):
|
|
1003
|
+
file_last_bar_time_ms = int(_tail_bar.timestamp * 1000)
|
|
1004
|
+
break
|
|
1005
|
+
|
|
1006
|
+
try:
|
|
1007
|
+
current_bar = 0
|
|
1008
|
+
|
|
1009
|
+
while True:
|
|
1010
|
+
# Wait for chart to signal this process
|
|
1011
|
+
advance_event.wait()
|
|
1012
|
+
advance_event.clear()
|
|
1013
|
+
|
|
1014
|
+
# Check for shutdown
|
|
1015
|
+
if stop_event.is_set():
|
|
1016
|
+
break
|
|
1017
|
+
|
|
1018
|
+
target_time = sync_block.get_target_time(sec_id)
|
|
1019
|
+
flags = sync_block.get_flags(sec_id)
|
|
1020
|
+
is_developing = bool(flags & FLAG_IS_DEVELOPING)
|
|
1021
|
+
closed_override = bool(flags & FLAG_CLOSED_OVERRIDE)
|
|
1022
|
+
|
|
1023
|
+
# ── (4) Live LTF window round ──
|
|
1024
|
+
# Checked first: a streaming request.security_lower_tf round carries
|
|
1025
|
+
# its own flag and never the HTF developing/closed-override flags.
|
|
1026
|
+
if _ltf_round is not None and is_ltf_window(flags):
|
|
1027
|
+
_ltf_round(flags)
|
|
1028
|
+
data_ready_event.set()
|
|
1029
|
+
done_event.set()
|
|
1030
|
+
continue
|
|
1031
|
+
|
|
1032
|
+
# ── (3) Live developing bar ──
|
|
1033
|
+
if is_developing:
|
|
1034
|
+
dev_open, dev_high, dev_low, dev_close, dev_volume, dev_time_ms = (
|
|
1035
|
+
sync_block.get_developing_bar(sec_id)
|
|
1036
|
+
)
|
|
1037
|
+
dev_ts_sec = dev_time_ms // 1000
|
|
1038
|
+
ohlcv = OHLCV(
|
|
1039
|
+
timestamp=dev_ts_sec,
|
|
1040
|
+
open=dev_open, high=dev_high, low=dev_low,
|
|
1041
|
+
close=dev_close, volume=dev_volume,
|
|
1042
|
+
)
|
|
1043
|
+
|
|
1044
|
+
is_new_dev_period = (last_dev_period_start != dev_time_ms)
|
|
1045
|
+
if is_new_dev_period:
|
|
1046
|
+
# Step the subprocess into a fresh bar slot — but reuse
|
|
1047
|
+
# the slot already pointed at by ``current_bar`` for the
|
|
1048
|
+
# very first live bar (the historical loop leaves it on
|
|
1049
|
+
# the next unprocessed index).
|
|
1050
|
+
if seen_live_bar:
|
|
1051
|
+
current_bar += 1
|
|
1052
|
+
seen_live_bar = True
|
|
1053
|
+
last_dev_period_start = dev_time_ms
|
|
1054
|
+
else:
|
|
1055
|
+
# Same dev period: restore var globals to the period
|
|
1056
|
+
# baseline (saved either after the prior closed run or
|
|
1057
|
+
# at the start of this dev period) and re-run.
|
|
1058
|
+
snap = _ensure_snapshot()
|
|
1059
|
+
if snap is not None:
|
|
1060
|
+
snap.restore()
|
|
1061
|
+
instance_state.reset()
|
|
1062
|
+
|
|
1063
|
+
_set_lib_properties(_ha_apply(ohlcv), current_bar, tz, lib, round_decimals)
|
|
1064
|
+
lib.last_bar_index = current_bar
|
|
1065
|
+
|
|
1066
|
+
barstate.isfirst = (current_bar == 0)
|
|
1067
|
+
barstate.islast = True
|
|
1068
|
+
barstate.isconfirmed = False
|
|
1069
|
+
barstate.ishistory = False
|
|
1070
|
+
barstate.isrealtime = True
|
|
1071
|
+
barstate.islastconfirmedhistory = False
|
|
1072
|
+
barstate.isnew = is_new_dev_period
|
|
1073
|
+
|
|
1074
|
+
if is_new_dev_period:
|
|
1075
|
+
snap = _ensure_snapshot()
|
|
1076
|
+
if snap is not None:
|
|
1077
|
+
snap.save()
|
|
1078
|
+
|
|
1079
|
+
_run_script_main()
|
|
1080
|
+
# Commit the developing HA open/close AFTER the run and the
|
|
1081
|
+
# new-period ``save()`` above; a same-period re-tick's
|
|
1082
|
+
# ``snap.restore()`` rolls these back to the period baseline.
|
|
1083
|
+
_ha_commit()
|
|
1084
|
+
|
|
1085
|
+
data_ready_event.set()
|
|
1086
|
+
done_event.set()
|
|
1087
|
+
continue
|
|
1088
|
+
|
|
1089
|
+
# ── (2) Live closed bar (OHLCV from SyncBlock) ──
|
|
1090
|
+
if closed_override:
|
|
1091
|
+
dev_open, dev_high, dev_low, dev_close, dev_volume, dev_time_ms = (
|
|
1092
|
+
sync_block.get_developing_bar(sec_id)
|
|
1093
|
+
)
|
|
1094
|
+
ts_sec = dev_time_ms // 1000
|
|
1095
|
+
ohlcv = OHLCV(
|
|
1096
|
+
timestamp=ts_sec,
|
|
1097
|
+
open=dev_open, high=dev_high, low=dev_low,
|
|
1098
|
+
close=dev_close, volume=dev_volume,
|
|
1099
|
+
)
|
|
1100
|
+
|
|
1101
|
+
# TV semantics: a developing HTF bar and its eventual close
|
|
1102
|
+
# share the same security-series index (Series.add() degrades
|
|
1103
|
+
# to set() because the bar_index hasn't moved). Only allocate
|
|
1104
|
+
# a NEW bar_index when this closed bar is not the closing of
|
|
1105
|
+
# an in-flight dev period (e.g. live closed bar arriving with
|
|
1106
|
+
# no prior dev — currently unused, but kept correct). The
|
|
1107
|
+
# very first live bar reuses the next-unprocessed index the
|
|
1108
|
+
# historical loop left in ``current_bar``.
|
|
1109
|
+
if last_dev_period_start == dev_time_ms:
|
|
1110
|
+
# Same HTF bar — restore var baseline, then re-run as
|
|
1111
|
+
# confirmed close. Series writes overwrite the dev value.
|
|
1112
|
+
snap = _ensure_snapshot()
|
|
1113
|
+
if snap is not None:
|
|
1114
|
+
snap.restore()
|
|
1115
|
+
instance_state.reset()
|
|
1116
|
+
is_new_closed_period = False
|
|
1117
|
+
else:
|
|
1118
|
+
if seen_live_bar:
|
|
1119
|
+
current_bar += 1
|
|
1120
|
+
seen_live_bar = True
|
|
1121
|
+
is_new_closed_period = True
|
|
1122
|
+
|
|
1123
|
+
last_dev_period_start = None
|
|
1124
|
+
|
|
1125
|
+
_set_lib_properties(_ha_apply(ohlcv), current_bar, tz, lib, round_decimals)
|
|
1126
|
+
lib.last_bar_index = current_bar
|
|
1127
|
+
barstate.isfirst = (current_bar == 0)
|
|
1128
|
+
barstate.islast = False
|
|
1129
|
+
barstate.isconfirmed = True
|
|
1130
|
+
barstate.ishistory = False
|
|
1131
|
+
barstate.isrealtime = True
|
|
1132
|
+
barstate.islastconfirmedhistory = False
|
|
1133
|
+
barstate.isnew = is_new_closed_period
|
|
1134
|
+
|
|
1135
|
+
_run_script_main()
|
|
1136
|
+
# Commit the closed HA open/close BEFORE the baseline ``save()``
|
|
1137
|
+
# below, so the confirmed values become the next period's seed.
|
|
1138
|
+
_ha_commit()
|
|
1139
|
+
|
|
1140
|
+
# Snapshot AFTER the closed run completes — baseline for
|
|
1141
|
+
# subsequent developing iterations of the next HTF period.
|
|
1142
|
+
snap = _ensure_snapshot()
|
|
1143
|
+
if snap is not None:
|
|
1144
|
+
snap.save()
|
|
1145
|
+
|
|
1146
|
+
data_ready_event.set()
|
|
1147
|
+
done_event.set()
|
|
1148
|
+
continue
|
|
1149
|
+
|
|
1150
|
+
# Historical path resets dev-period tracking.
|
|
1151
|
+
last_dev_period_start = None
|
|
1152
|
+
|
|
1153
|
+
# ── (1) Historical / cross-symbol-live closed bar from local source ──
|
|
1154
|
+
# In cross-symbol live mode (``live_streamer is not None``), the
|
|
1155
|
+
# upstream WS feed may publish the requested symbol's closed bar
|
|
1156
|
+
# a few seconds after the chart symbol's close. The chart side
|
|
1157
|
+
# has already advanced ``last_confirmed = target_time`` and will
|
|
1158
|
+
# not re-signal this period, so we must block briefly until the
|
|
1159
|
+
# bar arrives instead of falling through to ``write_na``.
|
|
1160
|
+
bars_run = False
|
|
1161
|
+
# File-backed LTF: values written while replaying feed bars that
|
|
1162
|
+
# open BEFORE the chart bar's own period (cold start mid-feed,
|
|
1163
|
+
# chart session gaps) are expression-state warmup, not array
|
|
1164
|
+
# content — TradingView arrays hold only the bar's own period.
|
|
1165
|
+
ltf_period_start = (
|
|
1166
|
+
sync_block.get_ltf_period_start(sec_id) if is_ltf else 0
|
|
1167
|
+
)
|
|
1168
|
+
ltf_prefix_len = 0
|
|
1169
|
+
grace_deadline: float | None = None
|
|
1170
|
+
if live_streamer is not None:
|
|
1171
|
+
grace_deadline = monotonic() + live_bar_grace_seconds
|
|
1172
|
+
while True:
|
|
1173
|
+
ohlcv_file_bar = _read_bar(current_bar)
|
|
1174
|
+
if ohlcv_file_bar is None:
|
|
1175
|
+
if grace_deadline is not None and live_streamer is not None:
|
|
1176
|
+
remaining = grace_deadline - monotonic()
|
|
1177
|
+
if remaining > 0:
|
|
1178
|
+
# Block on the streamer queue for the new bar.
|
|
1179
|
+
# ``_read_bar`` only consults the streamer when
|
|
1180
|
+
# ``current_bar >= len(bar_buffer)``, so any
|
|
1181
|
+
# bars returned here are appended to the buffer
|
|
1182
|
+
# via ``_read_bar`` on the next iteration.
|
|
1183
|
+
new_bars = live_streamer.wait_for_bars(remaining)
|
|
1184
|
+
if new_bars:
|
|
1185
|
+
for _bar in new_bars:
|
|
1186
|
+
if last_warmup_ts is None:
|
|
1187
|
+
bar_buffer.append(_bar)
|
|
1188
|
+
elif (_bar.timestamp == last_warmup_ts
|
|
1189
|
+
and bar_buffer):
|
|
1190
|
+
bar_buffer[-1] = _bar
|
|
1191
|
+
elif _bar.timestamp > last_warmup_ts:
|
|
1192
|
+
bar_buffer.append(_bar)
|
|
1193
|
+
continue
|
|
1194
|
+
break
|
|
1195
|
+
# A UTC->tz datetime roundtrip preserves the instant, so the
|
|
1196
|
+
# raw timestamp is already the answer
|
|
1197
|
+
bar_time_ms = int(ohlcv_file_bar.timestamp * 1000)
|
|
1198
|
+
if bar_time_ms > target_time:
|
|
1199
|
+
break
|
|
1200
|
+
|
|
1201
|
+
total_bars = _current_total()
|
|
1202
|
+
_set_lib_properties(_ha_apply(ohlcv_file_bar), current_bar, tz, lib, round_decimals)
|
|
1203
|
+
lib.last_bar_index = total_bars - 1
|
|
1204
|
+
if reader is not None:
|
|
1205
|
+
lib.last_bar_time = file_last_bar_time_ms
|
|
1206
|
+
barstate.isfirst = (current_bar == 0)
|
|
1207
|
+
barstate.islast = (current_bar == total_bars - 1)
|
|
1208
|
+
barstate.isconfirmed = True
|
|
1209
|
+
# In PluginSymbol mode, bars beyond the warmup tail come from
|
|
1210
|
+
# the live WS streamer and represent realtime closes. Without
|
|
1211
|
+
# a phase flip here, ``barstate.ishistory`` would stay ``True``
|
|
1212
|
+
# for streamed closed bars — diverging from the same-symbol
|
|
1213
|
+
# live path (see lines 511-513 and 568-570 above) and breaking
|
|
1214
|
+
# script branches that key off ``barstate.isrealtime``.
|
|
1215
|
+
if (live_streamer is not None
|
|
1216
|
+
and last_warmup_ts is not None
|
|
1217
|
+
and ohlcv_file_bar.timestamp > last_warmup_ts):
|
|
1218
|
+
barstate.ishistory = False
|
|
1219
|
+
barstate.isrealtime = True
|
|
1220
|
+
|
|
1221
|
+
_run_script_main()
|
|
1222
|
+
# Advance the HA recurrence for this confirmed bar; the batch
|
|
1223
|
+
# ``save()`` below captures the last bar's HA as the baseline.
|
|
1224
|
+
_ha_commit()
|
|
1225
|
+
|
|
1226
|
+
if is_ltf and bar_time_ms < ltf_period_start:
|
|
1227
|
+
ltf_prefix_len = ltf_buffer_len() # noqa - non-None when is_ltf
|
|
1228
|
+
current_bar += 1
|
|
1229
|
+
bars_run = True
|
|
1230
|
+
|
|
1231
|
+
if bars_run:
|
|
1232
|
+
snap = _ensure_snapshot()
|
|
1233
|
+
if snap is not None:
|
|
1234
|
+
snap.save()
|
|
1235
|
+
|
|
1236
|
+
if is_ltf:
|
|
1237
|
+
flush_fn(ltf_prefix_len)
|
|
1238
|
+
elif not bars_run and not plain_ltf:
|
|
1239
|
+
with result_locks[sec_id]:
|
|
1240
|
+
write_na(result_block, sync_block)
|
|
1241
|
+
|
|
1242
|
+
data_ready_event.set()
|
|
1243
|
+
done_event.set()
|
|
1244
|
+
|
|
1245
|
+
finally:
|
|
1246
|
+
cleanup()
|
|
1247
|
+
if reader is not None:
|
|
1248
|
+
reader.close()
|
|
1249
|
+
if live_streamer is not None:
|
|
1250
|
+
live_streamer.stop()
|
|
1251
|
+
result_block.close()
|
|
1252
|
+
sync_block.close()
|
|
1253
|
+
lib._lib_semaphore = False
|