opencode-pyneruntime 6.6.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- opencode_pyneruntime-6.6.4.dist-info/METADATA +281 -0
- opencode_pyneruntime-6.6.4.dist-info/RECORD +261 -0
- opencode_pyneruntime-6.6.4.dist-info/WHEEL +5 -0
- opencode_pyneruntime-6.6.4.dist-info/entry_points.txt +6 -0
- opencode_pyneruntime-6.6.4.dist-info/licenses/LICENSE +201 -0
- opencode_pyneruntime-6.6.4.dist-info/licenses/NOTICE +21 -0
- opencode_pyneruntime-6.6.4.dist-info/top_level.txt +1 -0
- pynecore/__init__.py +6 -0
- pynecore/cli/__init__.py +2 -0
- pynecore/cli/app.py +238 -0
- pynecore/cli/commands/__init__.py +343 -0
- pynecore/cli/commands/benchmark.py +186 -0
- pynecore/cli/commands/compile.py +198 -0
- pynecore/cli/commands/data.py +857 -0
- pynecore/cli/commands/debug.py +63 -0
- pynecore/cli/commands/optimize.py +956 -0
- pynecore/cli/commands/plugin.py +242 -0
- pynecore/cli/commands/run.py +2006 -0
- pynecore/cli/pluggable.py +132 -0
- pynecore/cli/utils/__init__.py +0 -0
- pynecore/cli/utils/api_error_handler.py +168 -0
- pynecore/cli/utils/broker_picker.py +330 -0
- pynecore/cli/utils/error_hook.py +28 -0
- pynecore/cli/utils/keyreader.py +178 -0
- pynecore/cli/utils/provider_picker.py +19 -0
- pynecore/cli/utils/symbol_browser.py +1149 -0
- pynecore/core/__init__.py +0 -0
- pynecore/core/aggregator.py +257 -0
- pynecore/core/bar_magnifier.py +168 -0
- pynecore/core/broker/__init__.py +64 -0
- pynecore/core/broker/defaults.py +113 -0
- pynecore/core/broker/disappearance.py +927 -0
- pynecore/core/broker/emulator.py +345 -0
- pynecore/core/broker/exceptions.py +346 -0
- pynecore/core/broker/idempotency.py +401 -0
- pynecore/core/broker/intent_builder.py +334 -0
- pynecore/core/broker/journal.py +1785 -0
- pynecore/core/broker/models.py +1600 -0
- pynecore/core/broker/native_failsafe_manager.py +1436 -0
- pynecore/core/broker/one_way_emulator.py +1128 -0
- pynecore/core/broker/position.py +787 -0
- pynecore/core/broker/run_identity.py +126 -0
- pynecore/core/broker/software_entry_stop_engine.py +351 -0
- pynecore/core/broker/software_partial_bracket_engine.py +1379 -0
- pynecore/core/broker/spot_inventory.py +1327 -0
- pynecore/core/broker/storage.py +2655 -0
- pynecore/core/broker/store_helpers.py +2161 -0
- pynecore/core/broker/sync_engine.py +16070 -0
- pynecore/core/broker/validation.py +382 -0
- pynecore/core/class_property.py +7 -0
- pynecore/core/config.py +392 -0
- pynecore/core/csv_file.py +547 -0
- pynecore/core/currency.py +262 -0
- pynecore/core/data_converter.py +1002 -0
- pynecore/core/datetime.py +296 -0
- pynecore/core/download_info.py +71 -0
- pynecore/core/download_runner.py +274 -0
- pynecore/core/htf_aggregator.py +181 -0
- pynecore/core/import_hook.py +358 -0
- pynecore/core/instance_state.py +494 -0
- pynecore/core/live_ltf_collector.py +442 -0
- pynecore/core/live_ltf_window.py +189 -0
- pynecore/core/live_runner.py +1347 -0
- pynecore/core/module_property.py +26 -0
- pynecore/core/ohlcv_file.py +1888 -0
- pynecore/core/overload.py +371 -0
- pynecore/core/pine_cast.py +113 -0
- pynecore/core/pine_export.py +95 -0
- pynecore/core/pine_method.py +244 -0
- pynecore/core/pine_range.py +86 -0
- pynecore/core/pine_udt.py +69 -0
- pynecore/core/plugin/__init__.py +394 -0
- pynecore/core/plugin/broker.py +781 -0
- pynecore/core/plugin/cli.py +96 -0
- pynecore/core/plugin/live_provider.py +208 -0
- pynecore/core/plugin/provider.py +331 -0
- pynecore/core/provider_string.py +148 -0
- pynecore/core/random.py +40 -0
- pynecore/core/resampler.py +686 -0
- pynecore/core/safe_convert.py +64 -0
- pynecore/core/script.py +1011 -0
- pynecore/core/script_runner.py +3202 -0
- pynecore/core/security.py +1749 -0
- pynecore/core/security_process.py +1253 -0
- pynecore/core/security_shm.py +456 -0
- pynecore/core/series.py +417 -0
- pynecore/core/strategy_stats.py +669 -0
- pynecore/core/symbol_map.py +134 -0
- pynecore/core/syminfo.py +505 -0
- pynecore/core/viz.py +591 -0
- pynecore/lib/__init__.py +1771 -0
- pynecore/lib/_fixnan.py +32 -0
- pynecore/lib/_math_stateful.py +202 -0
- pynecore/lib/_timeframe_change.py +101 -0
- pynecore/lib/adjustment.py +6 -0
- pynecore/lib/alert.py +39 -0
- pynecore/lib/alert.pyi +14 -0
- pynecore/lib/array.py +1051 -0
- pynecore/lib/barmerge.py +60 -0
- pynecore/lib/barstate.py +30 -0
- pynecore/lib/box.py +415 -0
- pynecore/lib/chart.py +128 -0
- pynecore/lib/color.py +152 -0
- pynecore/lib/color.pyi +50 -0
- pynecore/lib/currency.py +62 -0
- pynecore/lib/dayofweek.py +36 -0
- pynecore/lib/dayofweek.pyi +18 -0
- pynecore/lib/display.py +8 -0
- pynecore/lib/dividends.py +9 -0
- pynecore/lib/earnings.py +11 -0
- pynecore/lib/extend.py +6 -0
- pynecore/lib/font.py +5 -0
- pynecore/lib/footprint.py +79 -0
- pynecore/lib/format.py +11 -0
- pynecore/lib/hline.py +67 -0
- pynecore/lib/hline.pyi +24 -0
- pynecore/lib/label.py +409 -0
- pynecore/lib/line.py +433 -0
- pynecore/lib/linefill.py +93 -0
- pynecore/lib/location.py +11 -0
- pynecore/lib/log.py +362 -0
- pynecore/lib/map.py +150 -0
- pynecore/lib/math.py +385 -0
- pynecore/lib/matrix.py +708 -0
- pynecore/lib/order.py +8 -0
- pynecore/lib/pivotpointtype.py +8 -0
- pynecore/lib/plot.py +95 -0
- pynecore/lib/plot.pyi +33 -0
- pynecore/lib/polyline.py +91 -0
- pynecore/lib/position.py +15 -0
- pynecore/lib/request.py +281 -0
- pynecore/lib/runtime.py +5 -0
- pynecore/lib/scale.py +9 -0
- pynecore/lib/session.py +267 -0
- pynecore/lib/session.pyi +12 -0
- pynecore/lib/shape.py +18 -0
- pynecore/lib/size.py +12 -0
- pynecore/lib/splits.py +4 -0
- pynecore/lib/strategy/__init__.py +4778 -0
- pynecore/lib/strategy/closedtrades.py +347 -0
- pynecore/lib/strategy/closedtrades.pyi +53 -0
- pynecore/lib/strategy/commission.py +9 -0
- pynecore/lib/strategy/direction.py +9 -0
- pynecore/lib/strategy/oca.py +13 -0
- pynecore/lib/strategy/opentrades.py +281 -0
- pynecore/lib/strategy/opentrades.pyi +49 -0
- pynecore/lib/strategy/risk.py +109 -0
- pynecore/lib/string.py +649 -0
- pynecore/lib/syminfo.py +84 -0
- pynecore/lib/ta.py +2230 -0
- pynecore/lib/table.py +290 -0
- pynecore/lib/text.py +17 -0
- pynecore/lib/ticker.py +207 -0
- pynecore/lib/timeframe.py +293 -0
- pynecore/lib/volume_row.py +67 -0
- pynecore/lib/xloc.py +4 -0
- pynecore/lib/yloc.py +5 -0
- pynecore/providers/__init__.py +0 -0
- pynecore/providers/ccxt.py +664 -0
- pynecore/providers/replay.py +187 -0
- pynecore/pynesys/__init__.py +0 -0
- pynecore/pynesys/api.py +498 -0
- pynecore/pynesys/compiler.py +112 -0
- pynecore/standalone.py +99 -0
- pynecore/testing/__init__.py +1 -0
- pynecore/testing/broker_lab/__init__.py +41 -0
- pynecore/testing/broker_lab/__main__.py +5 -0
- pynecore/testing/broker_lab/cli.py +87 -0
- pynecore/testing/broker_lab/generate.py +47 -0
- pynecore/testing/broker_lab/model.py +84 -0
- pynecore/testing/broker_lab/reference.py +645 -0
- pynecore/testing/broker_lab/runner.py +372 -0
- pynecore/testing/broker_lab/scheduler.py +50 -0
- pynecore/testing/broker_lab/subprocess.py +73 -0
- pynecore/transformers/__init__.py +0 -0
- pynecore/transformers/builtin_shadow.py +136 -0
- pynecore/transformers/closure_arguments_transformer.py +428 -0
- pynecore/transformers/display_rewrite.py +140 -0
- pynecore/transformers/dynamic_default.py +147 -0
- pynecore/transformers/function_isolation.py +757 -0
- pynecore/transformers/import_lifter.py +61 -0
- pynecore/transformers/import_normalizer.py +328 -0
- pynecore/transformers/inline_series_hoist.py +178 -0
- pynecore/transformers/input_transformer.py +175 -0
- pynecore/transformers/lib_series.py +201 -0
- pynecore/transformers/locations.py +70 -0
- pynecore/transformers/module_properties.json +3387 -0
- pynecore/transformers/module_property.py +221 -0
- pynecore/transformers/ne_guard.py +70 -0
- pynecore/transformers/persistent.py +320 -0
- pynecore/transformers/persistent_series.py +76 -0
- pynecore/transformers/safe_convert_transformer.py +97 -0
- pynecore/transformers/safe_division_transformer.py +95 -0
- pynecore/transformers/script_requirements.py +308 -0
- pynecore/transformers/security.py +752 -0
- pynecore/transformers/security_instantiation.py +274 -0
- pynecore/transformers/series.py +275 -0
- pynecore/transformers/slot_layout.py +381 -0
- pynecore/transformers/type_checking_stripper.py +25 -0
- pynecore/transformers/unused_series_detector.py +267 -0
- pynecore/types/__init__.py +21 -0
- pynecore/types/alert.py +5 -0
- pynecore/types/barmerge.py +5 -0
- pynecore/types/base.py +39 -0
- pynecore/types/box.py +37 -0
- pynecore/types/chart.py +17 -0
- pynecore/types/color.py +107 -0
- pynecore/types/currency.py +5 -0
- pynecore/types/datetime.py +6 -0
- pynecore/types/display.py +5 -0
- pynecore/types/dividends.py +5 -0
- pynecore/types/earnings.py +5 -0
- pynecore/types/extend.py +5 -0
- pynecore/types/font.py +5 -0
- pynecore/types/footprint.py +41 -0
- pynecore/types/format.py +5 -0
- pynecore/types/hline.py +24 -0
- pynecore/types/ib_persistent.py +8 -0
- pynecore/types/ib_persistent.pyi +10 -0
- pynecore/types/label.py +35 -0
- pynecore/types/line.py +32 -0
- pynecore/types/linefill.py +13 -0
- pynecore/types/location.py +5 -0
- pynecore/types/matrix.py +999 -0
- pynecore/types/na.py +237 -0
- pynecore/types/na.pyi +83 -0
- pynecore/types/ohlcv.py +12 -0
- pynecore/types/order.py +5 -0
- pynecore/types/persistent.py +8 -0
- pynecore/types/persistent.pyi +13 -0
- pynecore/types/pine_types.py +11 -0
- pynecore/types/pine_types.pyi +15 -0
- pynecore/types/pivotpointtype.py +5 -0
- pynecore/types/plot.py +12 -0
- pynecore/types/plot_meta.py +60 -0
- pynecore/types/polyline.py +40 -0
- pynecore/types/position.py +5 -0
- pynecore/types/scale.py +5 -0
- pynecore/types/script_type.py +15 -0
- pynecore/types/series.py +23 -0
- pynecore/types/series.pyi +19 -0
- pynecore/types/session.py +35 -0
- pynecore/types/shape.py +5 -0
- pynecore/types/size.py +5 -0
- pynecore/types/source.py +33 -0
- pynecore/types/splits.py +5 -0
- pynecore/types/strategy.py +45 -0
- pynecore/types/table.py +87 -0
- pynecore/types/text.py +13 -0
- pynecore/types/type_checker.py +7 -0
- pynecore/types/type_checker.pyi +48 -0
- pynecore/types/volume_row.py +36 -0
- pynecore/types/weekdays.py +11 -0
- pynecore/types/xloc.py +5 -0
- pynecore/types/yloc.py +5 -0
- pynecore/utils/__init__.py +0 -0
- pynecore/utils/file_utils.py +50 -0
- pynecore/utils/rich/__init__.py +0 -0
- pynecore/utils/rich/date_column.py +25 -0
- pynecore/utils/sequence_view.py +92 -0
- pynecore/utils/stdlib_checker.py +17 -0
|
@@ -0,0 +1,1327 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Spot-venue inventory accounting: execution ledger, balance-invariant
|
|
3
|
+
reconciliation and position synthesis.
|
|
4
|
+
|
|
5
|
+
Spot venues expose no position object — the base-asset inventory *is*
|
|
6
|
+
the long exposure, and the venue balance pools the bot's inventory with
|
|
7
|
+
pre-existing holdings, manual trades and deposits. This module owns the
|
|
8
|
+
core bookkeeping a spot broker plugin needs:
|
|
9
|
+
|
|
10
|
+
- **Append-only execution ledger** (``spot_executions``): every venue
|
|
11
|
+
fill as signed base/quote deltas in exact decimal arithmetic, deduped
|
|
12
|
+
on the venue fill id, exempt from retention (an open position must
|
|
13
|
+
stay reconstructible for as long as it is open).
|
|
14
|
+
- **Inventory epoch** (``spot_inventory_epoch``): the reconciliation
|
|
15
|
+
baseline generation. The invariant is
|
|
16
|
+
``expected_total = foreign_baseline + bot_inventory(ledger)`` where
|
|
17
|
+
``foreign_baseline`` was frozen at epoch creation. Any unexplainable
|
|
18
|
+
drift in EITHER direction is an attribution conflict — a positive
|
|
19
|
+
drift can mask an external sale netted against a deposit, so
|
|
20
|
+
warn-and-continue is not an option (fail-closed both ways).
|
|
21
|
+
- **Asset-ownership lease** (``spot_asset_owner``): one active logical
|
|
22
|
+
run per ``(plugin, account, base asset)`` within a broker store; a
|
|
23
|
+
concurrent second run starts quarantined instead of double-booking.
|
|
24
|
+
- **Exactly-once ledger→engine handoff**: a live fill is recorded and
|
|
25
|
+
outbox-flipped (``delivered``) in one transaction before the plugin
|
|
26
|
+
emits its :class:`~pynecore.core.broker.models.OrderEvent`; on
|
|
27
|
+
restart the startup adoption folds the ENTIRE ledger into the
|
|
28
|
+
synthesized position the engine adopts, so every fill reaches the
|
|
29
|
+
engine on exactly one path — never both, never neither.
|
|
30
|
+
- **Quarantine, not adoption**: external intervention in the bot's
|
|
31
|
+
inventory is unsupported by contract. A confirmed conflict stops
|
|
32
|
+
trading via the engine quarantine latch (process stays alive as an
|
|
33
|
+
observer); the explicit ``halt`` policy exits instead. Recovery is an
|
|
34
|
+
operator ``rebaseline`` (new epoch, one transaction) plus restart.
|
|
35
|
+
|
|
36
|
+
The manager takes explicit ``now_ms`` timestamps so reconciliation
|
|
37
|
+
timing is fully deterministic under test, mirroring
|
|
38
|
+
:class:`~pynecore.core.broker.disappearance.DisappearanceTracker`.
|
|
39
|
+
"""
|
|
40
|
+
import logging
|
|
41
|
+
import math
|
|
42
|
+
from dataclasses import dataclass
|
|
43
|
+
from decimal import Decimal, InvalidOperation
|
|
44
|
+
from typing import TYPE_CHECKING, Any, Final, Protocol
|
|
45
|
+
|
|
46
|
+
from pynecore.core.broker.exceptions import SpotInventoryConflictError
|
|
47
|
+
from pynecore.core.broker.models import ExchangePosition
|
|
48
|
+
from pynecore.core.broker.store_helpers import PENDING_DISPATCH_STATES
|
|
49
|
+
|
|
50
|
+
if TYPE_CHECKING:
|
|
51
|
+
from collections.abc import Callable
|
|
52
|
+
from pynecore.core.broker.storage import (
|
|
53
|
+
RunContext,
|
|
54
|
+
SpotEpochRow,
|
|
55
|
+
SpotExecutionRow,
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
__all__ = [
|
|
59
|
+
'INVENTORY_CONFLICT_POLICIES',
|
|
60
|
+
'SpotExecution',
|
|
61
|
+
'SpotExecutionBatch',
|
|
62
|
+
'SpotInventoryPort',
|
|
63
|
+
'InventoryFold',
|
|
64
|
+
'SpotStartupResult',
|
|
65
|
+
'SpotInventoryManager',
|
|
66
|
+
'canonical_decimal',
|
|
67
|
+
'fold_inventory',
|
|
68
|
+
]
|
|
69
|
+
|
|
70
|
+
logger = logging.getLogger(__name__)
|
|
71
|
+
|
|
72
|
+
#: Valid ``on_inventory_conflict`` policies. Deliberately narrower than
|
|
73
|
+
#: the ``on_unexpected_cancel`` set: ``re_place`` would buy back an
|
|
74
|
+
#: operator's withdrawal and ``ignore`` would trade on corrupt books, so
|
|
75
|
+
#: neither is applicable to an attribution conflict.
|
|
76
|
+
INVENTORY_CONFLICT_POLICIES: Final = ('quarantine', 'halt')
|
|
77
|
+
|
|
78
|
+
#: Valid cursor scopes a port may declare for its execution-history API.
|
|
79
|
+
CURSOR_SCOPES: Final = ('account', 'product', 'time')
|
|
80
|
+
|
|
81
|
+
#: Hard cap on catch-up pagination per invocation — a runaway venue
|
|
82
|
+
#: cursor (next_cursor never converging) must not loop forever.
|
|
83
|
+
_MAX_CATCHUP_PAGES: Final = 10_000
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class _ForeignLedgerRow(Exception):
|
|
87
|
+
"""Internal signal: a fill id is booked under another logical run.
|
|
88
|
+
|
|
89
|
+
Raised INSIDE a ledger transaction (which rolls back) and converted
|
|
90
|
+
to the quarantine + :class:`SpotInventoryConflictError` OUTSIDE it —
|
|
91
|
+
the quarantine's own store writes must not ride the aborted span.
|
|
92
|
+
"""
|
|
93
|
+
|
|
94
|
+
def __init__(self, fill_id: str, owner_run_id: str) -> None:
|
|
95
|
+
super().__init__(fill_id)
|
|
96
|
+
self.fill_id = fill_id
|
|
97
|
+
self.owner_run_id = owner_run_id
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def canonical_decimal(value: Decimal | int | str) -> str:
|
|
101
|
+
"""Serialize a decimal to its canonical ledger string.
|
|
102
|
+
|
|
103
|
+
Canonical form: plain (exponent-free) notation, no trailing zeros,
|
|
104
|
+
``-0`` collapsed to ``0`` — one value, one string, so ledger rows
|
|
105
|
+
compare and round-trip exactly. Rejects non-finite values and
|
|
106
|
+
anything :class:`~decimal.Decimal` cannot parse exactly. Floats are
|
|
107
|
+
deliberately not accepted: binary floats carry representation error
|
|
108
|
+
that must not enter an exact ledger — the caller converts via
|
|
109
|
+
``Decimal(str(x))`` explicitly if a float source is unavoidable.
|
|
110
|
+
|
|
111
|
+
The trailing-zero strip is TEXTUAL, not ``Decimal.normalize()``:
|
|
112
|
+
``normalize`` rounds to the ambient decimal context (28 significant
|
|
113
|
+
digits by default), which would silently corrupt a higher-precision
|
|
114
|
+
value. ``format(d, 'f')`` renders the exact stored digits with no
|
|
115
|
+
context rounding, so the round trip stays exact at any precision.
|
|
116
|
+
|
|
117
|
+
:raises ValueError: On non-finite or unparseable input.
|
|
118
|
+
"""
|
|
119
|
+
if isinstance(value, float):
|
|
120
|
+
raise ValueError(
|
|
121
|
+
"canonical_decimal: float input is not accepted; convert "
|
|
122
|
+
"explicitly (Decimal(str(x))) so the representation choice "
|
|
123
|
+
"is the caller's"
|
|
124
|
+
)
|
|
125
|
+
try:
|
|
126
|
+
d = Decimal(value)
|
|
127
|
+
except InvalidOperation as exc:
|
|
128
|
+
raise ValueError(f"canonical_decimal: unparseable value {value!r}") from exc
|
|
129
|
+
if not d.is_finite():
|
|
130
|
+
raise ValueError(f"canonical_decimal: non-finite value {value!r}")
|
|
131
|
+
if d == 0:
|
|
132
|
+
return '0'
|
|
133
|
+
s = format(d, 'f')
|
|
134
|
+
if '.' in s:
|
|
135
|
+
s = s.rstrip('0').rstrip('.')
|
|
136
|
+
return s
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@dataclass(frozen=True)
|
|
140
|
+
class SpotExecution:
|
|
141
|
+
"""One venue execution (fill), in exact decimal arithmetic.
|
|
142
|
+
|
|
143
|
+
The canonical delta equations the plugin must apply when building
|
|
144
|
+
these from the venue's raw fill report:
|
|
145
|
+
|
|
146
|
+
- ``base_delta`` = signed executed base quantity, minus any fee
|
|
147
|
+
charged in the BASE currency (a base fee reduces what was actually
|
|
148
|
+
received). Positive on a buy, negative on a sell.
|
|
149
|
+
- ``quote_delta`` = the opposite-signed notional, minus any fee
|
|
150
|
+
charged in the QUOTE currency. Negative on a buy (quote spent),
|
|
151
|
+
positive on a sell (quote received).
|
|
152
|
+
- A fee charged in a third currency is recorded (``fee_amount`` /
|
|
153
|
+
``fee_currency``) but does not touch either delta — it does not
|
|
154
|
+
move the base invariant.
|
|
155
|
+
|
|
156
|
+
Validation is fail-closed: a fill that cannot be represented
|
|
157
|
+
exactly, or whose signs contradict its side, raises at construction
|
|
158
|
+
instead of corrupting the ledger.
|
|
159
|
+
|
|
160
|
+
:ivar venue_seq: The venue's monotonic execution-sequence number
|
|
161
|
+
when it exposes one, else ``None``. Breaks fold-ordering ties
|
|
162
|
+
within one millisecond; a venue whose fills can share a
|
|
163
|
+
millisecond MUST provide it, or a same-ms buy/sell pair may
|
|
164
|
+
replay reversed into a false oversell.
|
|
165
|
+
"""
|
|
166
|
+
fill_id: str
|
|
167
|
+
side: str # "buy" | "sell"
|
|
168
|
+
base_delta: Decimal
|
|
169
|
+
quote_delta: Decimal
|
|
170
|
+
price: Decimal
|
|
171
|
+
fee_amount: Decimal
|
|
172
|
+
fee_currency: str
|
|
173
|
+
ts_ms: int
|
|
174
|
+
exchange_order_id: str | None = None
|
|
175
|
+
client_order_id: str | None = None
|
|
176
|
+
venue_seq: int | None = None
|
|
177
|
+
|
|
178
|
+
def __post_init__(self) -> None:
|
|
179
|
+
if not self.fill_id:
|
|
180
|
+
raise ValueError("SpotExecution: empty fill_id")
|
|
181
|
+
if not self.client_order_id:
|
|
182
|
+
# Attribution floor: the ledger tracks the BOT's inventory,
|
|
183
|
+
# not the account's. The bot's OWN client_order_id is the only
|
|
184
|
+
# reference it controls — a raw exchange_order_id is not proof
|
|
185
|
+
# of bot ownership (a manual/web trade carries one too). The
|
|
186
|
+
# port must attribute each fill to a bot order and set its
|
|
187
|
+
# client_order_id (mapping via its own order records when the
|
|
188
|
+
# venue does not echo it); a fill it cannot attribute belongs
|
|
189
|
+
# to ``conclusive=False`` history, not the ledger. Booking an
|
|
190
|
+
# unattributed fill would move balance AND ledger together, so
|
|
191
|
+
# the invariant would never fire.
|
|
192
|
+
raise ValueError(
|
|
193
|
+
f"SpotExecution {self.fill_id!r}: needs the bot's own "
|
|
194
|
+
f"client_order_id — an execution not attributable to a bot "
|
|
195
|
+
f"dispatch must not enter the inventory ledger (an "
|
|
196
|
+
f"exchange_order_id alone is not proof of bot ownership)"
|
|
197
|
+
)
|
|
198
|
+
if self.side not in ('buy', 'sell'):
|
|
199
|
+
raise ValueError(
|
|
200
|
+
f"SpotExecution {self.fill_id!r}: unknown side {self.side!r}"
|
|
201
|
+
)
|
|
202
|
+
for name in ('base_delta', 'quote_delta', 'price', 'fee_amount'):
|
|
203
|
+
v = getattr(self, name)
|
|
204
|
+
if not isinstance(v, Decimal) or not v.is_finite():
|
|
205
|
+
raise ValueError(
|
|
206
|
+
f"SpotExecution {self.fill_id!r}: {name} must be a "
|
|
207
|
+
f"finite Decimal, got {v!r}"
|
|
208
|
+
)
|
|
209
|
+
if self.price <= 0:
|
|
210
|
+
raise ValueError(
|
|
211
|
+
f"SpotExecution {self.fill_id!r}: non-positive price "
|
|
212
|
+
f"{self.price}"
|
|
213
|
+
)
|
|
214
|
+
if self.fee_amount < 0:
|
|
215
|
+
raise ValueError(
|
|
216
|
+
f"SpotExecution {self.fill_id!r}: negative fee "
|
|
217
|
+
f"{self.fee_amount}"
|
|
218
|
+
)
|
|
219
|
+
if self.side == 'buy' and (self.base_delta <= 0 or self.quote_delta > 0):
|
|
220
|
+
raise ValueError(
|
|
221
|
+
f"SpotExecution {self.fill_id!r}: buy requires "
|
|
222
|
+
f"base_delta > 0 and quote_delta <= 0, got "
|
|
223
|
+
f"base={self.base_delta} quote={self.quote_delta}"
|
|
224
|
+
)
|
|
225
|
+
if self.side == 'sell' and (self.base_delta >= 0 or self.quote_delta < 0):
|
|
226
|
+
raise ValueError(
|
|
227
|
+
f"SpotExecution {self.fill_id!r}: sell requires "
|
|
228
|
+
f"base_delta < 0 and quote_delta >= 0, got "
|
|
229
|
+
f"base={self.base_delta} quote={self.quote_delta}"
|
|
230
|
+
)
|
|
231
|
+
if self.ts_ms <= 0:
|
|
232
|
+
raise ValueError(
|
|
233
|
+
f"SpotExecution {self.fill_id!r}: invalid ts_ms {self.ts_ms}"
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
@dataclass(frozen=True)
|
|
238
|
+
class SpotExecutionBatch:
|
|
239
|
+
"""One page of a port's execution-history read.
|
|
240
|
+
|
|
241
|
+
:ivar executions: The fills in this page (any order; the ledger
|
|
242
|
+
sorts deterministically).
|
|
243
|
+
:ivar next_cursor: Durable cursor to persist once every execution in
|
|
244
|
+
this page is recorded. ``None`` keeps the previous cursor.
|
|
245
|
+
:ivar has_more: ``True`` when another page must be fetched with
|
|
246
|
+
``next_cursor`` before catch-up is complete.
|
|
247
|
+
:ivar conclusive: ``False`` when the venue could not provide an
|
|
248
|
+
authoritative answer (endpoint degraded, history window
|
|
249
|
+
truncated). An inconclusive catch-up fails closed at startup.
|
|
250
|
+
"""
|
|
251
|
+
executions: tuple[SpotExecution, ...] = ()
|
|
252
|
+
next_cursor: str | None = None
|
|
253
|
+
has_more: bool = False
|
|
254
|
+
conclusive: bool = True
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
class SpotInventoryPort(Protocol):
|
|
258
|
+
"""Surface a spot broker plugin exposes to the core inventory manager.
|
|
259
|
+
|
|
260
|
+
Attributes are venue/product facts fixed for the run; the two
|
|
261
|
+
methods are the venue reads the reconciliation loop drives. All
|
|
262
|
+
quantities are exact :class:`~decimal.Decimal` — the port owns the
|
|
263
|
+
parse from the venue's wire format.
|
|
264
|
+
"""
|
|
265
|
+
|
|
266
|
+
product_id: str
|
|
267
|
+
"""Venue product identifier (e.g. ``"BTC-USD"``) — the ledger and
|
|
268
|
+
epoch key. Not necessarily the Pine symbol."""
|
|
269
|
+
|
|
270
|
+
base_asset: str
|
|
271
|
+
"""Base asset code (e.g. ``"BTC"``) — the exclusive position asset."""
|
|
272
|
+
|
|
273
|
+
quote_asset: str
|
|
274
|
+
"""Quote asset code (e.g. ``"USD"``) — the shared cash asset."""
|
|
275
|
+
|
|
276
|
+
cursor_scope: str
|
|
277
|
+
"""Scope of the execution-history cursor: ``'account'``,
|
|
278
|
+
``'product'`` or ``'time'``. Persisted with the cursor so a plugin
|
|
279
|
+
upgrade that changes the scope invalidates the stored cursor instead
|
|
280
|
+
of silently misreading it. Time-based APIs must fetch with an
|
|
281
|
+
overlapping window (the ledger dedups on fill id) rather than assume
|
|
282
|
+
a strict cursor."""
|
|
283
|
+
|
|
284
|
+
base_tolerance: Decimal
|
|
285
|
+
"""Asset-specific quantization tolerance for the balance invariant —
|
|
286
|
+
small and fixed (venue rounding of the total balance), NEVER a
|
|
287
|
+
settlement-lag allowance. Settlement lag is handled as a *temporal*
|
|
288
|
+
grace state, not a numeric widening."""
|
|
289
|
+
|
|
290
|
+
settlement_grace_s: float
|
|
291
|
+
"""How long a balance-invariant mismatch may persist (armed as a
|
|
292
|
+
pending conflict, re-checked with fresh executions + balance) before
|
|
293
|
+
it is confirmed as a conflict. Venue settlement latency, not a
|
|
294
|
+
tuning knob for hiding drift."""
|
|
295
|
+
|
|
296
|
+
position_dust_threshold: Decimal
|
|
297
|
+
"""Positive base quantity below which the venue intentionally exposes
|
|
298
|
+
no engine position. This is normally the product's minimum quantity
|
|
299
|
+
increment. A zero value disables dust-to-flat reconciliation."""
|
|
300
|
+
|
|
301
|
+
async def fetch_executions(self, cursor: str | None) -> SpotExecutionBatch:
|
|
302
|
+
"""Read the BOT's execution history from ``cursor``.
|
|
303
|
+
|
|
304
|
+
MUST return only executions attributable to THIS bot — every
|
|
305
|
+
:class:`SpotExecution` carries the bot's own ``client_order_id``
|
|
306
|
+
(map it from the venue fill's order id via the plugin's order
|
|
307
|
+
records when the venue does not echo it; a raw exchange order id
|
|
308
|
+
is not proof of bot ownership). The ledger accounts for the bot's
|
|
309
|
+
own inventory, not the account's: a manual or foreign trade folded
|
|
310
|
+
in as if it were the bot's would move the balance AND the ledger
|
|
311
|
+
together, so the invariant would never fire. A venue whose
|
|
312
|
+
account-history endpoint cannot be filtered/attributed to bot
|
|
313
|
+
orders MUST return ``conclusive=False`` (which fails closed)
|
|
314
|
+
rather than dump raw account trades.
|
|
315
|
+
|
|
316
|
+
``cursor=None`` means "no history belongs to the bot yet" (first
|
|
317
|
+
startup): return an EMPTY batch whose ``next_cursor`` anchors at
|
|
318
|
+
the venue's current watermark — the account's prior history is
|
|
319
|
+
foreign inventory and belongs to the epoch baseline, not the
|
|
320
|
+
ledger. Raise a transient error (connection) to abort the read;
|
|
321
|
+
return ``conclusive=False`` when the venue answered but cannot
|
|
322
|
+
be trusted as complete.
|
|
323
|
+
"""
|
|
324
|
+
...
|
|
325
|
+
|
|
326
|
+
async def fetch_base_balance(self) -> Decimal:
|
|
327
|
+
"""The account's TOTAL owned base-asset amount.
|
|
328
|
+
|
|
329
|
+
Must include available balance PLUS amounts locked in open
|
|
330
|
+
(sell) orders PLUS pending settlement — the invariant compares
|
|
331
|
+
against total ownership; an available-only read would false-fire
|
|
332
|
+
the moment a sell order rests. Raise on read failure (transient
|
|
333
|
+
errors skip the check cycle; at startup they fail closed).
|
|
334
|
+
"""
|
|
335
|
+
...
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
@dataclass(frozen=True)
|
|
339
|
+
class InventoryFold:
|
|
340
|
+
"""Result of folding the ledger into net inventory + cost basis.
|
|
341
|
+
|
|
342
|
+
``net_base`` is exact (pure decimal addition). ``cost_quote`` is the
|
|
343
|
+
quote actually spent on the current inventory (fees included via the
|
|
344
|
+
canonical deltas), reduced proportionally on partial sells — plain
|
|
345
|
+
VWAP cost-basis, matching the engine's realized-P&L expectations.
|
|
346
|
+
``violation`` reports an oversell: the ledger's sells exceed its
|
|
347
|
+
buys at some point, which spot cannot legitimately do — bookkeeping
|
|
348
|
+
corruption, handled fail-closed by the caller.
|
|
349
|
+
"""
|
|
350
|
+
net_base: Decimal = Decimal(0)
|
|
351
|
+
cost_quote: Decimal = Decimal(0)
|
|
352
|
+
fill_count: int = 0
|
|
353
|
+
violation: str | None = None
|
|
354
|
+
|
|
355
|
+
@property
|
|
356
|
+
def vwap(self) -> Decimal | None:
|
|
357
|
+
if self.net_base <= 0 or self.cost_quote <= 0:
|
|
358
|
+
return None
|
|
359
|
+
return self.cost_quote / self.net_base
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def fold_inventory(rows: 'list[SpotExecutionRow]') -> InventoryFold:
|
|
363
|
+
"""Fold ledger rows (oldest first) into net inventory and cost basis.
|
|
364
|
+
|
|
365
|
+
Buys add their net base delta and their absolute quote delta to the
|
|
366
|
+
cost basis; sells reduce the basis proportionally to the fraction of
|
|
367
|
+
inventory sold. An exact flat resets the basis to zero. A sell
|
|
368
|
+
exceeding the running inventory marks the fold ``violation`` (the
|
|
369
|
+
arithmetic still completes so the caller can report the terminal
|
|
370
|
+
state).
|
|
371
|
+
"""
|
|
372
|
+
inv = Decimal(0)
|
|
373
|
+
cost = Decimal(0)
|
|
374
|
+
violation: str | None = None
|
|
375
|
+
for row in rows:
|
|
376
|
+
base_delta = Decimal(row.base_delta)
|
|
377
|
+
quote_delta = Decimal(row.quote_delta)
|
|
378
|
+
if base_delta > 0:
|
|
379
|
+
inv += base_delta
|
|
380
|
+
cost += -quote_delta
|
|
381
|
+
else:
|
|
382
|
+
sold = -base_delta
|
|
383
|
+
if sold > inv:
|
|
384
|
+
if violation is None:
|
|
385
|
+
violation = (
|
|
386
|
+
f"sell {row.fill_id!r} of {sold} exceeds running "
|
|
387
|
+
f"inventory {inv}"
|
|
388
|
+
)
|
|
389
|
+
cost = Decimal(0)
|
|
390
|
+
elif sold == inv:
|
|
391
|
+
cost = Decimal(0)
|
|
392
|
+
else:
|
|
393
|
+
cost -= cost * (sold / inv)
|
|
394
|
+
inv -= sold
|
|
395
|
+
return InventoryFold(
|
|
396
|
+
net_base=inv,
|
|
397
|
+
cost_quote=cost,
|
|
398
|
+
fill_count=len(rows),
|
|
399
|
+
violation=violation,
|
|
400
|
+
)
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
@dataclass(frozen=True)
|
|
404
|
+
class SpotStartupResult:
|
|
405
|
+
"""Outcome of :meth:`SpotInventoryManager.startup`."""
|
|
406
|
+
quarantined: bool
|
|
407
|
+
reason: str | None
|
|
408
|
+
fold: InventoryFold
|
|
409
|
+
epoch: 'SpotEpochRow | None'
|
|
410
|
+
recovered_fills: int = 0
|
|
411
|
+
adopted_fills: int = 0
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
class SpotInventoryManager:
|
|
415
|
+
"""Core spot-inventory bookkeeping for one ``(account, product)``.
|
|
416
|
+
|
|
417
|
+
A spot plugin constructs one per run after authentication and drives
|
|
418
|
+
three touchpoints:
|
|
419
|
+
|
|
420
|
+
- :meth:`startup` once, before the engine's startup reconcile —
|
|
421
|
+
lease claim, execution catch-up, epoch/invariant validation,
|
|
422
|
+
adoption watermark. Fail-closed: any inconclusive read or
|
|
423
|
+
unexplainable drift quarantines before a single dispatch.
|
|
424
|
+
- :meth:`record_live_fill` for every fill observed on the live
|
|
425
|
+
stream, BEFORE emitting the corresponding
|
|
426
|
+
:class:`~pynecore.core.broker.models.OrderEvent` (the return value
|
|
427
|
+
says whether to emit — a dedup'd replay must not re-book).
|
|
428
|
+
- :meth:`reconcile` per poll cycle — lease heartbeat + balance
|
|
429
|
+
invariant with the persisted settlement-grace state machine. It
|
|
430
|
+
returns any fills a runtime catch-up recovered (a stream gap) so
|
|
431
|
+
the plugin can emit their events; the periodic engine reconcile
|
|
432
|
+
ignores position increases, so these would otherwise stay
|
|
433
|
+
invisible until the next restart.
|
|
434
|
+
|
|
435
|
+
``get_position()`` synthesis reads :meth:`synthesize_position`.
|
|
436
|
+
|
|
437
|
+
Quarantine delivery mirrors the disappearance tracker: the
|
|
438
|
+
``request_quarantine`` hook latches the engine (process stays alive,
|
|
439
|
+
ingestion keeps running); the ``halt`` policy — or a missing /
|
|
440
|
+
raising hook — arms :attr:`pending_halt`, which the plugin's event
|
|
441
|
+
stream raises so the run exits via the graceful
|
|
442
|
+
manual-intervention path. Never fail-open.
|
|
443
|
+
"""
|
|
444
|
+
|
|
445
|
+
def __init__(
|
|
446
|
+
self,
|
|
447
|
+
store_ctx: 'RunContext',
|
|
448
|
+
port: SpotInventoryPort,
|
|
449
|
+
*,
|
|
450
|
+
account_id: str,
|
|
451
|
+
symbol: str,
|
|
452
|
+
request_quarantine: 'Callable[[str, dict[str, Any]], None] | None' = None,
|
|
453
|
+
on_inventory_conflict: str = 'quarantine',
|
|
454
|
+
) -> None:
|
|
455
|
+
"""
|
|
456
|
+
:param store_ctx: Open run context of the unified broker store.
|
|
457
|
+
:param port: The plugin's venue surface.
|
|
458
|
+
:param account_id: The plugin's authenticated ``account_id`` —
|
|
459
|
+
the ledger's fill-id uniqueness dimension.
|
|
460
|
+
:param symbol: The Pine symbol the synthesized
|
|
461
|
+
:class:`~pynecore.core.broker.models.ExchangePosition`
|
|
462
|
+
carries (not necessarily ``port.product_id``).
|
|
463
|
+
:param request_quarantine: The engine quarantine latch, normally
|
|
464
|
+
the plugin's
|
|
465
|
+
:attr:`~pynecore.core.plugin.broker.BrokerPlugin.quarantine_sink`.
|
|
466
|
+
:param on_inventory_conflict: ``'quarantine'`` (default) or
|
|
467
|
+
``'halt'``.
|
|
468
|
+
"""
|
|
469
|
+
if on_inventory_conflict not in INVENTORY_CONFLICT_POLICIES:
|
|
470
|
+
raise ValueError(
|
|
471
|
+
f"on_inventory_conflict must be one of "
|
|
472
|
+
f"{INVENTORY_CONFLICT_POLICIES}, got "
|
|
473
|
+
f"{on_inventory_conflict!r}"
|
|
474
|
+
)
|
|
475
|
+
if port.cursor_scope not in CURSOR_SCOPES:
|
|
476
|
+
raise ValueError(
|
|
477
|
+
f"port.cursor_scope must be one of {CURSOR_SCOPES}, got "
|
|
478
|
+
f"{port.cursor_scope!r}"
|
|
479
|
+
)
|
|
480
|
+
if not isinstance(port.base_tolerance, Decimal) \
|
|
481
|
+
or not port.base_tolerance.is_finite() \
|
|
482
|
+
or port.base_tolerance < 0:
|
|
483
|
+
raise ValueError(
|
|
484
|
+
f"port.base_tolerance must be a finite non-negative "
|
|
485
|
+
f"Decimal, got {port.base_tolerance!r}"
|
|
486
|
+
)
|
|
487
|
+
dust_threshold = port.position_dust_threshold
|
|
488
|
+
if not isinstance(dust_threshold, Decimal) \
|
|
489
|
+
or not dust_threshold.is_finite() \
|
|
490
|
+
or dust_threshold < 0:
|
|
491
|
+
raise ValueError(
|
|
492
|
+
f"port.position_dust_threshold must be a finite non-negative "
|
|
493
|
+
f"Decimal, got {dust_threshold!r}"
|
|
494
|
+
)
|
|
495
|
+
grace = port.settlement_grace_s
|
|
496
|
+
if isinstance(grace, bool) \
|
|
497
|
+
or not isinstance(grace, (int, float)) \
|
|
498
|
+
or not math.isfinite(grace) \
|
|
499
|
+
or grace < 0:
|
|
500
|
+
# A NaN/inf grace makes the expiry comparison never true, so
|
|
501
|
+
# a confirmed conflict would stay pending forever while
|
|
502
|
+
# trading continues — fail closed at construction instead.
|
|
503
|
+
raise ValueError(
|
|
504
|
+
f"port.settlement_grace_s must be a finite non-negative "
|
|
505
|
+
f"real number, got {grace!r}"
|
|
506
|
+
)
|
|
507
|
+
self._store = store_ctx
|
|
508
|
+
self._port = port
|
|
509
|
+
self._account_id = account_id
|
|
510
|
+
self._symbol = symbol
|
|
511
|
+
self._request_quarantine = request_quarantine
|
|
512
|
+
self._policy = on_inventory_conflict
|
|
513
|
+
self._epoch: 'SpotEpochRow | None' = None
|
|
514
|
+
self._fold = InventoryFold()
|
|
515
|
+
self._quarantined = False
|
|
516
|
+
self._quarantine_reason: str | None = None
|
|
517
|
+
self._pending_halt: SpotInventoryConflictError | None = None
|
|
518
|
+
self._started = False
|
|
519
|
+
|
|
520
|
+
# --- Introspection ------------------------------------------------------
|
|
521
|
+
|
|
522
|
+
@property
|
|
523
|
+
def quarantined(self) -> bool:
|
|
524
|
+
return self._quarantined
|
|
525
|
+
|
|
526
|
+
@property
|
|
527
|
+
def quarantine_reason(self) -> str | None:
|
|
528
|
+
return self._quarantine_reason
|
|
529
|
+
|
|
530
|
+
@property
|
|
531
|
+
def pending_halt(self) -> SpotInventoryConflictError | None:
|
|
532
|
+
"""Armed process-exit signal (``halt`` policy or hook fallback).
|
|
533
|
+
|
|
534
|
+
The plugin's event stream checks this each pass and raises the
|
|
535
|
+
taken error so the engine's
|
|
536
|
+
:class:`~pynecore.core.broker.exceptions.BrokerManualInterventionError`
|
|
537
|
+
handling performs the graceful stop.
|
|
538
|
+
"""
|
|
539
|
+
return self._pending_halt
|
|
540
|
+
|
|
541
|
+
def consume_pending_halt(self) -> SpotInventoryConflictError | None:
|
|
542
|
+
"""Take (and clear) the armed halt — consume-once."""
|
|
543
|
+
halt = self._pending_halt
|
|
544
|
+
self._pending_halt = None
|
|
545
|
+
return halt
|
|
546
|
+
|
|
547
|
+
@property
|
|
548
|
+
def fold(self) -> InventoryFold:
|
|
549
|
+
"""The current in-memory inventory fold (refreshed on writes)."""
|
|
550
|
+
return self._fold
|
|
551
|
+
|
|
552
|
+
# --- Startup ------------------------------------------------------------
|
|
553
|
+
|
|
554
|
+
async def startup(self) -> SpotStartupResult:
|
|
555
|
+
"""Run the fail-closed startup sequence.
|
|
556
|
+
|
|
557
|
+
Order (plugin calls this after connect + auth, before the
|
|
558
|
+
engine's startup reconcile):
|
|
559
|
+
|
|
560
|
+
1. **Lease claim** — a live foreign lease on the base asset
|
|
561
|
+
means another logical run owns it: quarantine, touch nothing.
|
|
562
|
+
2. **Persisted quarantine check** — a ``quarantined`` epoch
|
|
563
|
+
survives restarts; only an operator :meth:`rebaseline`
|
|
564
|
+
clears it.
|
|
565
|
+
3. **Execution catch-up** from the epoch's durable cursor (the
|
|
566
|
+
crash window between a venue fill and its local persist is
|
|
567
|
+
closed here). Paged; each page commits its fills and the
|
|
568
|
+
advanced cursor in one transaction. Inconclusive → fail
|
|
569
|
+
closed.
|
|
570
|
+
4. **Epoch load / first-epoch freeze** — on the very first
|
|
571
|
+
startup the baseline is frozen as
|
|
572
|
+
``current_total − reconstructed bot inventory`` (NOT the raw
|
|
573
|
+
total: a crash after fills but before the first epoch write
|
|
574
|
+
must not launder those fills into the baseline).
|
|
575
|
+
5. **Balance invariant** — strict (the catch-up was conclusive,
|
|
576
|
+
so no settlement grace applies at startup).
|
|
577
|
+
6. **Adoption watermark** — every ledger row is folded into the
|
|
578
|
+
position the engine adopts, so all rows flip ``delivered``;
|
|
579
|
+
none may later re-enter as a live event.
|
|
580
|
+
"""
|
|
581
|
+
if self._started:
|
|
582
|
+
raise RuntimeError("SpotInventoryManager.startup() already ran")
|
|
583
|
+
self._started = True
|
|
584
|
+
port = self._port
|
|
585
|
+
|
|
586
|
+
# (1) Ownership lease. A live foreign lease on the base asset —
|
|
587
|
+
# or a base-vs-quote overlap with another live run trading the
|
|
588
|
+
# shared asset as cash — fails the claim: quarantine, touch
|
|
589
|
+
# nothing.
|
|
590
|
+
if not self._store.claim_spot_asset(
|
|
591
|
+
self._account_id, port.base_asset, port.quote_asset,
|
|
592
|
+
):
|
|
593
|
+
self._enter_quarantine(
|
|
594
|
+
'spot_lease_conflict',
|
|
595
|
+
{
|
|
596
|
+
'account_id': self._account_id,
|
|
597
|
+
'base_asset': port.base_asset,
|
|
598
|
+
'quote_asset': port.quote_asset,
|
|
599
|
+
},
|
|
600
|
+
)
|
|
601
|
+
return self._startup_result()
|
|
602
|
+
|
|
603
|
+
# (2) Persisted quarantine from a previous run.
|
|
604
|
+
epoch = self._store.get_latest_spot_epoch(port.product_id)
|
|
605
|
+
self._epoch = epoch
|
|
606
|
+
if epoch is not None and epoch.state == 'quarantined':
|
|
607
|
+
self._refresh_fold()
|
|
608
|
+
self._enter_quarantine(
|
|
609
|
+
'spot_epoch_quarantined',
|
|
610
|
+
{
|
|
611
|
+
'product_id': port.product_id,
|
|
612
|
+
'epoch_seq': epoch.epoch_seq,
|
|
613
|
+
'pending_conflict': epoch.pending_conflict,
|
|
614
|
+
},
|
|
615
|
+
)
|
|
616
|
+
return self._startup_result()
|
|
617
|
+
if epoch is not None and epoch.cursor_scope != port.cursor_scope:
|
|
618
|
+
# A plugin upgrade changed what the persisted cursor means —
|
|
619
|
+
# trusting it could silently skip history. Fail closed.
|
|
620
|
+
self._refresh_fold()
|
|
621
|
+
self._enter_quarantine(
|
|
622
|
+
'spot_cursor_scope_changed',
|
|
623
|
+
{
|
|
624
|
+
'product_id': port.product_id,
|
|
625
|
+
'stored_scope': epoch.cursor_scope,
|
|
626
|
+
'port_scope': port.cursor_scope,
|
|
627
|
+
},
|
|
628
|
+
)
|
|
629
|
+
return self._startup_result()
|
|
630
|
+
|
|
631
|
+
# (3) Execution catch-up from the durable cursor.
|
|
632
|
+
try:
|
|
633
|
+
recovered, conclusive, final_cursor = await self._catch_up(
|
|
634
|
+
epoch.exec_cursor if epoch is not None else None,
|
|
635
|
+
)
|
|
636
|
+
except SpotInventoryConflictError:
|
|
637
|
+
# _catch_up already quarantined (foreign ledger row).
|
|
638
|
+
return self._startup_result()
|
|
639
|
+
if epoch is not None:
|
|
640
|
+
# The catch-up advanced the persisted cursor page by page —
|
|
641
|
+
# refresh the in-memory snapshot to match.
|
|
642
|
+
epoch = self._store.get_latest_spot_epoch(port.product_id)
|
|
643
|
+
self._epoch = epoch
|
|
644
|
+
if not conclusive:
|
|
645
|
+
self._refresh_fold()
|
|
646
|
+
self._enter_quarantine(
|
|
647
|
+
'spot_catchup_inconclusive',
|
|
648
|
+
{'product_id': port.product_id},
|
|
649
|
+
)
|
|
650
|
+
return self._startup_result()
|
|
651
|
+
|
|
652
|
+
# (4) Fold + first-epoch baseline freeze / balance read.
|
|
653
|
+
self._refresh_fold()
|
|
654
|
+
if self._fold.violation is not None:
|
|
655
|
+
self._enter_quarantine(
|
|
656
|
+
'spot_ledger_negative_inventory',
|
|
657
|
+
{'product_id': port.product_id,
|
|
658
|
+
'violation': self._fold.violation},
|
|
659
|
+
)
|
|
660
|
+
return self._startup_result()
|
|
661
|
+
# noinspection PyBroadException
|
|
662
|
+
try:
|
|
663
|
+
balance = await port.fetch_base_balance()
|
|
664
|
+
except Exception as exc:
|
|
665
|
+
# Fail-closed boundary: WHATEVER went wrong with the read,
|
|
666
|
+
# startup must not proceed to trading on an unchecked book.
|
|
667
|
+
logger.exception(
|
|
668
|
+
"spot inventory: startup base-balance read failed for %r",
|
|
669
|
+
port.product_id,
|
|
670
|
+
)
|
|
671
|
+
self._enter_quarantine(
|
|
672
|
+
'spot_startup_balance_unavailable',
|
|
673
|
+
{'product_id': port.product_id, 'error': repr(exc)},
|
|
674
|
+
)
|
|
675
|
+
return self._startup_result()
|
|
676
|
+
if not isinstance(balance, Decimal) or not balance.is_finite():
|
|
677
|
+
self._enter_quarantine(
|
|
678
|
+
'spot_startup_balance_invalid',
|
|
679
|
+
{'product_id': port.product_id, 'balance': repr(balance)},
|
|
680
|
+
)
|
|
681
|
+
return self._startup_result()
|
|
682
|
+
|
|
683
|
+
if epoch is None:
|
|
684
|
+
baseline = balance - self._fold.net_base
|
|
685
|
+
if baseline < -port.base_tolerance:
|
|
686
|
+
# The account owns less base than the ledger says the bot
|
|
687
|
+
# holds — a foreign withdrawal or external sale in the
|
|
688
|
+
# crash window before the first epoch write. Freezing a
|
|
689
|
+
# negative foreign baseline would make the invariant hold
|
|
690
|
+
# by construction and synthesize inventory the account
|
|
691
|
+
# cannot sell. Fail closed instead.
|
|
692
|
+
self._enter_quarantine(
|
|
693
|
+
'spot_baseline_below_inventory',
|
|
694
|
+
{
|
|
695
|
+
'product_id': port.product_id,
|
|
696
|
+
'current_total': canonical_decimal(balance),
|
|
697
|
+
'bot_inventory': canonical_decimal(self._fold.net_base),
|
|
698
|
+
'implied_baseline': canonical_decimal(baseline),
|
|
699
|
+
},
|
|
700
|
+
)
|
|
701
|
+
return self._startup_result()
|
|
702
|
+
with self._store.transaction():
|
|
703
|
+
epoch = self._store.insert_spot_epoch(
|
|
704
|
+
account_id=self._account_id,
|
|
705
|
+
base_asset=port.base_asset,
|
|
706
|
+
product_id=port.product_id,
|
|
707
|
+
foreign_baseline=canonical_decimal(baseline),
|
|
708
|
+
cursor_scope=port.cursor_scope,
|
|
709
|
+
exec_cursor=final_cursor,
|
|
710
|
+
state='active',
|
|
711
|
+
)
|
|
712
|
+
self._store.log_event(
|
|
713
|
+
'spot_epoch_created',
|
|
714
|
+
payload={
|
|
715
|
+
'product_id': port.product_id,
|
|
716
|
+
'epoch_seq': epoch.epoch_seq,
|
|
717
|
+
'foreign_baseline': epoch.foreign_baseline,
|
|
718
|
+
'bot_inventory': canonical_decimal(self._fold.net_base),
|
|
719
|
+
'current_total': canonical_decimal(balance),
|
|
720
|
+
},
|
|
721
|
+
)
|
|
722
|
+
self._epoch = epoch
|
|
723
|
+
|
|
724
|
+
# (5) Strict startup invariant.
|
|
725
|
+
drift = self._invariant_drift(balance)
|
|
726
|
+
if abs(drift) > port.base_tolerance:
|
|
727
|
+
self._enter_quarantine(
|
|
728
|
+
'spot_inventory_conflict',
|
|
729
|
+
self._conflict_context(balance, drift),
|
|
730
|
+
)
|
|
731
|
+
return self._startup_result()
|
|
732
|
+
|
|
733
|
+
# (6) Adoption watermark: the synthesized position the engine is
|
|
734
|
+
# about to adopt folds every row, so none may be re-delivered.
|
|
735
|
+
assert self._epoch is not None
|
|
736
|
+
with self._store.transaction():
|
|
737
|
+
adopted = self._store.mark_spot_executions_delivered(
|
|
738
|
+
self._account_id, port.product_id,
|
|
739
|
+
)
|
|
740
|
+
if self._epoch.pending_conflict_ts_ms is not None:
|
|
741
|
+
# The invariant holds again — the persisted grace state
|
|
742
|
+
# from the previous run resolved itself (settlement
|
|
743
|
+
# landed while we were down).
|
|
744
|
+
self._store.set_spot_epoch_pending_conflict(
|
|
745
|
+
port.product_id, self._epoch.epoch_seq,
|
|
746
|
+
ts_ms=None, payload=None,
|
|
747
|
+
)
|
|
748
|
+
self._epoch = self._store.get_latest_spot_epoch(port.product_id)
|
|
749
|
+
self._store.log_event(
|
|
750
|
+
'spot_startup_adopted',
|
|
751
|
+
payload={
|
|
752
|
+
'product_id': port.product_id,
|
|
753
|
+
'net_base': canonical_decimal(self._fold.net_base),
|
|
754
|
+
'cost_quote': canonical_decimal(self._fold.cost_quote),
|
|
755
|
+
'fill_count': self._fold.fill_count,
|
|
756
|
+
'recovered_fills': recovered,
|
|
757
|
+
'adopted_fills': adopted,
|
|
758
|
+
},
|
|
759
|
+
)
|
|
760
|
+
return self._startup_result(recovered=recovered, adopted=adopted)
|
|
761
|
+
|
|
762
|
+
def _startup_result(
|
|
763
|
+
self, *, recovered: int = 0, adopted: int = 0,
|
|
764
|
+
) -> SpotStartupResult:
|
|
765
|
+
return SpotStartupResult(
|
|
766
|
+
quarantined=self._quarantined,
|
|
767
|
+
reason=self._quarantine_reason,
|
|
768
|
+
fold=self._fold,
|
|
769
|
+
epoch=self._epoch,
|
|
770
|
+
recovered_fills=recovered,
|
|
771
|
+
adopted_fills=adopted,
|
|
772
|
+
)
|
|
773
|
+
|
|
774
|
+
async def _catch_up(
|
|
775
|
+
self, cursor: str | None,
|
|
776
|
+
) -> tuple[int, bool, str | None]:
|
|
777
|
+
"""Drain the venue's execution history from ``cursor``.
|
|
778
|
+
|
|
779
|
+
Each CONCLUSIVE page commits atomically: all of its fills plus
|
|
780
|
+
the advanced cursor. A crash mid-pagination therefore resumes
|
|
781
|
+
exactly at the last committed page; the overlap a time-scoped API
|
|
782
|
+
re-serves is absorbed by the fill-id dedup.
|
|
783
|
+
|
|
784
|
+
An INCONCLUSIVE page's fills are still recorded (the fill-id
|
|
785
|
+
dedup makes a re-fetch safe), but its ``next_cursor`` is NOT
|
|
786
|
+
persisted and the returned cursor stays at the last conclusive
|
|
787
|
+
position — persisting a cursor past history the venue could not
|
|
788
|
+
vouch for would let a later retry or rebaseline skip the
|
|
789
|
+
uncertain range and launder the omitted fills into a baseline.
|
|
790
|
+
|
|
791
|
+
:return: ``(recovered_count, conclusive, final_cursor)`` —
|
|
792
|
+
``final_cursor`` is the last CONCLUSIVE cursor.
|
|
793
|
+
:raises SpotInventoryConflictError: After quarantining on a
|
|
794
|
+
foreign-owned ledger row.
|
|
795
|
+
"""
|
|
796
|
+
port = self._port
|
|
797
|
+
recovered = 0
|
|
798
|
+
current = cursor
|
|
799
|
+
for _ in range(_MAX_CATCHUP_PAGES):
|
|
800
|
+
batch = await port.fetch_executions(current)
|
|
801
|
+
try:
|
|
802
|
+
with self._store.transaction():
|
|
803
|
+
for execution in batch.executions:
|
|
804
|
+
if self._record_execution(execution, delivered=False):
|
|
805
|
+
recovered += 1
|
|
806
|
+
if batch.conclusive \
|
|
807
|
+
and batch.next_cursor is not None \
|
|
808
|
+
and self._epoch is not None:
|
|
809
|
+
self._store.set_spot_epoch_cursor(
|
|
810
|
+
port.product_id, self._epoch.epoch_seq,
|
|
811
|
+
batch.next_cursor,
|
|
812
|
+
)
|
|
813
|
+
except _ForeignLedgerRow as foreign:
|
|
814
|
+
# The page's transaction rolled back; quarantine OUTSIDE
|
|
815
|
+
# the aborted span so its own store writes survive.
|
|
816
|
+
raise self._quarantine_foreign_row(foreign) from None
|
|
817
|
+
if not batch.conclusive:
|
|
818
|
+
# ``current`` is still the last conclusive cursor — do not
|
|
819
|
+
# advance past a page the venue could not vouch for.
|
|
820
|
+
return recovered, False, current
|
|
821
|
+
if batch.next_cursor is not None:
|
|
822
|
+
current = batch.next_cursor
|
|
823
|
+
if not batch.has_more:
|
|
824
|
+
return recovered, True, current
|
|
825
|
+
logger.error(
|
|
826
|
+
"spot inventory: catch-up exceeded %d pages for %r; "
|
|
827
|
+
"treating as inconclusive", _MAX_CATCHUP_PAGES, port.product_id,
|
|
828
|
+
)
|
|
829
|
+
return recovered, False, current
|
|
830
|
+
|
|
831
|
+
def _record_execution(
|
|
832
|
+
self, execution: SpotExecution, *, delivered: bool,
|
|
833
|
+
) -> bool:
|
|
834
|
+
"""Insert one fill inside the caller's transaction.
|
|
835
|
+
|
|
836
|
+
:return: ``True`` when the row was inserted, ``False`` on a
|
|
837
|
+
benign own-run dedup.
|
|
838
|
+
:raises _ForeignLedgerRow: When the fill id is already booked
|
|
839
|
+
under ANOTHER logical run — the exclusivity contract is
|
|
840
|
+
broken and neither run's books can be trusted. The caller
|
|
841
|
+
converts this to a quarantine OUTSIDE the rolled-back span.
|
|
842
|
+
"""
|
|
843
|
+
port = self._port
|
|
844
|
+
inserted = self._store.record_spot_execution(
|
|
845
|
+
self._account_id, port.product_id,
|
|
846
|
+
fill_id=execution.fill_id,
|
|
847
|
+
side=execution.side,
|
|
848
|
+
base_delta=canonical_decimal(execution.base_delta),
|
|
849
|
+
quote_delta=canonical_decimal(execution.quote_delta),
|
|
850
|
+
price=canonical_decimal(execution.price),
|
|
851
|
+
fee_amount=canonical_decimal(execution.fee_amount),
|
|
852
|
+
fee_currency=execution.fee_currency,
|
|
853
|
+
ts_ms=execution.ts_ms,
|
|
854
|
+
venue_seq=execution.venue_seq,
|
|
855
|
+
exchange_order_id=execution.exchange_order_id,
|
|
856
|
+
client_order_id=execution.client_order_id,
|
|
857
|
+
delivered=delivered,
|
|
858
|
+
)
|
|
859
|
+
if inserted:
|
|
860
|
+
return True
|
|
861
|
+
# A missing owner row can only mean our own insert raced the
|
|
862
|
+
# dedup read — treat it as the benign own-run case.
|
|
863
|
+
owner = self._store.spot_execution_owner(
|
|
864
|
+
self._account_id, port.product_id, execution.fill_id,
|
|
865
|
+
) or self._store.run_id
|
|
866
|
+
if owner == self._store.run_id:
|
|
867
|
+
return False
|
|
868
|
+
raise _ForeignLedgerRow(execution.fill_id, owner)
|
|
869
|
+
|
|
870
|
+
def _quarantine_foreign_row(
|
|
871
|
+
self, foreign: _ForeignLedgerRow,
|
|
872
|
+
) -> SpotInventoryConflictError:
|
|
873
|
+
"""Quarantine on a foreign-owned ledger row; build the halt error."""
|
|
874
|
+
self._enter_quarantine(
|
|
875
|
+
'spot_foreign_ledger_row',
|
|
876
|
+
{
|
|
877
|
+
'product_id': self._port.product_id,
|
|
878
|
+
'fill_id': foreign.fill_id,
|
|
879
|
+
'owner_run_id': foreign.owner_run_id,
|
|
880
|
+
},
|
|
881
|
+
)
|
|
882
|
+
return SpotInventoryConflictError(
|
|
883
|
+
f"spot fill {foreign.fill_id!r} already booked under "
|
|
884
|
+
f"another logical run {foreign.owner_run_id!r}",
|
|
885
|
+
context={
|
|
886
|
+
'fill_id': foreign.fill_id,
|
|
887
|
+
'owner_run_id': foreign.owner_run_id,
|
|
888
|
+
},
|
|
889
|
+
)
|
|
890
|
+
|
|
891
|
+
# --- Live fills ---------------------------------------------------------
|
|
892
|
+
|
|
893
|
+
def record_live_fill(self, execution: SpotExecution) -> bool:
|
|
894
|
+
"""Record a fill observed on the live stream — outbox pattern.
|
|
895
|
+
|
|
896
|
+
The ledger row is inserted with ``delivered=1`` in one
|
|
897
|
+
transaction; the caller emits the corresponding
|
|
898
|
+
:class:`~pynecore.core.broker.models.OrderEvent` ONLY when this
|
|
899
|
+
returns ``True``. A crash after the commit but before the emit
|
|
900
|
+
loses the event, not the fill: the next startup folds the row
|
|
901
|
+
into the adopted position (exactly-once, adoption path).
|
|
902
|
+
|
|
903
|
+
:return: ``True`` → new fill, emit the event; ``False`` → replay
|
|
904
|
+
dedup, do NOT emit.
|
|
905
|
+
:raises SpotInventoryConflictError: When the fill is booked
|
|
906
|
+
under another logical run (after quarantining).
|
|
907
|
+
"""
|
|
908
|
+
if not self._started:
|
|
909
|
+
raise RuntimeError(
|
|
910
|
+
"record_live_fill before startup(): the adoption "
|
|
911
|
+
"watermark is not established yet"
|
|
912
|
+
)
|
|
913
|
+
try:
|
|
914
|
+
with self._store.transaction():
|
|
915
|
+
inserted = self._record_execution(execution, delivered=True)
|
|
916
|
+
except _ForeignLedgerRow as foreign:
|
|
917
|
+
raise self._quarantine_foreign_row(foreign) from None
|
|
918
|
+
if inserted:
|
|
919
|
+
self._refresh_fold()
|
|
920
|
+
if self._fold.violation is not None and not self._quarantined:
|
|
921
|
+
self._enter_quarantine(
|
|
922
|
+
'spot_ledger_negative_inventory',
|
|
923
|
+
{'product_id': self._port.product_id,
|
|
924
|
+
'violation': self._fold.violation},
|
|
925
|
+
)
|
|
926
|
+
return inserted
|
|
927
|
+
|
|
928
|
+
# --- Periodic reconcile ---------------------------------------------------
|
|
929
|
+
|
|
930
|
+
async def reconcile(self, now_ms: int) -> 'list[SpotExecutionRow]':
|
|
931
|
+
"""Per-poll invariant check + lease heartbeat.
|
|
932
|
+
|
|
933
|
+
Transient venue read failures skip the cycle (a live bot must
|
|
934
|
+
not halt on a recoverable read). A mismatch beyond the numeric
|
|
935
|
+
tolerance arms the persisted settlement-grace state and triggers
|
|
936
|
+
a fresh execution catch-up (late fills are the innocent
|
|
937
|
+
explanation); a mismatch still unexplained past
|
|
938
|
+
``port.settlement_grace_s`` is a confirmed attribution conflict.
|
|
939
|
+
|
|
940
|
+
The lease heartbeat is fenced by the physical instance: if a
|
|
941
|
+
replacement instance took the lease over while this (now zombie)
|
|
942
|
+
process was silent, the heartbeat reports the loss and this run
|
|
943
|
+
quarantines instead of trading on a lease it no longer holds.
|
|
944
|
+
|
|
945
|
+
:return: The ledger rows recovered by a runtime catch-up this
|
|
946
|
+
cycle, freshly flipped to ``delivered``. The caller (plugin)
|
|
947
|
+
MUST emit the corresponding
|
|
948
|
+
:class:`~pynecore.core.broker.models.OrderEvent` for each so
|
|
949
|
+
the sync engine's position tracks the recovered fills — the
|
|
950
|
+
periodic engine reconcile ignores position increases, so a
|
|
951
|
+
stream-gap fill recovered here would otherwise stay invisible
|
|
952
|
+
to the strategy until the next restart's adoption. Marking
|
|
953
|
+
them ``delivered`` before emission is crash-safe: a crash
|
|
954
|
+
before the emit loses only the event, and the next startup
|
|
955
|
+
re-folds the whole ledger into the adopted position. Empty on
|
|
956
|
+
a clean cycle.
|
|
957
|
+
"""
|
|
958
|
+
lease_held = self._store.heartbeat_spot_asset(
|
|
959
|
+
self._account_id, self._port.base_asset,
|
|
960
|
+
)
|
|
961
|
+
if not self._quarantined and self._epoch is not None and not lease_held:
|
|
962
|
+
self._enter_quarantine(
|
|
963
|
+
'spot_lease_lost',
|
|
964
|
+
{
|
|
965
|
+
'product_id': self._port.product_id,
|
|
966
|
+
'base_asset': self._port.base_asset,
|
|
967
|
+
},
|
|
968
|
+
)
|
|
969
|
+
return []
|
|
970
|
+
if self._quarantined or self._epoch is None:
|
|
971
|
+
return []
|
|
972
|
+
port = self._port
|
|
973
|
+
# noinspection PyBroadException
|
|
974
|
+
try:
|
|
975
|
+
balance = await port.fetch_base_balance()
|
|
976
|
+
except Exception:
|
|
977
|
+
# Transient read failure: skip the cycle, retry on the next
|
|
978
|
+
# poll — a live bot must not halt on a recoverable read.
|
|
979
|
+
logger.warning(
|
|
980
|
+
"spot inventory: base-balance read failed for %r; "
|
|
981
|
+
"skipping this reconcile cycle", port.product_id,
|
|
982
|
+
exc_info=True,
|
|
983
|
+
)
|
|
984
|
+
return []
|
|
985
|
+
if not isinstance(balance, Decimal) or not balance.is_finite():
|
|
986
|
+
logger.warning(
|
|
987
|
+
"spot inventory: invalid base balance %r for %r; "
|
|
988
|
+
"skipping this reconcile cycle", balance, port.product_id,
|
|
989
|
+
)
|
|
990
|
+
return []
|
|
991
|
+
|
|
992
|
+
drift = self._invariant_drift(balance)
|
|
993
|
+
epoch = self._epoch
|
|
994
|
+
if abs(drift) <= port.base_tolerance:
|
|
995
|
+
if epoch.pending_conflict_ts_ms is not None:
|
|
996
|
+
self._store.set_spot_epoch_pending_conflict(
|
|
997
|
+
port.product_id, epoch.epoch_seq,
|
|
998
|
+
ts_ms=None, payload=None,
|
|
999
|
+
)
|
|
1000
|
+
self._epoch = self._store.get_latest_spot_epoch(port.product_id)
|
|
1001
|
+
self._store.log_event(
|
|
1002
|
+
'spot_inventory_conflict_resolved',
|
|
1003
|
+
payload={'product_id': port.product_id},
|
|
1004
|
+
)
|
|
1005
|
+
return []
|
|
1006
|
+
|
|
1007
|
+
# Mismatch. Try the innocent explanation first: fills we have
|
|
1008
|
+
# not seen yet (settlement / stream gap).
|
|
1009
|
+
try:
|
|
1010
|
+
recovered, conclusive, _ = await self._catch_up(epoch.exec_cursor)
|
|
1011
|
+
except SpotInventoryConflictError:
|
|
1012
|
+
return [] # already quarantined
|
|
1013
|
+
delivered: list['SpotExecutionRow'] = []
|
|
1014
|
+
if recovered:
|
|
1015
|
+
self._refresh_fold()
|
|
1016
|
+
if self._fold.violation is not None:
|
|
1017
|
+
self._enter_quarantine(
|
|
1018
|
+
'spot_ledger_negative_inventory',
|
|
1019
|
+
{'product_id': port.product_id,
|
|
1020
|
+
'violation': self._fold.violation},
|
|
1021
|
+
)
|
|
1022
|
+
return []
|
|
1023
|
+
# The recovered fills must reach the sync engine — hand them
|
|
1024
|
+
# to the caller for emission and flip their outbox marker.
|
|
1025
|
+
delivered = self._deliver_recovered()
|
|
1026
|
+
drift = self._invariant_drift(balance)
|
|
1027
|
+
if abs(drift) <= port.base_tolerance:
|
|
1028
|
+
if epoch.pending_conflict_ts_ms is not None:
|
|
1029
|
+
self._store.set_spot_epoch_pending_conflict(
|
|
1030
|
+
port.product_id, epoch.epoch_seq,
|
|
1031
|
+
ts_ms=None, payload=None,
|
|
1032
|
+
)
|
|
1033
|
+
self._epoch = self._store.get_latest_spot_epoch(port.product_id)
|
|
1034
|
+
self._store.log_event(
|
|
1035
|
+
'spot_inventory_conflict_resolved',
|
|
1036
|
+
payload={'product_id': port.product_id,
|
|
1037
|
+
'recovered_fills': recovered},
|
|
1038
|
+
)
|
|
1039
|
+
return delivered
|
|
1040
|
+
self._epoch = self._store.get_latest_spot_epoch(port.product_id)
|
|
1041
|
+
epoch = self._epoch
|
|
1042
|
+
assert epoch is not None
|
|
1043
|
+
|
|
1044
|
+
pending_since = epoch.pending_conflict_ts_ms
|
|
1045
|
+
context = self._conflict_context(balance, drift)
|
|
1046
|
+
if pending_since is None:
|
|
1047
|
+
# First observation: arm the persisted grace state. The
|
|
1048
|
+
# timestamp survives crashes, so a restart loop cannot keep
|
|
1049
|
+
# resetting the window.
|
|
1050
|
+
self._store.set_spot_epoch_pending_conflict(
|
|
1051
|
+
port.product_id, epoch.epoch_seq,
|
|
1052
|
+
ts_ms=now_ms, payload=context,
|
|
1053
|
+
)
|
|
1054
|
+
self._epoch = self._store.get_latest_spot_epoch(port.product_id)
|
|
1055
|
+
self._store.log_event(
|
|
1056
|
+
'spot_inventory_conflict_pending',
|
|
1057
|
+
payload=context,
|
|
1058
|
+
)
|
|
1059
|
+
logger.warning(
|
|
1060
|
+
"spot inventory: balance invariant mismatch for %r "
|
|
1061
|
+
"(drift=%s); settlement grace armed (%.1fs)",
|
|
1062
|
+
port.product_id, context['drift'], port.settlement_grace_s,
|
|
1063
|
+
)
|
|
1064
|
+
return delivered
|
|
1065
|
+
if not conclusive:
|
|
1066
|
+
# Cannot re-verify against an authoritative history read —
|
|
1067
|
+
# keep the grace armed, do not extend or shorten it.
|
|
1068
|
+
logger.warning(
|
|
1069
|
+
"spot inventory: conflict re-check inconclusive for %r; "
|
|
1070
|
+
"grace stays armed", port.product_id,
|
|
1071
|
+
)
|
|
1072
|
+
if now_ms - pending_since >= port.settlement_grace_s * 1000.0:
|
|
1073
|
+
self._enter_quarantine('spot_inventory_conflict', context)
|
|
1074
|
+
return delivered
|
|
1075
|
+
|
|
1076
|
+
def _invariant_drift(self, balance: Decimal) -> Decimal:
|
|
1077
|
+
"""``actual − (foreign_baseline + bot_inventory)``, exact."""
|
|
1078
|
+
assert self._epoch is not None
|
|
1079
|
+
baseline = Decimal(self._epoch.foreign_baseline)
|
|
1080
|
+
return balance - (baseline + self._fold.net_base)
|
|
1081
|
+
|
|
1082
|
+
def _conflict_context(
|
|
1083
|
+
self, balance: Decimal, drift: Decimal,
|
|
1084
|
+
) -> dict[str, Any]:
|
|
1085
|
+
assert self._epoch is not None
|
|
1086
|
+
return {
|
|
1087
|
+
'product_id': self._port.product_id,
|
|
1088
|
+
'base_asset': self._port.base_asset,
|
|
1089
|
+
'epoch_seq': self._epoch.epoch_seq,
|
|
1090
|
+
'foreign_baseline': self._epoch.foreign_baseline,
|
|
1091
|
+
'bot_inventory': canonical_decimal(self._fold.net_base),
|
|
1092
|
+
'current_total': canonical_decimal(balance),
|
|
1093
|
+
'drift': canonical_decimal(drift),
|
|
1094
|
+
'tolerance': canonical_decimal(self._port.base_tolerance),
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
# --- Position synthesis ---------------------------------------------------
|
|
1098
|
+
|
|
1099
|
+
def synthesize_position(self, mark: float) -> ExchangePosition | None:
|
|
1100
|
+
"""Build the :class:`ExchangePosition` the plugin's
|
|
1101
|
+
``get_position()`` returns.
|
|
1102
|
+
|
|
1103
|
+
``None`` only when the bot's net inventory is genuinely flat —
|
|
1104
|
+
the engine reads ``None`` as an authoritative flat, so a spot
|
|
1105
|
+
plugin must never return it merely because the venue has no
|
|
1106
|
+
position object.
|
|
1107
|
+
"""
|
|
1108
|
+
fold = self._fold
|
|
1109
|
+
if fold.net_base <= 0:
|
|
1110
|
+
return None
|
|
1111
|
+
vwap = fold.vwap
|
|
1112
|
+
entry_price = float(vwap) if vwap is not None else 0.0
|
|
1113
|
+
size = float(fold.net_base)
|
|
1114
|
+
unrealized = (mark - entry_price) * size if vwap is not None else 0.0
|
|
1115
|
+
return ExchangePosition(
|
|
1116
|
+
symbol=self._symbol,
|
|
1117
|
+
side='long',
|
|
1118
|
+
size=size,
|
|
1119
|
+
entry_price=entry_price,
|
|
1120
|
+
unrealized_pnl=unrealized,
|
|
1121
|
+
liquidation_price=None,
|
|
1122
|
+
leverage=1.0,
|
|
1123
|
+
margin_mode='cash',
|
|
1124
|
+
)
|
|
1125
|
+
|
|
1126
|
+
# --- Rebaseline (operator recovery) ---------------------------------------
|
|
1127
|
+
|
|
1128
|
+
async def rebaseline(self) -> 'SpotEpochRow':
|
|
1129
|
+
"""Freeze a new baseline epoch after an operator intervened.
|
|
1130
|
+
|
|
1131
|
+
Preconditions (all fail-closed, raising ``ValueError``):
|
|
1132
|
+
|
|
1133
|
+
- dispatch is frozen — the run is quarantined (rebaselining a
|
|
1134
|
+
live-trading run would launder in-flight drift);
|
|
1135
|
+
- no unresolved dispatches: no parked verifications and no live
|
|
1136
|
+
order rows in a pending dispatch state (their eventual fills
|
|
1137
|
+
would land on the wrong side of the new baseline);
|
|
1138
|
+
- a FRESH conclusive execution catch-up and balance read succeed
|
|
1139
|
+
right here.
|
|
1140
|
+
|
|
1141
|
+
The new epoch (bumped ``epoch_seq``, recomputed
|
|
1142
|
+
``foreign_baseline``, ``state='active'``) and the old epoch's
|
|
1143
|
+
``closed`` flip commit in ONE transaction. The engine's
|
|
1144
|
+
quarantine latch is in-memory by design — the operator restarts
|
|
1145
|
+
the run after a successful rebaseline; the fresh startup then
|
|
1146
|
+
finds the active epoch and trades again.
|
|
1147
|
+
|
|
1148
|
+
:return: The freshly activated epoch row.
|
|
1149
|
+
"""
|
|
1150
|
+
if not self._quarantined:
|
|
1151
|
+
raise ValueError(
|
|
1152
|
+
"rebaseline requires the run to be quarantined — "
|
|
1153
|
+
"dispatch must be frozen while the baseline moves"
|
|
1154
|
+
)
|
|
1155
|
+
port = self._port
|
|
1156
|
+
unresolved = self._unresolved_dispatches()
|
|
1157
|
+
if unresolved:
|
|
1158
|
+
raise ValueError(
|
|
1159
|
+
f"rebaseline blocked: unresolved dispatches remain "
|
|
1160
|
+
f"({', '.join(unresolved)}) — their eventual fills would "
|
|
1161
|
+
f"land on the wrong side of the new baseline"
|
|
1162
|
+
)
|
|
1163
|
+
old_epoch = self._store.get_latest_spot_epoch(port.product_id)
|
|
1164
|
+
recovered, conclusive, final_cursor = await self._catch_up(
|
|
1165
|
+
old_epoch.exec_cursor if old_epoch is not None else None,
|
|
1166
|
+
)
|
|
1167
|
+
if not conclusive:
|
|
1168
|
+
raise ValueError(
|
|
1169
|
+
"rebaseline blocked: execution catch-up was inconclusive"
|
|
1170
|
+
)
|
|
1171
|
+
self._refresh_fold()
|
|
1172
|
+
if self._fold.violation is not None:
|
|
1173
|
+
raise ValueError(
|
|
1174
|
+
f"rebaseline blocked: ledger corrupt "
|
|
1175
|
+
f"({self._fold.violation})"
|
|
1176
|
+
)
|
|
1177
|
+
balance = await port.fetch_base_balance()
|
|
1178
|
+
if not isinstance(balance, Decimal) or not balance.is_finite():
|
|
1179
|
+
raise ValueError(
|
|
1180
|
+
f"rebaseline blocked: invalid base balance {balance!r}"
|
|
1181
|
+
)
|
|
1182
|
+
baseline = balance - self._fold.net_base
|
|
1183
|
+
if baseline < -port.base_tolerance:
|
|
1184
|
+
# A negative foreign baseline would launder exactly the
|
|
1185
|
+
# withdrawal / external-sale conflict that forced the
|
|
1186
|
+
# quarantine: it would make the invariant hold while the
|
|
1187
|
+
# synthesized position exceeds what the account owns. Refuse
|
|
1188
|
+
# until the operator restores enough base holdings.
|
|
1189
|
+
raise ValueError(
|
|
1190
|
+
f"rebaseline blocked: base balance {balance} is below the "
|
|
1191
|
+
f"ledger's bot inventory {self._fold.net_base} — restore "
|
|
1192
|
+
f"the missing base holdings before rebaselining"
|
|
1193
|
+
)
|
|
1194
|
+
with self._store.transaction():
|
|
1195
|
+
if old_epoch is not None:
|
|
1196
|
+
self._store.set_spot_epoch_state(
|
|
1197
|
+
port.product_id, old_epoch.epoch_seq, 'closed',
|
|
1198
|
+
)
|
|
1199
|
+
epoch = self._store.insert_spot_epoch(
|
|
1200
|
+
account_id=self._account_id,
|
|
1201
|
+
base_asset=port.base_asset,
|
|
1202
|
+
product_id=port.product_id,
|
|
1203
|
+
foreign_baseline=canonical_decimal(baseline),
|
|
1204
|
+
cursor_scope=port.cursor_scope,
|
|
1205
|
+
exec_cursor=final_cursor,
|
|
1206
|
+
state='active',
|
|
1207
|
+
)
|
|
1208
|
+
self._store.mark_spot_executions_delivered(
|
|
1209
|
+
self._account_id, port.product_id,
|
|
1210
|
+
)
|
|
1211
|
+
self._store.log_event(
|
|
1212
|
+
'spot_epoch_rebaselined',
|
|
1213
|
+
payload={
|
|
1214
|
+
'product_id': port.product_id,
|
|
1215
|
+
'old_epoch_seq': (
|
|
1216
|
+
None if old_epoch is None else old_epoch.epoch_seq
|
|
1217
|
+
),
|
|
1218
|
+
'epoch_seq': epoch.epoch_seq,
|
|
1219
|
+
'foreign_baseline': epoch.foreign_baseline,
|
|
1220
|
+
'bot_inventory': canonical_decimal(self._fold.net_base),
|
|
1221
|
+
'current_total': canonical_decimal(balance),
|
|
1222
|
+
'recovered_fills': recovered,
|
|
1223
|
+
},
|
|
1224
|
+
)
|
|
1225
|
+
self._epoch = epoch
|
|
1226
|
+
return epoch
|
|
1227
|
+
|
|
1228
|
+
def _unresolved_dispatches(self) -> list[str]:
|
|
1229
|
+
"""Names of dispatches whose outcome is still in flight."""
|
|
1230
|
+
unresolved: list[str] = []
|
|
1231
|
+
_envelopes, pending = self._store.replay()
|
|
1232
|
+
unresolved.extend(
|
|
1233
|
+
f"parked:{coid}" for coid in sorted(pending)
|
|
1234
|
+
)
|
|
1235
|
+
for row in self._store.iter_live_orders():
|
|
1236
|
+
if row.state in PENDING_DISPATCH_STATES:
|
|
1237
|
+
unresolved.append(f"order:{row.client_order_id}({row.state})")
|
|
1238
|
+
return unresolved
|
|
1239
|
+
|
|
1240
|
+
# --- Teardown -------------------------------------------------------------
|
|
1241
|
+
|
|
1242
|
+
def close(self) -> None:
|
|
1243
|
+
"""Release the asset lease on a clean shutdown."""
|
|
1244
|
+
self._store.release_spot_asset(
|
|
1245
|
+
self._account_id, self._port.base_asset,
|
|
1246
|
+
)
|
|
1247
|
+
|
|
1248
|
+
# --- Internals --------------------------------------------------------------
|
|
1249
|
+
|
|
1250
|
+
def _refresh_fold(self) -> None:
|
|
1251
|
+
rows = self._store.iter_spot_executions(
|
|
1252
|
+
self._account_id, self._port.product_id,
|
|
1253
|
+
)
|
|
1254
|
+
self._fold = fold_inventory(rows)
|
|
1255
|
+
|
|
1256
|
+
def _deliver_recovered(self) -> 'list[SpotExecutionRow]':
|
|
1257
|
+
"""Flip and return the undelivered rows a runtime catch-up left.
|
|
1258
|
+
|
|
1259
|
+
Startup adoption and :meth:`record_live_fill` both leave the
|
|
1260
|
+
ledger fully delivered, so undelivered rows are exactly the
|
|
1261
|
+
catch-up recoveries of the current cycle. Flipping them here
|
|
1262
|
+
(before the caller emits) mirrors :meth:`record_live_fill`'s
|
|
1263
|
+
insert-delivered-then-emit ordering; the whole-ledger re-fold on
|
|
1264
|
+
the next startup is the crash-safety net for a lost emit.
|
|
1265
|
+
"""
|
|
1266
|
+
rows = self._store.iter_spot_executions(
|
|
1267
|
+
self._account_id, self._port.product_id,
|
|
1268
|
+
undelivered_only=True,
|
|
1269
|
+
)
|
|
1270
|
+
if rows:
|
|
1271
|
+
self._store.mark_spot_executions_delivered(
|
|
1272
|
+
self._account_id, self._port.product_id,
|
|
1273
|
+
[row.fill_id for row in rows],
|
|
1274
|
+
)
|
|
1275
|
+
return rows
|
|
1276
|
+
|
|
1277
|
+
def _enter_quarantine(self, reason: str, context: dict[str, Any]) -> None:
|
|
1278
|
+
"""Latch the conflict — dual signal, mirroring the tracker.
|
|
1279
|
+
|
|
1280
|
+
Persists the epoch's ``quarantined`` state (when an epoch
|
|
1281
|
+
exists), then delivers per policy: the ``request_quarantine``
|
|
1282
|
+
hook latches the engine and the process stays alive; ``halt`` —
|
|
1283
|
+
or a missing / raising hook — arms :attr:`pending_halt`. Never
|
|
1284
|
+
fail-open.
|
|
1285
|
+
"""
|
|
1286
|
+
if self._quarantined:
|
|
1287
|
+
return
|
|
1288
|
+
self._quarantined = True
|
|
1289
|
+
self._quarantine_reason = reason
|
|
1290
|
+
epoch = self._epoch
|
|
1291
|
+
with self._store.transaction():
|
|
1292
|
+
if epoch is not None and epoch.state != 'quarantined':
|
|
1293
|
+
self._store.set_spot_epoch_state(
|
|
1294
|
+
self._port.product_id, epoch.epoch_seq, 'quarantined',
|
|
1295
|
+
)
|
|
1296
|
+
self._epoch = self._store.get_latest_spot_epoch(
|
|
1297
|
+
self._port.product_id,
|
|
1298
|
+
)
|
|
1299
|
+
self._store.log_event(
|
|
1300
|
+
'spot_inventory_quarantine',
|
|
1301
|
+
payload={'reason': reason, **context},
|
|
1302
|
+
)
|
|
1303
|
+
message = (
|
|
1304
|
+
f"spot inventory conflict ({reason}): trading stops; "
|
|
1305
|
+
f"operator rebaseline + restart required"
|
|
1306
|
+
)
|
|
1307
|
+
quarantined = False
|
|
1308
|
+
if self._policy == 'quarantine' and self._request_quarantine is not None:
|
|
1309
|
+
# noinspection PyBroadException
|
|
1310
|
+
try:
|
|
1311
|
+
self._request_quarantine(message, dict(context))
|
|
1312
|
+
except Exception:
|
|
1313
|
+
logger.exception(
|
|
1314
|
+
"request_quarantine hook failed for %r; falling back "
|
|
1315
|
+
"to the process-exiting halt", reason,
|
|
1316
|
+
)
|
|
1317
|
+
else:
|
|
1318
|
+
quarantined = True
|
|
1319
|
+
if not quarantined:
|
|
1320
|
+
# 'halt' policy, or a quarantining policy whose hook is
|
|
1321
|
+
# missing / raised: arm the process-exiting signal.
|
|
1322
|
+
self._pending_halt = SpotInventoryConflictError(
|
|
1323
|
+
message, context=dict(context),
|
|
1324
|
+
)
|
|
1325
|
+
logger.error(
|
|
1326
|
+
"spot inventory QUARANTINE (%s): %s", reason, context,
|
|
1327
|
+
)
|