perp-cli 0.12.17 → 0.14.0

package/dist/bot/strategies/funding-arb-v2.js

@@ -1,7 +1,16 @@
 import { registerStrategy } from "../strategy-registry.js";
+import { isQuarantined, quarantineKey, quarantineSymbol } from "../quarantine.js";
+import { PerpError } from "../../errors.js";
+import { closePerpVerified, isOutcomeUnknown, placePerpVerified, trimFirstLegToHedge } from "../perp-order.js";
+import { fillSizeString, sizeDecimalsOf } from "../../exchanges/spot-result.js";
+import { isPositionGone, assertFundingArbLegs, cancelBotSymbols } from "./funding-arb-utils.js";
+/** Consecutive failed closes before the symbol is quarantined. */
+const MAX_CLOSE_FAILURES = 3;
 export class FundingArbV2Strategy {
     name = "funding-arb-v2";
     _config = {};
+    /** Symbols this bot entered or recovered this run — the only symbols its stop cancels orders on. */
+    _traded = new Set();
     describe() {
         return {
             description: "Enhanced multi-exchange funding rate arbitrage with auto-close",
@@ -20,11 +29,68 @@ export class FundingArbV2Strategy {
     get sizeUsd() { return Number(this._config.sizeUsd ?? 50); }
     get scanIntervalSec() { return Number(this._config.scanIntervalSec ?? 60); }
     async init(ctx, _snapshot) {
+        assertFundingArbLegs(ctx);
         this._config = ctx.config;
-        ctx.state.set("positions", []);
+        // A restart must never re-enter (or ignore) what is already held: rebuild the pairs from the venues.
+        const recovered = await this.recoverPositions(this.getAdapters(ctx), ctx);
+        for (const p of recovered)
+            this._traded.add(p.symbol);
+        ctx.state.set("positions", recovered);
         ctx.state.set("lastScan", 0);
+        if (recovered.length > 0)
+            ctx.log(`  [ARB-V2] Recovered ${recovered.length} pair(s): ${recovered.map(p => `${p.symbol} ${p.longExchange}↔${p.shortExchange} ${p.size}`).join(", ")}`);
         ctx.log(`  [ARB-V2] Ready | minSpread=${this.minSpread}% closeSpread=${this.closeSpread}% maxPos=${this.maxPositions} size=$${this.sizeUsd}`);
     }
+    /**
+     * Pairs held on the venues: a long on one venue + a short of the same symbol and EXACTLY the
+     * same size on another. A single leg or a size-mismatched long/short is quarantined (both
+     * sizes recorded) — never tracked, never stacked on. A venue whose positions cannot be read
+     * throws (the start fails and is retried): unknown exposure is never "flat".
+     */
+    async recoverPositions(adapters, ctx) {
+        const legs = [];
+        for (const [venue, a] of adapters) {
+            let list;
+            try {
+                list = await a.getPositions();
+            }
+            catch (e) {
+                throw new PerpError("EXCHANGE_ERROR", `[ARB-V2] ${venue} positions unreadable at start (${e instanceof Error ? e.message : String(e)}) — not started (held pairs unknown)`, {
+                    exchange: venue, remediation: `Check ${venue} connectivity (perp -e ${venue} account positions); the bot retries the start`,
+                });
+            }
+            for (const p of list) {
+                const size = Math.abs(Number(p.size));
+                if (size > 0)
+                    legs.push({ venue, symbol: p.symbol.toUpperCase(), side: p.side, size, raw: p.size.replace(/^-/, "") });
+            }
+        }
+        const used = new Set();
+        const out = [];
+        for (const l of legs) {
+            if (l.side !== "long" || used.has(l))
+                continue;
+            const s = legs.find(x => !used.has(x) && x.side === "short" && x.symbol === l.symbol && x.venue !== l.venue && x.size === l.size);
+            if (s === undefined)
+                continue;
+            used.add(l);
+            used.add(s);
+            out.push({ symbol: l.symbol, longExchange: l.venue, shortExchange: s.venue, entrySpread: 0, entryTime: 0, sizeUsd: 0, size: l.raw, longClosed: false, closeFailures: 0 });
+        }
+        const leftover = new Map();
+        for (const l of legs)
+            if (!used.has(l))
+                leftover.set(quarantineKey(l.symbol), [...(leftover.get(quarantineKey(l.symbol)) ?? []), l]);
+        for (const [key, group] of leftover) {
+            if (isQuarantined(key, ctx.log))
+                continue;
+            const desc = group.map(l => `${l.side} ${l.raw} on ${l.venue}`).join(", ");
+            const venues = [...new Set(group.map(l => l.venue))];
+            const mismatched = group.some(l => l.side === "long") && group.some(l => l.side === "short");
+            quarantineSymbol(this.name, key, mismatched ? `recovery: ${key} long/short sizes do not match (${desc}) — not tracked as a pair` : `recovery: unpaired ${key} perp leg (${desc}) — no hedge found, not tracked`, `Check ${key} on ${venues.join(" and ")} (${venues.map(v => `perp -e ${v} positions`).join("; ")}); close or re-hedge by hand, then: perp strategy quarantine-clear ${key}`, ctx.log);
+        }
+        return out;
+    }
     async onTick(ctx, _snapshot) {
         const positions = ctx.state.get("positions");
         const lastScan = ctx.state.get("lastScan");
@@ -54,14 +120,17 @@ export class FundingArbV2Strategy {
                 }
                 ratesByExchange.set(name, map);
             }
-            catch {
-                // Skip unavailable exchange
+            catch (err) {
+                // Logged; this venue is left out of this scan (no close decision, no entry on it).
+                ctx.log(`  [ARB-V2] ${name} markets unavailable: ${err instanceof Error ? err.message : String(err)}`);
             }
         }
         // Phase 1: Check existing positions for auto-close
         const toClose = [];
         for (let i = 0; i < positions.length; i++) {
             const pos = positions[i];
+            if (isQuarantined(pos.symbol, ctx.log))
+                continue;
             const longRates = ratesByExchange.get(pos.longExchange);
             const shortRates = ratesByExchange.get(pos.shortExchange);
             if (!longRates || !shortRates)
@@ -73,26 +142,8 @@ export class FundingArbV2Strategy {
             const currentSpread = this.annualizeSpread(shortInfo.rate, longInfo.rate);
             if (currentSpread <= this.closeSpread) {
                 ctx.log(`  [ARB-V2] Auto-close ${pos.symbol}: spread ${currentSpread.toFixed(1)}% <= ${this.closeSpread}% (entry was ${pos.entrySpread.toFixed(1)}%)`);
-                // Close both legs
-                const longAdapter = adapters.get(pos.longExchange);
-                const shortAdapter = adapters.get(pos.shortExchange);
-                if (longAdapter && shortAdapter) {
-                    const price = longInfo.price;
-                    const size = price > 0 ? (pos.sizeUsd / price).toFixed(6) : "0";
-                    try {
-                        await Promise.all([
-                            longAdapter.marketOrder(pos.symbol, "sell", size),
-                            shortAdapter.marketOrder(pos.symbol, "buy", size),
-                        ]);
-                        ctx.log(`  [ARB-V2] Closed ${pos.symbol} (long ${pos.longExchange}, short ${pos.shortExchange})`);
-                    }
-                    catch (err) {
-                        const msg = err instanceof Error ? err.message : String(err);
-                        ctx.log(`  [ARB-V2] Close failed for ${pos.symbol}: ${msg}`);
-                        continue; // Don't mark as closed
-                    }
-                }
-                toClose.push(i);
+                if (await this.closePosition(ctx, pos, adapters))
+                    toClose.push(i);
             }
         }
         // Remove closed positions (reverse order to preserve indices)
@@ -103,6 +154,11 @@ export class FundingArbV2Strategy {
         // Phase 2: Scan for new opportunities
         if (positions.length >= this.maxPositions)
             return [];
+        // Engine: equity unknown → no new entries (the closes above still ran).
+        if (ctx.state.get("equityUnknown") === true) {
+            ctx.log(`  [ARB-V2] Equity unknown — no new entries this scan`);
+            return positions.length > 0 ? [{ type: "noop" }] : [];
+        }
         const exchangeNames = [...ratesByExchange.keys()];
         const allSymbols = new Set();
         for (const [, map] of ratesByExchange) {
@@ -115,6 +171,8 @@ export class FundingArbV2Strategy {
         for (const sym of allSymbols) {
             if (activeSymbols.has(sym))
                 continue;
+            if (isQuarantined(sym, ctx.log))
+                continue;
             let minRate = Infinity, maxRate = -Infinity;
             let minExchange = "", maxExchange = "";
             let bestPrice = 0;
@@ -149,18 +207,19 @@ export class FundingArbV2Strategy {
         // Open new positions
         const slotsAvailable = this.maxPositions - positions.length;
         const toOpen = opportunities.slice(0, slotsAvailable);
+        const live = new Map();
         for (const opp of toOpen) {
             const longAdapter = adapters.get(opp.longExchange);
             const shortAdapter = adapters.get(opp.shortExchange);
             if (!longAdapter || !shortAdapter || opp.price <= 0)
                 continue;
+            if (await this.heldUntracked(opp.symbol, [[opp.longExchange, longAdapter], [opp.shortExchange, shortAdapter]], live, ctx))
+                continue;
+            this._traded.add(opp.symbol);
             const size = (this.sizeUsd / opp.price).toFixed(6);
-            try {
-                ctx.log(`  [ARB-V2] Opening ${opp.symbol}: spread ${opp.spread.toFixed(1)}% (long ${opp.longExchange}, short ${opp.shortExchange})`);
-                await Promise.all([
-                    longAdapter.marketOrder(opp.symbol, "buy", size),
-                    shortAdapter.marketOrder(opp.symbol, "sell", size),
-                ]);
+            ctx.log(`  [ARB-V2] Opening ${opp.symbol}: spread ${opp.spread.toFixed(1)}% (long ${opp.longExchange}, short ${opp.shortExchange})`);
+            const opened = await this.openPosition(ctx, opp, size, longAdapter, shortAdapter);
+            if (opened !== null) {
                 positions.push({
                     symbol: opp.symbol,
                     longExchange: opp.longExchange,
@@ -168,39 +227,191 @@ export class FundingArbV2Strategy {
                     entrySpread: opp.spread,
                     entryTime: now,
                     sizeUsd: this.sizeUsd,
+                    size: opened,
+                    longClosed: false,
+                    closeFailures: 0,
                 });
                 ctx.log(`  [ARB-V2] Position opened! (${positions.length}/${this.maxPositions})`);
             }
-            catch (err) {
-                const msg = err instanceof Error ? err.message : String(err);
-                ctx.log(`  [ARB-V2] Execution failed: ${msg}`);
-            }
         }
         ctx.state.set("positions", positions);
         // Return noop — arb executes directly via multi-adapter coordination
         return positions.length > 0 || toOpen.length > 0 ? [{ type: "noop" }] : [];
     }
+    /**
+     * Close every tracked position (reduce-only, verified). A position that cannot be
+     * closed is never dropped silently: it is quarantined (persisted with what is open)
+     * so no bot / arb path touches it until a human resolves it.
+     */
     async onStop(ctx) {
         const positions = ctx.state.get("positions");
-        if (positions.length === 0)
-            return [];
         const adapters = this.getAdapters(ctx);
+        const remaining = [];
         for (const pos of positions) {
-            const longAdapter = adapters.get(pos.longExchange);
-            const shortAdapter = adapters.get(pos.shortExchange);
-            if (longAdapter && shortAdapter) {
-                try {
-                    // Best-effort close all legs
-                    await Promise.all([
-                        longAdapter.cancelAllOrders(pos.symbol),
-                        shortAdapter.cancelAllOrders(pos.symbol),
-                    ]);
+            if (isQuarantined(pos.symbol, ctx.log)) {
+                ctx.log(`  [ARB-V2] CRITICAL: ${pos.symbol} is quarantined — left open on stop; resolve manually (perp strategy quarantine)`);
+                remaining.push(pos);
+                continue;
+            }
+            ctx.log(`  [ARB-V2] Closing ${pos.symbol} on stop (${pos.size})`);
+            if (!(await this.closePosition(ctx, pos, adapters))) {
+                remaining.push(pos);
+                if (!isQuarantined(pos.symbol, ctx.log)) {
+                    this.quarantine(pos, `still open when the bot stopped (long ${pos.longClosed ? "closed" : `${pos.size} on ${pos.longExchange}`}, short ${pos.size} on ${pos.shortExchange})`, ctx);
+                }
+            }
+        }
+        ctx.state.set("positions", remaining);
+        // Only the bot's own symbols, on each venue it used — never an account-wide cancel.
+        await cancelBotSymbols(adapters, this._traded, "[ARB-V2]", ctx.log);
+        return [];
+    }
+    /** A live position of `symbol` on one of the venues that the bot does not track (or an unreadable venue): no entry on top of it. */
+    async heldUntracked(symbol, venues, cache, ctx) {
+        const key = quarantineKey(symbol);
+        for (const [v, a] of venues) {
+            if (!cache.has(v))
+                cache.set(v, a.getPositions());
+            let list;
+            try {
+                list = await cache.get(v);
+            }
+            catch (e) {
+                ctx.log(`  [ARB-V2] Skip ${symbol}: ${v} positions unreadable (${e instanceof Error ? e.message : String(e)}) — no entry on unknown exposure`);
+                return true;
+            }
+            if (list.some(p => quarantineKey(p.symbol) === key && Number(p.size) !== 0)) {
+                ctx.log(`  [ARB-V2] Skip ${symbol}: a ${symbol} position on ${v} is not one of the bot's pairs — no entry on top of it`);
+                return true;
+            }
+        }
+        return false;
+    }
+    quarantine(pos, reason, ctx) {
+        quarantineSymbol(this.name, pos.symbol, reason, `Check ${pos.symbol} on ${pos.longExchange} and ${pos.shortExchange} (perp -e ${pos.longExchange} positions; perp -e ${pos.shortExchange} positions); close or re-hedge by hand, then: perp strategy quarantine-clear ${pos.symbol}`, ctx.log);
+    }
+    /**
+     * Long then short, each verified; the short is sized to the long's verified fill and
+     * any excess the short did not cover is trimmed off the long (reduce-only, verified).
+     * A definite short failure rolls the long back reduce-only. Returns the hedged size
+     * (both legs), or null when nothing is open / the symbol was quarantined.
+     */
+    async openPosition(ctx, opp, size, longAdapter, shortAdapter) {
+        const errText = (e) => e instanceof Error ? e.message : String(e);
+        const dec = sizeDecimalsOf(size);
+        let longFilled;
+        try {
+            longFilled = (await placePerpVerified(longAdapter, opp.longExchange, opp.symbol, "buy", size)).filledSize;
+        }
+        catch (err) {
+            if (isOutcomeUnknown(err))
+                this.quarantine(opp, `long leg ${size} on ${opp.longExchange} outcome unknown (${errText(err)}) — no short placed`, ctx);
+            else
+                ctx.log(`  [ARB-V2] Execution failed (long leg): ${errText(err)}`);
+            return null;
+        }
+        const longOpen = fillSizeString(longFilled, dec);
+        let shortFilled;
+        try {
+            shortFilled = (await placePerpVerified(shortAdapter, opp.shortExchange, opp.symbol, "sell", longOpen)).filledSize;
+        }
+        catch (err) {
+            const msg = errText(err);
+            if (isOutcomeUnknown(err)) {
+                // The short may exist: closing the long could leave a naked short.
+                this.quarantine(opp, `short leg ${longOpen} on ${opp.shortExchange} outcome unknown (${msg}); long ${longOpen} open on ${opp.longExchange} — not rolled back`, ctx);
+                return null;
+            }
+            ctx.log(`  [ARB-V2] Execution failed (short leg): ${msg} — rolling back the long (reduce-only)`);
+            try {
+                const rb = await closePerpVerified(longAdapter, opp.longExchange, opp.symbol, "sell", longOpen, { decimals: dec });
+                if (rb.residual > 0) {
+                    this.quarantine(opp, `short leg failed (${msg}); the long rollback closed ${fillSizeString(rb.closed, dec)} of ${longOpen}, ${fillSizeString(rb.residual, dec)} still open${rb.note !== null ? ` (${rb.note})` : ""} — orphaned long on ${opp.longExchange}`, ctx);
+                    return null;
                 }
-                catch { /* best effort */ }
+                ctx.log(`  [ARB-V2] Rollback OK: closed long ${longOpen} ${opp.symbol} on ${opp.longExchange}`);
+            }
+            catch (rbErr) {
+                this.quarantine(opp, `short leg failed (${msg}) and the long rollback failed or is unconfirmed (${errText(rbErr)}) — orphaned long ${longOpen} on ${opp.longExchange}`, ctx);
+            }
+            return null;
+        }
+        const hedged = fillSizeString(shortFilled, dec);
+        try {
+            const trimmed = await trimFirstLegToHedge(longAdapter, opp.longExchange, opp.symbol, "buy", longFilled, shortFilled, dec);
+            if (trimmed !== "0")
+                ctx.log(`  [ARB-V2] Short filled ${hedged} of ${longOpen} — trimmed long by ${trimmed} (reduce-only)`);
+        }
+        catch (trimErr) {
+            this.quarantine(opp, `short filled ${hedged} of ${longOpen}; trimming the long's excess on ${opp.longExchange} failed or is unconfirmed (${errText(trimErr)}) — long ${longOpen} vs short ${hedged}`, ctx);
+            return null;
+        }
+        return hedged;
+    }
+    /**
+     * Close both legs reduce-only with the opened size, verified. Reduce-only makes a
+     * retry safe (a leg already flat is rejected, never re-opened), so a definite failure
+     * of the LONG close keeps the (still hedged) position for the next tick; MAX_CLOSE_FAILURES
+     * in a row, an unknown outcome, or a short close that definitely failed once the long is
+     * closed (naked short) quarantines it. Returns true when both legs are flat.
+     */
+    async closePosition(ctx, pos, adapters) {
+        const longAdapter = adapters.get(pos.longExchange);
+        const shortAdapter = adapters.get(pos.shortExchange);
+        if (!longAdapter || !shortAdapter) {
+            ctx.log(`  [ARB-V2] Close ${pos.symbol}: adapter missing (${pos.longExchange}/${pos.shortExchange})`);
+            return false;
+        }
+        const errText = (e) => e instanceof Error ? e.message : String(e);
+        const failed = (why) => {
+            pos.closeFailures++;
+            ctx.log(`  [ARB-V2] Close failed for ${pos.symbol}: ${why} (${pos.closeFailures}/${MAX_CLOSE_FAILURES})`);
+            if (pos.closeFailures >= MAX_CLOSE_FAILURES)
+                this.quarantine(pos, `close failed ${pos.closeFailures} times in a row (last: ${why})`, ctx);
+            return false;
+        };
+        if (!pos.longClosed) {
+            try {
+                const lc = await closePerpVerified(longAdapter, pos.longExchange, pos.symbol, "sell", pos.size);
+                if (lc.residual > 0) {
+                    this.quarantine(pos, `close of long on ${pos.longExchange} closed ${lc.closed} of ${pos.size}, ${lc.residual} still open${lc.note !== null ? ` (${lc.note})` : ""}; short ${pos.size} on ${pos.shortExchange} not closed`, ctx);
+                    return false;
+                }
+                pos.longClosed = true;
+            }
+            catch (err) {
+                const msg = errText(err);
+                if (isOutcomeUnknown(err)) {
+                    this.quarantine(pos, `close of long ${pos.size} on ${pos.longExchange} outcome unknown (${msg})`, ctx);
+                    return false;
+                }
+                if (!isPositionGone(err))
+                    return failed(`long leg: ${msg}`);
+                pos.longClosed = true; // already flat
+            }
+        }
+        try {
+            const sc = await closePerpVerified(shortAdapter, pos.shortExchange, pos.symbol, "buy", pos.size);
+            if (sc.residual > 0) {
+                this.quarantine(pos, `the long is closed; close of short on ${pos.shortExchange} closed ${sc.closed} of ${pos.size}, ${sc.residual} still open${sc.note !== null ? ` (${sc.note})` : ""} — residual short unhedged`, ctx);
+                return false;
+            }
+        }
+        catch (err) {
+            const msg = errText(err);
+            if (isOutcomeUnknown(err)) {
+                this.quarantine(pos, `close of short ${pos.size} on ${pos.shortExchange} outcome unknown (${msg}); the long is closed`, ctx);
+                return false;
+            }
+            if (!isPositionGone(err)) {
+                // The long is closed, so the short is naked: quarantine NOW — never carry a naked
+                // short across ticks waiting for MAX_CLOSE_FAILURES.
+                this.quarantine(pos, `long closed on ${pos.longExchange}; close of short ${pos.size} on ${pos.shortExchange} failed (${msg}) — naked short`, ctx);
+                return false;
             }
         }
-        ctx.state.set("positions", []);
-        return [{ type: "cancel_all" }];
+        ctx.log(`  [ARB-V2] Closed ${pos.symbol} (long ${pos.longExchange}, short ${pos.shortExchange})`);
+        return true;
     }
     getAdapters(ctx) {
         const adapters = new Map();