wos-deploy 2.2.0__tar.gz → 2.3.0__tar.gz

{wos_deploy-2.2.0/src/wos_deploy.egg-info → wos_deploy-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wos-deploy
-Version: 2.2.0
+Version: 2.3.0
 Summary: Deploy and operate WOS server instances (ring, global, tick-quote add-on) with Docker Compose on published images
 License-Expression: Apache-2.0
 Keywords: wos,deployment,docker,docker-compose

{wos_deploy-2.2.0 → wos_deploy-2.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "wos-deploy"
-version = "2.2.0"
+version = "2.3.0"
 description = "Deploy and operate WOS server instances (ring, global, tick-quote add-on) with Docker Compose on published images"
 readme = "README.md"
 requires-python = ">=3.9"

{wos_deploy-2.2.0 → wos_deploy-2.3.0}/src/wos_deploy/__init__.py

@@ -1,3 +1,3 @@
 """WOS Deployment Manager."""
 
-__version__ = "2.2.0"
+__version__ = "2.3.0"

{wos_deploy-2.2.0 → wos_deploy-2.3.0}/src/wos_deploy/templates/README.md

@@ -281,13 +281,39 @@ rendered there into a 0600 ini; never in `.env`, the compose config or a log. `w
 shows the two endpoint addresses and prints everything else as `***`. Re-running
 `setup-rfq` re-renders everything and keeps stored inputs unless re-entered.
 
+**What the feeder captures, and how it follows a roll.** Two inputs decide the set:
+
+- the **markets** are those the full ring's web API lists for the client uuid
+  (`--dirac-client-uuid`: the markets whose `dirac` is that uuid);
+- the **commodities** of those markets are `--commodities` (`WOS_RFQ_COMMODITIES`): a comma list of
+  `MARKET:commodity` entries, `MARKET:*` for every commodity of a market, or `*` (the default) for
+  every commodity that has a `<00>` mapping. An entry the market list does not hold stops the
+  feeder at start with one `FATAL` line.
+
+| Setting (flag / variable) | Default | Meaning |
+|---|---|---|
+| `--subscribe` / `WOS_RFQ_SUBSCRIBE` | `all` | `all`: every listed (unexpired) contract of each captured commodity, so a roll never loses data; `main`: only the `<00>` main, plus the previous main through the trade day of a roll (`roll <market> <commodity>: <old> -> <new>` is logged once) |
+| `--roll-source` / `WOS_RFQ_ROLL_SOURCE` | `metadata` | the main contract is the web API's `<commodity><00>` → `real_contract_code`, the ring's own mapping, so the add-on's main never deviates from the ring's. `open_interest` is reserved (refused) |
+| `--commodities` / `WOS_RFQ_COMMODITIES` | `*` | see above |
+| `--market-fails` / `WOS_RFQ_MARKET_FAILS` | `6` | consecutive market-list failures, with no list held, before the feeder exits non-zero |
+
+The settings are kept in `./rfq-ctp.env` with the inputs above (they are not secret; `wos-deploy
+config` shows them). The feeder keeps its last good market list in `rfq_markets.json` in its work
+directory (`volumes/data-secondary/rfq-ctp/`, or `volumes/data/rfq/ctp/` on the primary) and uses it
+at start when the web API fails (`market list from cache (age …)`). A failing market list is retried
+with back-off (15 s, 30 s, 60 s … 5 min), each attempt logged with its count; a rejected token
+(`Invalid token or token expired`) makes it log in again with its own account. With no list and no
+cache it exits after `WOS_RFQ_MARKET_FAILS` attempts (about 8 minutes at the default).
+**The market account must be used by this feeder only:** one account admits one session, and any
+other login of it (a person, another feeder) evicts the feeder's.
+
 What the developer must bring (the tool cannot provide it):
 
 | Needed | Without it |
 |---|---|
 | an image tag carrying `WOS_ROLE=rfq` | `setup-rfq: <image> has no WOS_ROLE=rfq start role ...` and nothing is initialised |
 | (realtime) a CTP account: broker id, investor id, password, a market-data front | front unreachable: `[wos-rfq] ERROR: CTP front ... is NOT reachable`, then `ERROR: no CTP login after 60 s and no answer from the front`; refused login: `ERROR: CTP login REFUSED ... OnRspUserLogin <code> (<msg>)` |
-| (realtime) an account + a client uuid on a full ring's web API (the market list) | the feeder log shows `query_markets failed status=...` and subscribes nothing |
+| (realtime) an account + a client uuid on a full ring's web API (the market list), used by nothing else | the feeder log shows `market list fetch failed (attempt N …)` with back-off; with no cached list it exits after `WOS_RFQ_MARKET_FAILS` attempts with one `FATAL` line |
 | (realtime) an x64 host, or amd64 emulation on arm64 | the feeder binary exists only in the amd64 image |
 | (realtime) an image whose feeder reconnects to a restarted master | after `wos-deploy restart wos-primary` an older feeder stays disconnected: restart `wos-secondary` too (the tool warns) |
 

{wos_deploy-2.2.0 → wos_deploy-2.3.0}/src/wos_deploy/templates/scripts/functions/private/100_rfq_setup_addon.py

@@ -85,9 +85,9 @@ fi
 mkdir -p "$INI_DIR" "$WORK"
 chown wolverine:wolverine "$INI_DIR" "$WORK" 2>/dev/null || true
 chmod 700 "$INI_DIR"
-{PY} - "$SECRETS" "$INI" "$MASTER_HOST" "$MASTER_PORT" "$FEEDER_LOG" <<'PYEOF' || {{ log "ERROR: the feeder ini could not be rendered (see the line above) - the realtime feeder is NOT started"; exit 0; }}
+{PY} - "$SECRETS" "$INI" "$MASTER_HOST" "$MASTER_PORT" "$FEEDER_LOG" "$WORK" <<'PYEOF' || {{ log "ERROR: the feeder ini could not be rendered (see the line above) - the realtime feeder is NOT started"; exit 0; }}
 import configparser, os, socket, sys
-src, dst, mhost, mport, flog = sys.argv[1:6]
+src, dst, mhost, mport, flog, work = sys.argv[1:7]
 vals = {{}}
 for raw in open(src, encoding="utf-8"):
     line = raw.rstrip("\\n")
@@ -108,6 +108,24 @@ ini["dirac_client"] = {{"uuid": vals["WOS_RFQ_DIRAC_CLIENT_UUID"]}}
 ini["slave"] = {{"ip": mhost, "port": mport, "namespace": "1"}}
 ini["ctp"] = {{"broker_id": vals["WOS_RFQ_CTP_BROKER_ID"], "user_id": vals["WOS_RFQ_CTP_INVESTOR_ID"],
               "password": vals["WOS_RFQ_CTP_PASSWORD"], "front_end": vals["WOS_RFQ_CTP_FRONT_END"]}}
+# The capture settings (not secret; defaults when the secrets file predates them):
+#   subscribe    all  = every listed contract of each captured commodity, so a roll never loses data
+#                main = the <00> main only, plus the previous main through the trade day of a roll
+#   roll_source  metadata = the main is the web API's <commodity><00> -> real_contract_code, i.e. the
+#                ring's own mapping, so the stored main never deviates from the ring's (open_interest is
+#                reserved: the feeder refuses it)
+#   commodities  MARKET:commodity,... | MARKET:* | * -- WHICH commodities; the MARKETS are those the web API
+#                lists for [dirac_client] uuid. An entry the list does not hold stops the feeder at start.
+#   market_fails consecutive market-list failures with no list held (no web answer, no cache) before the
+#                feeder exits non-zero; retries back off 15 s, 30 s, 60 s ... 5 min
+#   market_cache the last good market list, used at start when the web API fails (kept on the bind mount)
+# The market account ([web_service]) must be used by this feeder ONLY: one account admits one session and
+# a later login (a person, another feeder) evicts this one; the feeder then re-logs in with back-off.
+ini["rfq"] = {{"subscribe": vals.get("WOS_RFQ_SUBSCRIBE") or "all",
+              "roll_source": vals.get("WOS_RFQ_ROLL_SOURCE") or "metadata",
+              "commodities": vals.get("WOS_RFQ_COMMODITIES") or "*",
+              "market_fails": vals.get("WOS_RFQ_MARKET_FAILS") or "6",
+              "market_cache": work + "/rfq_markets.json"}}
 ini["logger"] = {{"log_file_path": flog}}
 old = os.umask(0o077)
 with open(dst, "w", encoding="utf-8") as handle:
@@ -115,7 +133,8 @@ with open(dst, "w", encoding="utf-8") as handle:
 os.umask(old)
 os.chmod(dst, 0o600)
 shown_keys = {{("web_service", "url"), ("ctp", "front_end"), ("slave", "ip"), ("slave", "port"),
-              ("slave", "namespace"), ("logger", "log_file_path")}}
+              ("slave", "namespace"), ("logger", "log_file_path"), ("rfq", "subscribe"), ("rfq", "roll_source"),
+              ("rfq", "commodities"), ("rfq", "market_fails"), ("rfq", "market_cache")}}
 for sec in ini.sections():
     for key, value in ini[sec].items():
         shown = value if (sec, key) in shown_keys else "***"

{wos_deploy-2.2.0 → wos_deploy-2.3.0}/src/wos_deploy/templates/wos-deploy.py

@@ -24,7 +24,7 @@ from typing import Dict, Optional
 
 # Kept equal to wos-deploy-pkg's version (pyproject.toml, wos_deploy/__init__.py);
 # wos-deploy-pkg/build.sh refuses to build when they differ.
-WOS_DEPLOY_VERSION = "2.2.0"
+WOS_DEPLOY_VERSION = "2.3.0"
 
 DEFAULT_ENV = "prod"
 DEFAULT_PORT_OFFSET = 3
@@ -131,6 +131,54 @@ RFQ_CTP_INPUTS = (
     ("WOS_RFQ_MARKET_PASSWORD", None, "market-list web API password", True),
     ("WOS_RFQ_DIRAC_CLIENT_UUID", "dirac_client_uuid", "client uuid the feeder identifies with on that ring", False),
 )
+# (key, flag, default, help) -- the feeder's non-secret capture settings. Stored in the same
+# RFQ_SECRETS_FILE (the one file the feeder launcher reads) and shown in clear by `config`.
+RFQ_FEEDER_SETTINGS = (
+    ("WOS_RFQ_SUBSCRIBE", "rfq_subscribe", "all",
+     "capture scope: all = every listed contract of each captured commodity (a roll never loses data); "
+     "main = only the <00> main, plus the previous main through the trade day of a roll"),
+    ("WOS_RFQ_ROLL_SOURCE", "rfq_roll_source", "metadata",
+     "which contract is the main: metadata = the web API's <commodity><00> -> real_contract_code, the "
+     "ring's own mapping (open_interest is reserved; the feeder refuses it)"),
+    ("WOS_RFQ_COMMODITIES", "rfq_commodities", "*",
+     "the commodities captured: MARKET:commodity,...; MARKET:* = every commodity of a market; * = every "
+     "commodity (with a <00>) of every market the web API serves for the client uuid"),
+    ("WOS_RFQ_MARKET_FAILS", "rfq_market_fails", "6",
+     "consecutive market-list failures, with no list held (neither a web answer nor the on-disk cache), "
+     "after which the feeder exits non-zero (retries back off 15 s, 30 s, ... 5 min: 6 = ~8 min)"),
+)
+RFQ_COMMODITY_ENTRY = re.compile(r"^[A-Za-z0-9_]+:([A-Za-z0-9_]+|\*)$")
+
+
+def resolve_rfq_feeder_settings(args: argparse.Namespace, existing: Dict[str, str]) -> Dict[str, str]:
+    """flag > environment > the existing secrets file > the default; validated here so a bad value
+    stops setup-rfq instead of the feeder."""
+    values: Dict[str, str] = {}
+    for key, flag, default, _ in RFQ_FEEDER_SETTINGS:
+        value = getattr(args, flag, None) or os.environ.get(key) or existing.get(key) or default
+        values[key] = str(value).strip()
+    if values["WOS_RFQ_SUBSCRIBE"] not in ("all", "main"):
+        raise SystemExit(f"WOS_RFQ_SUBSCRIBE must be all or main, got {values['WOS_RFQ_SUBSCRIBE']!r}")
+    if values["WOS_RFQ_ROLL_SOURCE"] == "open_interest":
+        raise SystemExit("WOS_RFQ_ROLL_SOURCE=open_interest is reserved: the feeder follows the ring's <00> "
+                         "mapping only (metadata), so the stored main never deviates from the ring's. The "
+                         "open-interest main is a client-side reading of the stored ticks.")
+    if values["WOS_RFQ_ROLL_SOURCE"] != "metadata":
+        raise SystemExit(f"WOS_RFQ_ROLL_SOURCE must be metadata, got {values['WOS_RFQ_ROLL_SOURCE']!r}")
+    if not values["WOS_RFQ_MARKET_FAILS"].isdigit() or int(values["WOS_RFQ_MARKET_FAILS"]) < 1:
+        raise SystemExit(f"WOS_RFQ_MARKET_FAILS must be a positive integer, got {values['WOS_RFQ_MARKET_FAILS']!r}")
+    entries = [e.strip() for e in re.split(r"[,\s]+", values["WOS_RFQ_COMMODITIES"]) if e.strip()]
+    if not entries or "*" in entries:
+        values["WOS_RFQ_COMMODITIES"] = "*"
+    else:
+        bad = [e for e in entries if not RFQ_COMMODITY_ENTRY.match(e)]
+        if bad:
+            raise SystemExit("WOS_RFQ_COMMODITIES entries must be MARKET:commodity, MARKET:* or *; got "
+                             + ", ".join(repr(b) for b in bad))
+        values["WOS_RFQ_COMMODITIES"] = ",".join(entries)
+    return values
+
+
 class Colors:
     RED = "\033[91m"
     GREEN = "\033[92m"
@@ -551,6 +599,7 @@ class WOSDeployment:
             if '"' in values[key] or "\\" in values[key] or "\n" in values[key]:
                 raise SystemExit(f"{key} contains a quote, a backslash or a newline, which the feeder "
                                  "cannot pass through (it writes the value into a JSON login body / an ini)")
+        values.update(resolve_rfq_feeder_settings(args, existing))
         kept = [k for k, _, _, _ in RFQ_CTP_INPUTS if existing.get(k) and existing.get(k) == values[k]]
         if kept:
             self.logger.info(f"setup-rfq: kept from {RFQ_SECRETS_FILE}: {', '.join(kept)}")
@@ -566,6 +615,12 @@ class WOSDeployment:
             handle.write("# The tick-quote add-on's realtime inputs (wos-deploy setup-rfq). Mode 0600; never commit or share.\n")
             for key, _, _, _ in RFQ_CTP_INPUTS:
                 handle.write(f"{key}={values[key]}\n")
+            handle.write("# The feeder's capture settings (not secret). The market account above must be used by\n"
+                         "# this feeder ONLY: one account admits one session, a later login (a person, another\n"
+                         "# feeder) evicts this one -- the feeder then re-logs in with back-off, and with no\n"
+                         "# market list and no cache it exits after WOS_RFQ_MARKET_FAILS failures.\n")
+            for key, _, default, _ in RFQ_FEEDER_SETTINGS:
+                handle.write(f"{key}={values.get(key, default)}\n")
         os.chmod(tmp, 0o600)
         os.replace(tmp, path)
         self.logger.info(f"setup-rfq: wrote {path.name} (mode 0600)")
@@ -1158,6 +1213,9 @@ class WOSDeployment:
                 if key in inputs:
                     shown = inputs[key] if key in RFQ_SHOWN_INPUTS or not inputs[key] else "***"
                     self.logger.info(f"{key}={shown}")
+            if inputs:
+                for key, _, default, _ in RFQ_FEEDER_SETTINGS:
+                    self.logger.info(f"{key}={inputs.get(key) or default}")
 
     def confirm_setup(self, namespace: str, env_vars: Dict[str, str],
                       rfq_inputs: Optional[Dict[str, str]] = None) -> None:
@@ -1839,7 +1897,16 @@ def build_parser() -> argparse.ArgumentParser:
     rfq.add_argument("--market-account", dest="market_account",
                      help="account on that web API   [WOS_RFQ_MARKET_ACCOUNT]; password: [WOS_RFQ_MARKET_PASSWORD] or prompt")
     rfq.add_argument("--dirac-client-uuid", dest="dirac_client_uuid",
-                     help="client uuid the feeder identifies with on that ring   [WOS_RFQ_DIRAC_CLIENT_UUID]")
+                     help="client uuid the feeder identifies with on that ring   [WOS_RFQ_DIRAC_CLIENT_UUID]; "
+                     "it selects the MARKETS: the web API lists the markets whose dirac is this uuid")
+    rfq.add_argument("--subscribe", dest="rfq_subscribe", choices=["all", "main"],
+                     help="capture scope (default all)   [WOS_RFQ_SUBSCRIBE]: " + RFQ_FEEDER_SETTINGS[0][3])
+    rfq.add_argument("--roll-source", dest="rfq_roll_source", choices=["metadata", "open_interest"],
+                     help="main-contract source (default metadata)   [WOS_RFQ_ROLL_SOURCE]: " + RFQ_FEEDER_SETTINGS[1][3])
+    rfq.add_argument("--commodities", dest="rfq_commodities", metavar="LIST",
+                     help="commodities captured (default *)   [WOS_RFQ_COMMODITIES]: " + RFQ_FEEDER_SETTINGS[2][3])
+    rfq.add_argument("--market-fails", dest="rfq_market_fails", metavar="N",
+                     help="(default 6)   [WOS_RFQ_MARKET_FAILS]: " + RFQ_FEEDER_SETTINGS[3][3])
 
     deploy = subparsers.add_parser("deploy", help="Start services")
     deploy.add_argument("--skip-pull", action="store_true")

{wos_deploy-2.2.0 → wos_deploy-2.3.0/src/wos_deploy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wos-deploy
-Version: 2.2.0
+Version: 2.3.0
 Summary: Deploy and operate WOS server instances (ring, global, tick-quote add-on) with Docker Compose on published images
 License-Expression: Apache-2.0
 Keywords: wos,deployment,docker,docker-compose