vortex-api 2.1.8b1__tar.gz → 2.1.8b3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {vortex_api-2.1.8b1/vortex_api.egg-info → vortex_api-2.1.8b3}/PKG-INFO +1 -1
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b3}/pyproject.toml +1 -1
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b3}/vortex_api/__init__.py +2 -1
- vortex_api-2.1.8b3/vortex_api/__version__.py +1 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b3}/vortex_api/api.py +16 -0
- vortex_api-2.1.8b3/vortex_api/instruments.py +657 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b3/vortex_api.egg-info}/PKG-INFO +1 -1
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b3}/vortex_api.egg-info/SOURCES.txt +1 -0
- vortex_api-2.1.8b1/vortex_api/__version__.py +0 -1
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b3}/LICENSE +0 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b3}/README.md +0 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b3}/setup.cfg +0 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b3}/vortex_api/backtest/__init__.py +0 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b3}/vortex_api/backtest/_backtestingpy.py +0 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b3}/vortex_api/backtest/_backtrader.py +0 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b3}/vortex_api/backtest/_common.py +0 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b3}/vortex_api/backtest/_vectorbt.py +0 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b3}/vortex_api/vortex_feed.py +0 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b3}/vortex_api.egg-info/dependency_links.txt +0 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b3}/vortex_api.egg-info/requires.txt +0 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b3}/vortex_api.egg-info/top_level.txt +0 -0
|
@@ -35,4 +35,5 @@ Getting started
|
|
|
35
35
|
from __future__ import unicode_literals, absolute_import
|
|
36
36
|
from vortex_api.api import VortexAPI,Constants
|
|
37
37
|
from vortex_api.vortex_feed import VortexFeed
|
|
38
|
-
|
|
38
|
+
from vortex_api.instruments import InstrumentManager, Instrument, InstrumentNotFound
|
|
39
|
+
__all__ = [VortexAPI,Constants,VortexFeed,InstrumentManager,Instrument,InstrumentNotFound]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "2.1.8b3"
|
|
@@ -205,6 +205,22 @@ class VortexAPI:
|
|
|
205
205
|
self.enable_logging = enable_logging
|
|
206
206
|
if self.enable_logging:
|
|
207
207
|
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
|
208
|
+
|
|
209
|
+
self._instruments = None # lazily constructed InstrumentManager
|
|
210
|
+
|
|
211
|
+
@property
|
|
212
|
+
def instruments(self):
|
|
213
|
+
"""Lazy ``InstrumentManager`` for this client.
|
|
214
|
+
|
|
215
|
+
The first access constructs the manager but does **not** download
|
|
216
|
+
``master.csv``; the download happens on the first lookup (or on an
|
|
217
|
+
explicit ``.load()`` call) and is cached in memory and on disk so
|
|
218
|
+
subsequent script runs hit the network at most once per IST trading day.
|
|
219
|
+
"""
|
|
220
|
+
if self._instruments is None:
|
|
221
|
+
from vortex_api.instruments import InstrumentManager
|
|
222
|
+
self._instruments = InstrumentManager()
|
|
223
|
+
return self._instruments
|
|
208
224
|
|
|
209
225
|
def _make_api_request(self, method: str, endpoint: str, data: dict = None, params=None) -> dict:
|
|
210
226
|
"""
|
|
@@ -0,0 +1,657 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Instrument manager for the Vortex SDK.
|
|
3
|
+
|
|
4
|
+
Downloads the Vortex instruments master (``master.csv``) and exposes fast
|
|
5
|
+
lookups by ticker, ``(exchange, token)``, or ISIN. The master file is cached
|
|
6
|
+
both in memory and on disk so that ``master.csv`` is fetched at most once per
|
|
7
|
+
trading day, even across process restarts.
|
|
8
|
+
|
|
9
|
+
Typical use::
|
|
10
|
+
|
|
11
|
+
from vortex_api import VortexAPI
|
|
12
|
+
|
|
13
|
+
client = VortexAPI(...)
|
|
14
|
+
|
|
15
|
+
inst = client.instruments.get_by_ticker("NSE:RELIANCE")
|
|
16
|
+
print(inst.token, inst.exchange, inst.lot_size)
|
|
17
|
+
|
|
18
|
+
# Reverse lookup
|
|
19
|
+
inst = client.instruments.get_by_exchange_token("NSE_EQ", 2885)
|
|
20
|
+
|
|
21
|
+
# All option contracts under NIFTY
|
|
22
|
+
chain = client.instruments.all_by_underlying("NSE_FO", "NIFTY")
|
|
23
|
+
|
|
24
|
+
Environment variables (all optional — the SDK works with zero config)
|
|
25
|
+
---------------------------------------------------------------------
|
|
26
|
+
|
|
27
|
+
The SDK reads these at ``InstrumentManager()`` construction so that infra
|
|
28
|
+
changes (mounting a shared volume, switching to an internal mirror, etc.)
|
|
29
|
+
take effect immediately on the next pod restart — no code change required.
|
|
30
|
+
|
|
31
|
+
``PYVORTEX_CACHE_DIR``
|
|
32
|
+
Directory for the on-disk cache. Set this when running in containers
|
|
33
|
+
with read-only root filesystems or when you want to point at a shared
|
|
34
|
+
volume (e.g. EFS) so the entire fleet shares one ``master.csv``.
|
|
35
|
+
Default: ``$XDG_CACHE_HOME/pyvortex`` or ``~/.cache/pyvortex`` or
|
|
36
|
+
``/tmp/pyvortex`` (first writable wins).
|
|
37
|
+
|
|
38
|
+
``PYVORTEX_MASTER_URL``
|
|
39
|
+
Override the URL used to download ``master.csv``. Point this at an
|
|
40
|
+
internal S3 mirror or pre-warmed CDN to keep fleet-wide traffic off
|
|
41
|
+
the public origin. Default: ``https://static.rupeezy.in/master.csv``.
|
|
42
|
+
|
|
43
|
+
``PYVORTEX_REVALIDATE_SECONDS``
|
|
44
|
+
How long (in seconds) to trust the disk cache before issuing a
|
|
45
|
+
conditional GET. Default: 900 (15 minutes). Set to ``0`` to revalidate
|
|
46
|
+
on every load.
|
|
47
|
+
|
|
48
|
+
``PYVORTEX_REQUEST_TIMEOUT``
|
|
49
|
+
HTTP request timeout in seconds for the master download. Default: 30.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
from __future__ import annotations
|
|
53
|
+
|
|
54
|
+
import csv
|
|
55
|
+
import datetime
|
|
56
|
+
import gzip
|
|
57
|
+
import io
|
|
58
|
+
import json
|
|
59
|
+
import logging
|
|
60
|
+
import os
|
|
61
|
+
import tempfile
|
|
62
|
+
import threading
|
|
63
|
+
from dataclasses import dataclass, field
|
|
64
|
+
from typing import Callable, Dict, List, Optional
|
|
65
|
+
|
|
66
|
+
import requests
|
|
67
|
+
|
|
68
|
+
DEFAULT_MASTER_URL = "https://static.rupeezy.in/master.csv"
|
|
69
|
+
DEFAULT_REVALIDATE_AFTER_SECONDS = 15 * 60 # how long to trust disk before conditional GET
|
|
70
|
+
DEFAULT_REQUEST_TIMEOUT_SECONDS = 30.0
|
|
71
|
+
_IST_OFFSET = datetime.timedelta(hours=5, minutes=30)
|
|
72
|
+
|
|
73
|
+
logger = logging.getLogger(__name__)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _env_int(name: str, default: int) -> int:
|
|
77
|
+
raw = os.environ.get(name)
|
|
78
|
+
if raw is None or raw == "":
|
|
79
|
+
return default
|
|
80
|
+
try:
|
|
81
|
+
return int(raw)
|
|
82
|
+
except ValueError:
|
|
83
|
+
logger.warning("pyvortex: ignoring invalid %s=%r (expected int)", name, raw)
|
|
84
|
+
return default
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _env_float(name: str, default: float) -> float:
|
|
88
|
+
raw = os.environ.get(name)
|
|
89
|
+
if raw is None or raw == "":
|
|
90
|
+
return default
|
|
91
|
+
try:
|
|
92
|
+
return float(raw)
|
|
93
|
+
except ValueError:
|
|
94
|
+
logger.warning("pyvortex: ignoring invalid %s=%r (expected float)", name, raw)
|
|
95
|
+
return default
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class InstrumentNotFound(Exception):
|
|
99
|
+
"""Raised when a lookup cannot find the requested instrument."""
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@dataclass
|
|
103
|
+
class Instrument:
|
|
104
|
+
"""A single tradable instrument row from ``master.csv``.
|
|
105
|
+
|
|
106
|
+
Fields mirror the columns produced by the Vortex master file. Missing or
|
|
107
|
+
empty columns produce default values rather than raising.
|
|
108
|
+
"""
|
|
109
|
+
|
|
110
|
+
ticker: str = "" # e.g. "NSE:RELIANCE"
|
|
111
|
+
token: int = 0
|
|
112
|
+
exchange: str = "" # e.g. "NSE_EQ", "NSE_FO"
|
|
113
|
+
instrument_name: str = ""
|
|
114
|
+
symbol: str = ""
|
|
115
|
+
series: str = ""
|
|
116
|
+
expiry_date: str = ""
|
|
117
|
+
isin: str = ""
|
|
118
|
+
option_type: str = ""
|
|
119
|
+
strike_price: float = 0.0
|
|
120
|
+
tick: float = 0.0
|
|
121
|
+
lot_size: int = 0
|
|
122
|
+
eligibility: bool = False
|
|
123
|
+
name: str = ""
|
|
124
|
+
last_trading_date: Optional[datetime.datetime] = None
|
|
125
|
+
asm_gsm_stage: str = ""
|
|
126
|
+
extras: Dict[str, str] = field(default_factory=dict)
|
|
127
|
+
|
|
128
|
+
@property
|
|
129
|
+
def exchange_token_key(self) -> str:
|
|
130
|
+
"""Composite key used internally for ``(exchange, token)`` lookups.
|
|
131
|
+
|
|
132
|
+
Matches the Go package's ``TokenIdentifier`` format: first three
|
|
133
|
+
characters of the exchange (NSE/BSE/MCX) joined with the token.
|
|
134
|
+
"""
|
|
135
|
+
return _exchange_token_key(self.exchange, self.token)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _exchange_token_key(exchange: str, token: int) -> str:
|
|
139
|
+
short = (exchange or "")[:3]
|
|
140
|
+
return f"{short}:{int(token)}"
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _now_ist() -> datetime.datetime:
|
|
144
|
+
return datetime.datetime.now(datetime.timezone.utc).astimezone(
|
|
145
|
+
datetime.timezone(_IST_OFFSET)
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _default_cache_path() -> str:
|
|
150
|
+
"""Resolve where to cache ``master.csv``.
|
|
151
|
+
|
|
152
|
+
Resolution order (first usable wins):
|
|
153
|
+
|
|
154
|
+
1. ``$PYVORTEX_CACHE_DIR`` — explicit override, recommended for containers
|
|
155
|
+
and multi-process deployments where you want to point at a shared volume.
|
|
156
|
+
2. ``$XDG_CACHE_HOME`` — standard XDG location.
|
|
157
|
+
3. ``~/.cache`` — only if ``$HOME`` is set and not ``/``.
|
|
158
|
+
4. ``/tmp`` — last-resort fallback for containers/serverless where the
|
|
159
|
+
above are unavailable. Survives the process but not the container.
|
|
160
|
+
"""
|
|
161
|
+
explicit = os.environ.get("PYVORTEX_CACHE_DIR")
|
|
162
|
+
if explicit:
|
|
163
|
+
return os.path.join(explicit, "master.csv")
|
|
164
|
+
|
|
165
|
+
xdg = os.environ.get("XDG_CACHE_HOME")
|
|
166
|
+
if xdg:
|
|
167
|
+
return os.path.join(xdg, "pyvortex", "master.csv")
|
|
168
|
+
|
|
169
|
+
home = os.environ.get("HOME")
|
|
170
|
+
if home and home not in ("/", ""):
|
|
171
|
+
expanded = os.path.expanduser("~")
|
|
172
|
+
# ``expanduser`` returns ``~`` verbatim when no home can be resolved
|
|
173
|
+
# (e.g. distroless container, no /etc/passwd entry). Guard against that.
|
|
174
|
+
if expanded != "~":
|
|
175
|
+
return os.path.join(expanded, ".cache", "pyvortex", "master.csv")
|
|
176
|
+
|
|
177
|
+
return os.path.join(tempfile.gettempdir(), "pyvortex", "master.csv")
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
class InstrumentManager:
|
|
181
|
+
"""Thread-safe, lazily-loaded instruments cache.
|
|
182
|
+
|
|
183
|
+
Args:
|
|
184
|
+
master_url: Override the master CSV URL. Falls back to the
|
|
185
|
+
``PYVORTEX_MASTER_URL`` env var, then to the public default.
|
|
186
|
+
cache_path: Where to persist the downloaded master file. Falls back to
|
|
187
|
+
the ``PYVORTEX_CACHE_DIR`` env var, then to ``~/.cache/pyvortex/master.csv``
|
|
188
|
+
(or ``/tmp/pyvortex/master.csv`` if HOME isn't writable).
|
|
189
|
+
enable_disk_cache: If ``False``, every load re-downloads. Default ``True``.
|
|
190
|
+
Mostly useful for tests; in production prefer mounting a writable
|
|
191
|
+
volume and letting the cache do its job.
|
|
192
|
+
request_timeout: HTTP request timeout in seconds. Falls back to
|
|
193
|
+
``PYVORTEX_REQUEST_TIMEOUT``. Default 30.
|
|
194
|
+
revalidate_after_seconds: How long to trust the disk cache without
|
|
195
|
+
re-checking the server. Falls back to ``PYVORTEX_REVALIDATE_SECONDS``.
|
|
196
|
+
Default 900 (15 minutes). Set to 0 to revalidate on every load.
|
|
197
|
+
|
|
198
|
+
Explicit Python arguments always win over env vars; env vars win over
|
|
199
|
+
built-in defaults. This lets operators tune behaviour from the pod spec
|
|
200
|
+
while still letting tests pass explicit values.
|
|
201
|
+
"""
|
|
202
|
+
|
|
203
|
+
def __init__(
|
|
204
|
+
self,
|
|
205
|
+
master_url: Optional[str] = None,
|
|
206
|
+
cache_path: Optional[str] = None,
|
|
207
|
+
enable_disk_cache: Optional[bool] = None,
|
|
208
|
+
request_timeout: Optional[float] = None,
|
|
209
|
+
revalidate_after_seconds: Optional[int] = None,
|
|
210
|
+
) -> None:
|
|
211
|
+
self._master_url = (
|
|
212
|
+
master_url
|
|
213
|
+
or os.environ.get("PYVORTEX_MASTER_URL")
|
|
214
|
+
or DEFAULT_MASTER_URL
|
|
215
|
+
)
|
|
216
|
+
self._cache_path = cache_path or _default_cache_path()
|
|
217
|
+
self._meta_path = self._cache_path + ".meta.json"
|
|
218
|
+
|
|
219
|
+
self._enable_disk_cache = True if enable_disk_cache is None else bool(enable_disk_cache)
|
|
220
|
+
|
|
221
|
+
self._request_timeout = (
|
|
222
|
+
request_timeout
|
|
223
|
+
if request_timeout is not None
|
|
224
|
+
else _env_float("PYVORTEX_REQUEST_TIMEOUT", DEFAULT_REQUEST_TIMEOUT_SECONDS)
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
if revalidate_after_seconds is None:
|
|
228
|
+
revalidate_after_seconds = _env_int(
|
|
229
|
+
"PYVORTEX_REVALIDATE_SECONDS", DEFAULT_REVALIDATE_AFTER_SECONDS
|
|
230
|
+
)
|
|
231
|
+
self._revalidate_after_seconds = max(0, int(revalidate_after_seconds))
|
|
232
|
+
|
|
233
|
+
self._lock = threading.RLock()
|
|
234
|
+
self._by_ticker: Dict[str, Instrument] = {}
|
|
235
|
+
self._by_exchange_token: Dict[str, Instrument] = {}
|
|
236
|
+
self._by_isin: Dict[str, List[Instrument]] = {}
|
|
237
|
+
self._last_loaded_at: Optional[datetime.datetime] = None
|
|
238
|
+
self._loaded = False
|
|
239
|
+
|
|
240
|
+
# Container-safety: if the configured cache path can't be written
|
|
241
|
+
# (read-only rootfs is common on EKS / Cloud Run / Lambda), we transparently
|
|
242
|
+
# fall back to a path under the OS temp dir. The fallback is computed
|
|
243
|
+
# lazily and remembered so we don't spam logs.
|
|
244
|
+
self._disk_disabled = False # set True if even the /tmp fallback fails
|
|
245
|
+
self._fallback_announced = False
|
|
246
|
+
|
|
247
|
+
# ------------------------------------------------------------------ Public
|
|
248
|
+
|
|
249
|
+
def load(self, force: bool = False) -> int:
|
|
250
|
+
"""Ensure instruments are loaded. Returns the instrument count.
|
|
251
|
+
|
|
252
|
+
Freshness strategy (in order, cheapest first):
|
|
253
|
+
|
|
254
|
+
1. In-memory cache: already populated this process → no I/O at all.
|
|
255
|
+
2. Disk cache + revalidation cooldown: if we validated less than
|
|
256
|
+
``revalidate_after_seconds`` ago, trust the disk cache without
|
|
257
|
+
touching the network.
|
|
258
|
+
3. Conditional GET: send ``If-None-Match`` / ``If-Modified-Since`` from
|
|
259
|
+
the cached metadata. A ``304`` response confirms the cache is
|
|
260
|
+
current (a few hundred bytes on the wire). A ``200`` brings the new
|
|
261
|
+
master and we re-index.
|
|
262
|
+
4. Unconditional GET: only when there is no usable disk cache.
|
|
263
|
+
|
|
264
|
+
Set ``force=True`` to bypass every cache and re-download.
|
|
265
|
+
"""
|
|
266
|
+
with self._lock:
|
|
267
|
+
if not force and self._loaded:
|
|
268
|
+
return len(self._by_exchange_token)
|
|
269
|
+
|
|
270
|
+
csv_text: Optional[str] = None
|
|
271
|
+
meta = self._read_meta() if self._enable_disk_cache else None
|
|
272
|
+
|
|
273
|
+
if not force and meta is not None and self._has_disk_cache():
|
|
274
|
+
# (2) Recent revalidation → skip the network entirely.
|
|
275
|
+
if self._within_revalidation_cooldown(meta):
|
|
276
|
+
csv_text = self._read_disk_cache()
|
|
277
|
+
if csv_text is not None:
|
|
278
|
+
logger.debug("Using disk cache (within cooldown): %s", self._cache_path)
|
|
279
|
+
|
|
280
|
+
# (3) Otherwise issue a conditional GET.
|
|
281
|
+
if csv_text is None:
|
|
282
|
+
try:
|
|
283
|
+
csv_text, new_meta = self._conditional_download(meta)
|
|
284
|
+
except requests.RequestException as exc:
|
|
285
|
+
# Network down but we have a usable disk cache — serve stale.
|
|
286
|
+
logger.warning(
|
|
287
|
+
"pyvortex: conditional refresh of instruments failed (%s). "
|
|
288
|
+
"Serving stale copy from disk cache (%s).",
|
|
289
|
+
exc, self._cache_path,
|
|
290
|
+
)
|
|
291
|
+
csv_text = self._read_disk_cache()
|
|
292
|
+
else:
|
|
293
|
+
if csv_text is None:
|
|
294
|
+
# 304 path — disk is still current.
|
|
295
|
+
csv_text = self._read_disk_cache()
|
|
296
|
+
self._touch_meta(meta, new_meta)
|
|
297
|
+
else:
|
|
298
|
+
# 200 path — write through.
|
|
299
|
+
self._write_disk_cache(csv_text)
|
|
300
|
+
self._write_meta(new_meta)
|
|
301
|
+
|
|
302
|
+
if csv_text is None:
|
|
303
|
+
# (4) No usable disk cache, or force=True.
|
|
304
|
+
csv_text, new_meta = self._unconditional_download()
|
|
305
|
+
if self._enable_disk_cache:
|
|
306
|
+
self._write_disk_cache(csv_text)
|
|
307
|
+
self._write_meta(new_meta)
|
|
308
|
+
|
|
309
|
+
self._parse_and_index(csv_text)
|
|
310
|
+
self._last_loaded_at = _now_ist()
|
|
311
|
+
self._loaded = True
|
|
312
|
+
return len(self._by_exchange_token)
|
|
313
|
+
|
|
314
|
+
def refresh(self) -> int:
|
|
315
|
+
"""Force a network refresh, bypassing both caches."""
|
|
316
|
+
return self.load(force=True)
|
|
317
|
+
|
|
318
|
+
def count(self) -> int:
|
|
319
|
+
with self._lock:
|
|
320
|
+
return len(self._by_exchange_token)
|
|
321
|
+
|
|
322
|
+
# ------------------------------------------------------------- Lookups
|
|
323
|
+
|
|
324
|
+
def get_by_ticker(self, ticker: str) -> Instrument:
|
|
325
|
+
self._ensure_loaded()
|
|
326
|
+
with self._lock:
|
|
327
|
+
try:
|
|
328
|
+
return self._by_ticker[ticker]
|
|
329
|
+
except KeyError:
|
|
330
|
+
raise InstrumentNotFound(f"ticker not found: {ticker!r}")
|
|
331
|
+
|
|
332
|
+
def get_by_exchange_token(self, exchange: str, token: int) -> Instrument:
|
|
333
|
+
self._ensure_loaded()
|
|
334
|
+
key = _exchange_token_key(exchange, token)
|
|
335
|
+
with self._lock:
|
|
336
|
+
try:
|
|
337
|
+
return self._by_exchange_token[key]
|
|
338
|
+
except KeyError:
|
|
339
|
+
raise InstrumentNotFound(
|
|
340
|
+
f"instrument not found for exchange={exchange!r}, token={token}"
|
|
341
|
+
)
|
|
342
|
+
|
|
343
|
+
def get_by_isin(self, isin: str) -> List[Instrument]:
|
|
344
|
+
self._ensure_loaded()
|
|
345
|
+
with self._lock:
|
|
346
|
+
insts = self._by_isin.get(isin)
|
|
347
|
+
if not insts:
|
|
348
|
+
raise InstrumentNotFound(f"ISIN not found: {isin!r}")
|
|
349
|
+
return list(insts)
|
|
350
|
+
|
|
351
|
+
def ticker_to_token(self, ticker: str) -> "tuple[str, int]":
|
|
352
|
+
inst = self.get_by_ticker(ticker)
|
|
353
|
+
return inst.exchange, inst.token
|
|
354
|
+
|
|
355
|
+
def token_to_ticker(self, exchange: str, token: int) -> str:
|
|
356
|
+
return self.get_by_exchange_token(exchange, token).ticker
|
|
357
|
+
|
|
358
|
+
def all_by_underlying(self, exchange: str, underlying: str) -> List[Instrument]:
|
|
359
|
+
"""Return all instruments on ``exchange`` whose ``symbol`` matches
|
|
360
|
+
``underlying`` (typically used to fetch an option chain)."""
|
|
361
|
+
self._ensure_loaded()
|
|
362
|
+
with self._lock:
|
|
363
|
+
return [
|
|
364
|
+
i for i in self._by_exchange_token.values()
|
|
365
|
+
if i.exchange == exchange and i.symbol == underlying
|
|
366
|
+
]
|
|
367
|
+
|
|
368
|
+
def filter(self, predicate: Callable[[Instrument], bool]) -> List[Instrument]:
|
|
369
|
+
self._ensure_loaded()
|
|
370
|
+
with self._lock:
|
|
371
|
+
return [i for i in self._by_exchange_token.values() if predicate(i)]
|
|
372
|
+
|
|
373
|
+
# --------------------------------------------------------- Internals
|
|
374
|
+
|
|
375
|
+
def _ensure_loaded(self) -> None:
|
|
376
|
+
if not self._loaded:
|
|
377
|
+
self.load()
|
|
378
|
+
|
|
379
|
+
# ---- cache I/O ----
|
|
380
|
+
|
|
381
|
+
def _has_disk_cache(self) -> bool:
|
|
382
|
+
return os.path.exists(self._cache_path)
|
|
383
|
+
|
|
384
|
+
def _read_disk_cache(self) -> Optional[str]:
|
|
385
|
+
try:
|
|
386
|
+
with open(self._cache_path, "r", encoding="utf-8") as fh:
|
|
387
|
+
return fh.read()
|
|
388
|
+
except OSError as exc:
|
|
389
|
+
logger.warning("Failed to read instruments cache %s: %s", self._cache_path, exc)
|
|
390
|
+
return None
|
|
391
|
+
|
|
392
|
+
def _write_disk_cache(self, csv_text: str) -> None:
|
|
393
|
+
try:
|
|
394
|
+
new_path = self._atomic_write_with_fallback(self._cache_path, csv_text)
|
|
395
|
+
except OSError:
|
|
396
|
+
# The fallback helper already emitted a one-time warning explaining
|
|
397
|
+
# what happened. Disk cache is now disabled; in-memory still works.
|
|
398
|
+
return
|
|
399
|
+
if new_path != self._cache_path:
|
|
400
|
+
# Disk cache moved — repoint everything so future loads use it.
|
|
401
|
+
self._cache_path = new_path
|
|
402
|
+
self._meta_path = new_path + ".meta.json"
|
|
403
|
+
|
|
404
|
+
def _read_meta(self) -> Optional[Dict[str, str]]:
|
|
405
|
+
if not os.path.exists(self._meta_path):
|
|
406
|
+
return None
|
|
407
|
+
try:
|
|
408
|
+
with open(self._meta_path, "r", encoding="utf-8") as fh:
|
|
409
|
+
data = json.load(fh)
|
|
410
|
+
if isinstance(data, dict):
|
|
411
|
+
return data
|
|
412
|
+
except (OSError, ValueError) as exc:
|
|
413
|
+
logger.warning("Failed to read instruments meta %s: %s", self._meta_path, exc)
|
|
414
|
+
return None
|
|
415
|
+
|
|
416
|
+
def _write_meta(self, meta: Dict[str, str]) -> None:
|
|
417
|
+
# Meta is small; on failure we silently skip it. Without meta the next
|
|
418
|
+
# process will lose conditional-GET capability but still works correctly.
|
|
419
|
+
try:
|
|
420
|
+
self._atomic_write_with_fallback(self._meta_path, json.dumps(meta))
|
|
421
|
+
except OSError as exc:
|
|
422
|
+
logger.debug("Skipping instruments meta write (%s): %s", self._meta_path, exc)
|
|
423
|
+
|
|
424
|
+
def _touch_meta(self, old: Dict[str, str], new: Dict[str, str]) -> None:
|
|
425
|
+
"""Merge ``new`` validation timestamp / headers into ``old`` and persist."""
|
|
426
|
+
merged = dict(old)
|
|
427
|
+
merged.update(new)
|
|
428
|
+
self._write_meta(merged)
|
|
429
|
+
|
|
430
|
+
def _atomic_write(self, path: str, text: str) -> None:
|
|
431
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
432
|
+
fd, tmp_path = tempfile.mkstemp(
|
|
433
|
+
prefix=".master-", suffix=".tmp", dir=os.path.dirname(path)
|
|
434
|
+
)
|
|
435
|
+
try:
|
|
436
|
+
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
|
437
|
+
fh.write(text)
|
|
438
|
+
os.replace(tmp_path, path)
|
|
439
|
+
except Exception:
|
|
440
|
+
try:
|
|
441
|
+
os.unlink(tmp_path)
|
|
442
|
+
except OSError:
|
|
443
|
+
pass
|
|
444
|
+
raise
|
|
445
|
+
|
|
446
|
+
def _atomic_write_with_fallback(self, path: str, text: str) -> str:
|
|
447
|
+
"""Atomic write with auto-fallback to ``/tmp`` on read-only filesystems.
|
|
448
|
+
|
|
449
|
+
Returns the path that was actually written. If even the fallback fails,
|
|
450
|
+
the disk cache is disabled for this manager and a one-time warning is
|
|
451
|
+
emitted; subsequent loads still work from memory.
|
|
452
|
+
"""
|
|
453
|
+
try:
|
|
454
|
+
self._atomic_write(path, text)
|
|
455
|
+
return path
|
|
456
|
+
except OSError as primary_exc:
|
|
457
|
+
# Try /tmp as a fallback — this covers EKS readOnlyRootFilesystem,
|
|
458
|
+
# Cloud Run, Lambda, and similar locked-down container setups.
|
|
459
|
+
fallback_dir = os.path.join(tempfile.gettempdir(), "pyvortex")
|
|
460
|
+
fallback_path = os.path.join(
|
|
461
|
+
fallback_dir, os.path.basename(path)
|
|
462
|
+
)
|
|
463
|
+
if fallback_path == path:
|
|
464
|
+
# Already trying /tmp — give up gracefully.
|
|
465
|
+
self._announce_disk_disabled(path, primary_exc)
|
|
466
|
+
raise
|
|
467
|
+
try:
|
|
468
|
+
self._atomic_write(fallback_path, text)
|
|
469
|
+
except OSError as fallback_exc:
|
|
470
|
+
self._announce_disk_disabled(path, fallback_exc)
|
|
471
|
+
raise
|
|
472
|
+
self._announce_fallback(path, fallback_path, primary_exc)
|
|
473
|
+
return fallback_path
|
|
474
|
+
|
|
475
|
+
def _announce_fallback(
|
|
476
|
+
self, original: str, fallback: str, exc: OSError
|
|
477
|
+
) -> None:
|
|
478
|
+
if self._fallback_announced:
|
|
479
|
+
return
|
|
480
|
+
self._fallback_announced = True
|
|
481
|
+
logger.warning(
|
|
482
|
+
"pyvortex: cannot write instruments cache to %s (%s). "
|
|
483
|
+
"Falling back to %s for the rest of this process. "
|
|
484
|
+
"To silence this warning in containers, set "
|
|
485
|
+
"PYVORTEX_CACHE_DIR=/tmp/pyvortex (or any writable path).",
|
|
486
|
+
original, exc, fallback,
|
|
487
|
+
)
|
|
488
|
+
|
|
489
|
+
def _announce_disk_disabled(self, path: str, exc: OSError) -> None:
|
|
490
|
+
if self._disk_disabled:
|
|
491
|
+
return
|
|
492
|
+
self._disk_disabled = True
|
|
493
|
+
logger.warning(
|
|
494
|
+
"pyvortex: disk cache disabled (could not write to %s or its fallback: %s). "
|
|
495
|
+
"Instruments will be loaded into memory only — expect a fresh download "
|
|
496
|
+
"from %s every time this process starts. To fix, mount a writable "
|
|
497
|
+
"volume and set PYVORTEX_CACHE_DIR to point at it.",
|
|
498
|
+
path, exc, self._master_url,
|
|
499
|
+
)
|
|
500
|
+
|
|
501
|
+
# ---- freshness ----
|
|
502
|
+
|
|
503
|
+
def _within_revalidation_cooldown(self, meta: Dict[str, str]) -> bool:
|
|
504
|
+
if self._revalidate_after_seconds <= 0:
|
|
505
|
+
return False
|
|
506
|
+
fetched_at = meta.get("fetched_at")
|
|
507
|
+
if not fetched_at:
|
|
508
|
+
return False
|
|
509
|
+
try:
|
|
510
|
+
ts = datetime.datetime.fromisoformat(fetched_at)
|
|
511
|
+
except ValueError:
|
|
512
|
+
return False
|
|
513
|
+
if ts.tzinfo is None:
|
|
514
|
+
ts = ts.replace(tzinfo=datetime.timezone.utc)
|
|
515
|
+
age = datetime.datetime.now(datetime.timezone.utc) - ts
|
|
516
|
+
return age.total_seconds() < self._revalidate_after_seconds
|
|
517
|
+
|
|
518
|
+
# ---- network ----
|
|
519
|
+
|
|
520
|
+
def _conditional_download(
|
|
521
|
+
self, meta: Dict[str, str]
|
|
522
|
+
) -> "tuple[Optional[str], Dict[str, str]]":
|
|
523
|
+
"""Issue a conditional GET. Returns ``(csv_text_or_None, new_meta)``.
|
|
524
|
+
|
|
525
|
+
``csv_text`` is ``None`` iff the server returned ``304 Not Modified``
|
|
526
|
+
(in which case ``new_meta`` only carries an updated ``fetched_at``).
|
|
527
|
+
"""
|
|
528
|
+
headers = {"Accept-Encoding": "gzip"}
|
|
529
|
+
if meta.get("etag"):
|
|
530
|
+
headers["If-None-Match"] = meta["etag"]
|
|
531
|
+
if meta.get("last_modified"):
|
|
532
|
+
headers["If-Modified-Since"] = meta["last_modified"]
|
|
533
|
+
|
|
534
|
+
logger.debug("Conditional GET master.csv (etag=%s)", meta.get("etag"))
|
|
535
|
+
resp = requests.get(self._master_url, headers=headers, timeout=self._request_timeout)
|
|
536
|
+
|
|
537
|
+
if resp.status_code == 304:
|
|
538
|
+
logger.debug("master.csv unchanged (304); reusing disk cache")
|
|
539
|
+
return None, {"fetched_at": _utc_now_iso()}
|
|
540
|
+
|
|
541
|
+
resp.raise_for_status()
|
|
542
|
+
return _decode_body(resp.content), _meta_from_response(resp)
|
|
543
|
+
|
|
544
|
+
def _unconditional_download(self) -> "tuple[str, Dict[str, str]]":
|
|
545
|
+
logger.debug("Downloading master.csv from %s", self._master_url)
|
|
546
|
+
resp = requests.get(
|
|
547
|
+
self._master_url,
|
|
548
|
+
headers={"Accept-Encoding": "gzip"},
|
|
549
|
+
timeout=self._request_timeout,
|
|
550
|
+
)
|
|
551
|
+
resp.raise_for_status()
|
|
552
|
+
return _decode_body(resp.content), _meta_from_response(resp)
|
|
553
|
+
|
|
554
|
+
def _parse_and_index(self, csv_text: str) -> None:
|
|
555
|
+
reader = csv.DictReader(io.StringIO(csv_text))
|
|
556
|
+
by_ticker: Dict[str, Instrument] = {}
|
|
557
|
+
by_exchange_token: Dict[str, Instrument] = {}
|
|
558
|
+
by_isin: Dict[str, List[Instrument]] = {}
|
|
559
|
+
|
|
560
|
+
for row in reader:
|
|
561
|
+
inst = _row_to_instrument(row)
|
|
562
|
+
if inst.ticker:
|
|
563
|
+
by_ticker[inst.ticker] = inst
|
|
564
|
+
if inst.exchange and inst.token:
|
|
565
|
+
by_exchange_token[inst.exchange_token_key] = inst
|
|
566
|
+
if inst.isin:
|
|
567
|
+
by_isin.setdefault(inst.isin, []).append(inst)
|
|
568
|
+
|
|
569
|
+
# Atomic swap
|
|
570
|
+
self._by_ticker = by_ticker
|
|
571
|
+
self._by_exchange_token = by_exchange_token
|
|
572
|
+
self._by_isin = by_isin
|
|
573
|
+
logger.debug("Indexed %d instruments", len(by_exchange_token))
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def _utc_now_iso() -> str:
|
|
577
|
+
return datetime.datetime.now(datetime.timezone.utc).isoformat()
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
def _decode_body(content: bytes) -> str:
|
|
581
|
+
"""Decode a response body that may be gzipped at the application layer.
|
|
582
|
+
|
|
583
|
+
``requests`` already strips ``Content-Encoding: gzip`` transparently, but
|
|
584
|
+
the static asset is sometimes served as ``application/octet-stream`` with
|
|
585
|
+
the gzip magic bytes intact — handle both.
|
|
586
|
+
"""
|
|
587
|
+
if content[:2] == b"\x1f\x8b":
|
|
588
|
+
content = gzip.decompress(content)
|
|
589
|
+
return content.decode("utf-8")
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
def _meta_from_response(resp: requests.Response) -> Dict[str, str]:
|
|
593
|
+
meta: Dict[str, str] = {"fetched_at": _utc_now_iso()}
|
|
594
|
+
etag = resp.headers.get("ETag")
|
|
595
|
+
if etag:
|
|
596
|
+
meta["etag"] = etag
|
|
597
|
+
last_modified = resp.headers.get("Last-Modified")
|
|
598
|
+
if last_modified:
|
|
599
|
+
meta["last_modified"] = last_modified
|
|
600
|
+
return meta
|
|
601
|
+
|
|
602
|
+
|
|
603
|
+
def _safe_int(v: str) -> int:
|
|
604
|
+
try:
|
|
605
|
+
return int(v)
|
|
606
|
+
except (TypeError, ValueError):
|
|
607
|
+
return 0
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def _safe_float(v: str) -> float:
|
|
611
|
+
try:
|
|
612
|
+
return float(v)
|
|
613
|
+
except (TypeError, ValueError):
|
|
614
|
+
return 0.0
|
|
615
|
+
|
|
616
|
+
|
|
617
|
+
def _row_to_instrument(row: Dict[str, str]) -> Instrument:
|
|
618
|
+
last_trading_date: Optional[datetime.datetime] = None
|
|
619
|
+
raw_ltd = row.get("last_trading_date") or ""
|
|
620
|
+
if raw_ltd:
|
|
621
|
+
try:
|
|
622
|
+
# Master file stores epoch seconds shifted by IST offset
|
|
623
|
+
# (see Go parser: `time.Unix(int64(t)-19800, 0)`).
|
|
624
|
+
secs = float(raw_ltd) - 19800
|
|
625
|
+
last_trading_date = datetime.datetime.fromtimestamp(
|
|
626
|
+
secs, tz=datetime.timezone(_IST_OFFSET)
|
|
627
|
+
)
|
|
628
|
+
except (TypeError, ValueError):
|
|
629
|
+
last_trading_date = None
|
|
630
|
+
|
|
631
|
+
known = {
|
|
632
|
+
"ticker", "symbol", "token", "exchange", "instrument_name", "series",
|
|
633
|
+
"expiry_date", "isin_code", "option_type", "strike_price", "tick",
|
|
634
|
+
"lot_size", "eligibility", "security_desc", "last_trading_date",
|
|
635
|
+
"asm_gsm_stage",
|
|
636
|
+
}
|
|
637
|
+
extras = {k: v for k, v in row.items() if k not in known and v}
|
|
638
|
+
|
|
639
|
+
return Instrument(
|
|
640
|
+
ticker=row.get("ticker", "") or "",
|
|
641
|
+
token=_safe_int(row.get("token", "")),
|
|
642
|
+
exchange=row.get("exchange", "") or "",
|
|
643
|
+
instrument_name=row.get("instrument_name", "") or "",
|
|
644
|
+
symbol=row.get("symbol", "") or "",
|
|
645
|
+
series=row.get("series", "") or "",
|
|
646
|
+
expiry_date=row.get("expiry_date", "") or "",
|
|
647
|
+
isin=row.get("isin_code", "") or "",
|
|
648
|
+
option_type=row.get("option_type", "") or "",
|
|
649
|
+
strike_price=_safe_float(row.get("strike_price", "")),
|
|
650
|
+
tick=_safe_float(row.get("tick", "")),
|
|
651
|
+
lot_size=_safe_int(row.get("lot_size", "")),
|
|
652
|
+
eligibility=_safe_int(row.get("eligibility", "")) > 0,
|
|
653
|
+
name=row.get("security_desc", "") or "",
|
|
654
|
+
last_trading_date=last_trading_date,
|
|
655
|
+
asm_gsm_stage=row.get("asm_gsm_stage", "") or "",
|
|
656
|
+
extras=extras,
|
|
657
|
+
)
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "2.1.8b1"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|