vortex-api 2.1.8b1__tar.gz → 2.1.8b2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {vortex_api-2.1.8b1/vortex_api.egg-info → vortex_api-2.1.8b2}/PKG-INFO +1 -1
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b2}/pyproject.toml +1 -1
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b2}/vortex_api/__init__.py +2 -1
- vortex_api-2.1.8b2/vortex_api/__version__.py +1 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b2}/vortex_api/api.py +16 -0
- vortex_api-2.1.8b2/vortex_api/instruments.py +478 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b2/vortex_api.egg-info}/PKG-INFO +1 -1
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b2}/vortex_api.egg-info/SOURCES.txt +1 -0
- vortex_api-2.1.8b1/vortex_api/__version__.py +0 -1
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b2}/LICENSE +0 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b2}/README.md +0 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b2}/setup.cfg +0 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b2}/vortex_api/backtest/__init__.py +0 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b2}/vortex_api/backtest/_backtestingpy.py +0 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b2}/vortex_api/backtest/_backtrader.py +0 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b2}/vortex_api/backtest/_common.py +0 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b2}/vortex_api/backtest/_vectorbt.py +0 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b2}/vortex_api/vortex_feed.py +0 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b2}/vortex_api.egg-info/dependency_links.txt +0 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b2}/vortex_api.egg-info/requires.txt +0 -0
- {vortex_api-2.1.8b1 → vortex_api-2.1.8b2}/vortex_api.egg-info/top_level.txt +0 -0
|
@@ -35,4 +35,5 @@ Getting started
|
|
|
35
35
|
from __future__ import unicode_literals, absolute_import
|
|
36
36
|
from vortex_api.api import VortexAPI,Constants
|
|
37
37
|
from vortex_api.vortex_feed import VortexFeed
|
|
38
|
-
|
|
38
|
+
from vortex_api.instruments import InstrumentManager, Instrument, InstrumentNotFound
|
|
39
|
+
__all__ = [VortexAPI,Constants,VortexFeed,InstrumentManager,Instrument,InstrumentNotFound]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "2.1.8b2"
|
|
@@ -205,6 +205,22 @@ class VortexAPI:
|
|
|
205
205
|
self.enable_logging = enable_logging
|
|
206
206
|
if self.enable_logging:
|
|
207
207
|
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
|
208
|
+
|
|
209
|
+
self._instruments = None # lazily constructed InstrumentManager
|
|
210
|
+
|
|
211
|
+
@property
|
|
212
|
+
def instruments(self):
|
|
213
|
+
"""Lazy ``InstrumentManager`` for this client.
|
|
214
|
+
|
|
215
|
+
The first access constructs the manager but does **not** download
|
|
216
|
+
``master.csv``; the download happens on the first lookup (or on an
|
|
217
|
+
explicit ``.load()`` call) and is cached in memory and on disk so
|
|
218
|
+
subsequent script runs hit the network at most once per IST trading day.
|
|
219
|
+
"""
|
|
220
|
+
if self._instruments is None:
|
|
221
|
+
from vortex_api.instruments import InstrumentManager
|
|
222
|
+
self._instruments = InstrumentManager()
|
|
223
|
+
return self._instruments
|
|
208
224
|
|
|
209
225
|
def _make_api_request(self, method: str, endpoint: str, data: dict = None, params=None) -> dict:
|
|
210
226
|
"""
|
|
@@ -0,0 +1,478 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Instrument manager for the Vortex SDK.
|
|
3
|
+
|
|
4
|
+
Downloads the Vortex instruments master (``master.csv``) and exposes fast
|
|
5
|
+
lookups by ticker, ``(exchange, token)``, or ISIN. The master file is cached
|
|
6
|
+
both in memory and on disk so that ``master.csv`` is fetched at most once per
|
|
7
|
+
trading day, even across process restarts.
|
|
8
|
+
|
|
9
|
+
Typical use::
|
|
10
|
+
|
|
11
|
+
from vortex_api import VortexAPI
|
|
12
|
+
|
|
13
|
+
client = VortexAPI(...)
|
|
14
|
+
|
|
15
|
+
inst = client.instruments.get_by_ticker("NSE:RELIANCE")
|
|
16
|
+
print(inst.token, inst.exchange, inst.lot_size)
|
|
17
|
+
|
|
18
|
+
# Reverse lookup
|
|
19
|
+
inst = client.instruments.get_by_exchange_token("NSE_EQ", 2885)
|
|
20
|
+
|
|
21
|
+
# All option contracts under NIFTY
|
|
22
|
+
chain = client.instruments.all_by_underlying("NSE_FO", "NIFTY")
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import csv
|
|
28
|
+
import datetime
|
|
29
|
+
import gzip
|
|
30
|
+
import io
|
|
31
|
+
import json
|
|
32
|
+
import logging
|
|
33
|
+
import os
|
|
34
|
+
import tempfile
|
|
35
|
+
import threading
|
|
36
|
+
from dataclasses import dataclass, field
|
|
37
|
+
from typing import Callable, Dict, List, Optional
|
|
38
|
+
|
|
39
|
+
import requests
|
|
40
|
+
|
|
41
|
+
DEFAULT_MASTER_URL = "https://static.rupeezy.in/master.csv"
|
|
42
|
+
DEFAULT_REVALIDATE_AFTER_SECONDS = 15 * 60 # how long to trust disk before conditional GET
|
|
43
|
+
_IST_OFFSET = datetime.timedelta(hours=5, minutes=30)
|
|
44
|
+
|
|
45
|
+
logger = logging.getLogger(__name__)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class InstrumentNotFound(Exception):
|
|
49
|
+
"""Raised when a lookup cannot find the requested instrument."""
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass
|
|
53
|
+
class Instrument:
|
|
54
|
+
"""A single tradable instrument row from ``master.csv``.
|
|
55
|
+
|
|
56
|
+
Fields mirror the columns produced by the Vortex master file. Missing or
|
|
57
|
+
empty columns produce default values rather than raising.
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
ticker: str = "" # e.g. "NSE:RELIANCE"
|
|
61
|
+
token: int = 0
|
|
62
|
+
exchange: str = "" # e.g. "NSE_EQ", "NSE_FO"
|
|
63
|
+
instrument_name: str = ""
|
|
64
|
+
symbol: str = ""
|
|
65
|
+
series: str = ""
|
|
66
|
+
expiry_date: str = ""
|
|
67
|
+
isin: str = ""
|
|
68
|
+
option_type: str = ""
|
|
69
|
+
strike_price: float = 0.0
|
|
70
|
+
tick: float = 0.0
|
|
71
|
+
lot_size: int = 0
|
|
72
|
+
eligibility: bool = False
|
|
73
|
+
name: str = ""
|
|
74
|
+
last_trading_date: Optional[datetime.datetime] = None
|
|
75
|
+
asm_gsm_stage: str = ""
|
|
76
|
+
extras: Dict[str, str] = field(default_factory=dict)
|
|
77
|
+
|
|
78
|
+
@property
|
|
79
|
+
def exchange_token_key(self) -> str:
|
|
80
|
+
"""Composite key used internally for ``(exchange, token)`` lookups.
|
|
81
|
+
|
|
82
|
+
Matches the Go package's ``TokenIdentifier`` format: first three
|
|
83
|
+
characters of the exchange (NSE/BSE/MCX) joined with the token.
|
|
84
|
+
"""
|
|
85
|
+
return _exchange_token_key(self.exchange, self.token)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _exchange_token_key(exchange: str, token: int) -> str:
|
|
89
|
+
short = (exchange or "")[:3]
|
|
90
|
+
return f"{short}:{int(token)}"
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _now_ist() -> datetime.datetime:
|
|
94
|
+
return datetime.datetime.now(datetime.timezone.utc).astimezone(
|
|
95
|
+
datetime.timezone(_IST_OFFSET)
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _default_cache_path() -> str:
|
|
100
|
+
base = os.environ.get("XDG_CACHE_HOME") or os.path.join(
|
|
101
|
+
os.path.expanduser("~"), ".cache"
|
|
102
|
+
)
|
|
103
|
+
return os.path.join(base, "pyvortex", "master.csv")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class InstrumentManager:
|
|
107
|
+
"""Thread-safe, lazily-loaded instruments cache.
|
|
108
|
+
|
|
109
|
+
Args:
|
|
110
|
+
master_url: Override the master CSV URL (mostly for testing).
|
|
111
|
+
cache_path: Where to persist the downloaded master file. Defaults to
|
|
112
|
+
``~/.cache/pyvortex/master.csv`` (or ``$XDG_CACHE_HOME``).
|
|
113
|
+
enable_disk_cache: If ``False``, every load re-downloads. Default ``True``.
|
|
114
|
+
request_timeout: HTTP request timeout in seconds.
|
|
115
|
+
revalidate_after_seconds: How long to trust the disk cache without
|
|
116
|
+
re-checking the server. After this interval, the next ``load()``
|
|
117
|
+
issues a conditional GET using ``If-None-Match`` / ``If-Modified-Since``.
|
|
118
|
+
A ``304`` response keeps the disk cache; a ``200`` swaps it in.
|
|
119
|
+
Default 900 (15 minutes). Set to 0 to revalidate on every load.
|
|
120
|
+
"""
|
|
121
|
+
|
|
122
|
+
def __init__(
|
|
123
|
+
self,
|
|
124
|
+
master_url: str = DEFAULT_MASTER_URL,
|
|
125
|
+
cache_path: Optional[str] = None,
|
|
126
|
+
enable_disk_cache: bool = True,
|
|
127
|
+
request_timeout: float = 30.0,
|
|
128
|
+
revalidate_after_seconds: int = DEFAULT_REVALIDATE_AFTER_SECONDS,
|
|
129
|
+
) -> None:
|
|
130
|
+
self._master_url = master_url
|
|
131
|
+
self._cache_path = cache_path or _default_cache_path()
|
|
132
|
+
self._meta_path = self._cache_path + ".meta.json"
|
|
133
|
+
self._enable_disk_cache = enable_disk_cache
|
|
134
|
+
self._request_timeout = request_timeout
|
|
135
|
+
self._revalidate_after_seconds = max(0, int(revalidate_after_seconds))
|
|
136
|
+
|
|
137
|
+
self._lock = threading.RLock()
|
|
138
|
+
self._by_ticker: Dict[str, Instrument] = {}
|
|
139
|
+
self._by_exchange_token: Dict[str, Instrument] = {}
|
|
140
|
+
self._by_isin: Dict[str, List[Instrument]] = {}
|
|
141
|
+
self._last_loaded_at: Optional[datetime.datetime] = None
|
|
142
|
+
self._loaded = False
|
|
143
|
+
|
|
144
|
+
# ------------------------------------------------------------------ Public
|
|
145
|
+
|
|
146
|
+
def load(self, force: bool = False) -> int:
|
|
147
|
+
"""Ensure instruments are loaded. Returns the instrument count.
|
|
148
|
+
|
|
149
|
+
Freshness strategy (in order, cheapest first):
|
|
150
|
+
|
|
151
|
+
1. In-memory cache: already populated this process → no I/O at all.
|
|
152
|
+
2. Disk cache + revalidation cooldown: if we validated less than
|
|
153
|
+
``revalidate_after_seconds`` ago, trust the disk cache without
|
|
154
|
+
touching the network.
|
|
155
|
+
3. Conditional GET: send ``If-None-Match`` / ``If-Modified-Since`` from
|
|
156
|
+
the cached metadata. A ``304`` response confirms the cache is
|
|
157
|
+
current (a few hundred bytes on the wire). A ``200`` brings the new
|
|
158
|
+
master and we re-index.
|
|
159
|
+
4. Unconditional GET: only when there is no usable disk cache.
|
|
160
|
+
|
|
161
|
+
Set ``force=True`` to bypass every cache and re-download.
|
|
162
|
+
"""
|
|
163
|
+
with self._lock:
|
|
164
|
+
if not force and self._loaded:
|
|
165
|
+
return len(self._by_exchange_token)
|
|
166
|
+
|
|
167
|
+
csv_text: Optional[str] = None
|
|
168
|
+
meta = self._read_meta() if self._enable_disk_cache else None
|
|
169
|
+
|
|
170
|
+
if not force and meta is not None and self._has_disk_cache():
|
|
171
|
+
# (2) Recent revalidation → skip the network entirely.
|
|
172
|
+
if self._within_revalidation_cooldown(meta):
|
|
173
|
+
csv_text = self._read_disk_cache()
|
|
174
|
+
if csv_text is not None:
|
|
175
|
+
logger.debug("Using disk cache (within cooldown): %s", self._cache_path)
|
|
176
|
+
|
|
177
|
+
# (3) Otherwise issue a conditional GET.
|
|
178
|
+
if csv_text is None:
|
|
179
|
+
csv_text, new_meta = self._conditional_download(meta)
|
|
180
|
+
if csv_text is None:
|
|
181
|
+
# 304 path — disk is still current.
|
|
182
|
+
csv_text = self._read_disk_cache()
|
|
183
|
+
self._touch_meta(meta, new_meta)
|
|
184
|
+
else:
|
|
185
|
+
# 200 path — write through.
|
|
186
|
+
self._write_disk_cache(csv_text)
|
|
187
|
+
self._write_meta(new_meta)
|
|
188
|
+
|
|
189
|
+
if csv_text is None:
|
|
190
|
+
# (4) No usable disk cache, or force=True.
|
|
191
|
+
csv_text, new_meta = self._unconditional_download()
|
|
192
|
+
if self._enable_disk_cache:
|
|
193
|
+
self._write_disk_cache(csv_text)
|
|
194
|
+
self._write_meta(new_meta)
|
|
195
|
+
|
|
196
|
+
self._parse_and_index(csv_text)
|
|
197
|
+
self._last_loaded_at = _now_ist()
|
|
198
|
+
self._loaded = True
|
|
199
|
+
return len(self._by_exchange_token)
|
|
200
|
+
|
|
201
|
+
def refresh(self) -> int:
|
|
202
|
+
"""Force a network refresh, bypassing both caches."""
|
|
203
|
+
return self.load(force=True)
|
|
204
|
+
|
|
205
|
+
def count(self) -> int:
|
|
206
|
+
with self._lock:
|
|
207
|
+
return len(self._by_exchange_token)
|
|
208
|
+
|
|
209
|
+
# ------------------------------------------------------------- Lookups
|
|
210
|
+
|
|
211
|
+
def get_by_ticker(self, ticker: str) -> Instrument:
|
|
212
|
+
self._ensure_loaded()
|
|
213
|
+
with self._lock:
|
|
214
|
+
try:
|
|
215
|
+
return self._by_ticker[ticker]
|
|
216
|
+
except KeyError:
|
|
217
|
+
raise InstrumentNotFound(f"ticker not found: {ticker!r}")
|
|
218
|
+
|
|
219
|
+
def get_by_exchange_token(self, exchange: str, token: int) -> Instrument:
|
|
220
|
+
self._ensure_loaded()
|
|
221
|
+
key = _exchange_token_key(exchange, token)
|
|
222
|
+
with self._lock:
|
|
223
|
+
try:
|
|
224
|
+
return self._by_exchange_token[key]
|
|
225
|
+
except KeyError:
|
|
226
|
+
raise InstrumentNotFound(
|
|
227
|
+
f"instrument not found for exchange={exchange!r}, token={token}"
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
def get_by_isin(self, isin: str) -> List[Instrument]:
|
|
231
|
+
self._ensure_loaded()
|
|
232
|
+
with self._lock:
|
|
233
|
+
insts = self._by_isin.get(isin)
|
|
234
|
+
if not insts:
|
|
235
|
+
raise InstrumentNotFound(f"ISIN not found: {isin!r}")
|
|
236
|
+
return list(insts)
|
|
237
|
+
|
|
238
|
+
def ticker_to_token(self, ticker: str) -> "tuple[str, int]":
|
|
239
|
+
inst = self.get_by_ticker(ticker)
|
|
240
|
+
return inst.exchange, inst.token
|
|
241
|
+
|
|
242
|
+
def token_to_ticker(self, exchange: str, token: int) -> str:
|
|
243
|
+
return self.get_by_exchange_token(exchange, token).ticker
|
|
244
|
+
|
|
245
|
+
def all_by_underlying(self, exchange: str, underlying: str) -> List[Instrument]:
|
|
246
|
+
"""Return all instruments on ``exchange`` whose ``symbol`` matches
|
|
247
|
+
``underlying`` (typically used to fetch an option chain)."""
|
|
248
|
+
self._ensure_loaded()
|
|
249
|
+
with self._lock:
|
|
250
|
+
return [
|
|
251
|
+
i for i in self._by_exchange_token.values()
|
|
252
|
+
if i.exchange == exchange and i.symbol == underlying
|
|
253
|
+
]
|
|
254
|
+
|
|
255
|
+
def filter(self, predicate: Callable[[Instrument], bool]) -> List[Instrument]:
|
|
256
|
+
self._ensure_loaded()
|
|
257
|
+
with self._lock:
|
|
258
|
+
return [i for i in self._by_exchange_token.values() if predicate(i)]
|
|
259
|
+
|
|
260
|
+
# --------------------------------------------------------- Internals
|
|
261
|
+
|
|
262
|
+
def _ensure_loaded(self) -> None:
|
|
263
|
+
if not self._loaded:
|
|
264
|
+
self.load()
|
|
265
|
+
|
|
266
|
+
# ---- cache I/O ----
|
|
267
|
+
|
|
268
|
+
def _has_disk_cache(self) -> bool:
|
|
269
|
+
return os.path.exists(self._cache_path)
|
|
270
|
+
|
|
271
|
+
def _read_disk_cache(self) -> Optional[str]:
|
|
272
|
+
try:
|
|
273
|
+
with open(self._cache_path, "r", encoding="utf-8") as fh:
|
|
274
|
+
return fh.read()
|
|
275
|
+
except OSError as exc:
|
|
276
|
+
logger.warning("Failed to read instruments cache %s: %s", self._cache_path, exc)
|
|
277
|
+
return None
|
|
278
|
+
|
|
279
|
+
def _write_disk_cache(self, csv_text: str) -> None:
|
|
280
|
+
self._atomic_write(self._cache_path, csv_text)
|
|
281
|
+
|
|
282
|
+
def _read_meta(self) -> Optional[Dict[str, str]]:
|
|
283
|
+
if not os.path.exists(self._meta_path):
|
|
284
|
+
return None
|
|
285
|
+
try:
|
|
286
|
+
with open(self._meta_path, "r", encoding="utf-8") as fh:
|
|
287
|
+
data = json.load(fh)
|
|
288
|
+
if isinstance(data, dict):
|
|
289
|
+
return data
|
|
290
|
+
except (OSError, ValueError) as exc:
|
|
291
|
+
logger.warning("Failed to read instruments meta %s: %s", self._meta_path, exc)
|
|
292
|
+
return None
|
|
293
|
+
|
|
294
|
+
def _write_meta(self, meta: Dict[str, str]) -> None:
|
|
295
|
+
try:
|
|
296
|
+
self._atomic_write(self._meta_path, json.dumps(meta))
|
|
297
|
+
except OSError as exc:
|
|
298
|
+
logger.warning("Failed to write instruments meta %s: %s", self._meta_path, exc)
|
|
299
|
+
|
|
300
|
+
def _touch_meta(self, old: Dict[str, str], new: Dict[str, str]) -> None:
|
|
301
|
+
"""Merge ``new`` validation timestamp / headers into ``old`` and persist."""
|
|
302
|
+
merged = dict(old)
|
|
303
|
+
merged.update(new)
|
|
304
|
+
self._write_meta(merged)
|
|
305
|
+
|
|
306
|
+
def _atomic_write(self, path: str, text: str) -> None:
|
|
307
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
308
|
+
fd, tmp_path = tempfile.mkstemp(
|
|
309
|
+
prefix=".master-", suffix=".tmp", dir=os.path.dirname(path)
|
|
310
|
+
)
|
|
311
|
+
try:
|
|
312
|
+
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
|
313
|
+
fh.write(text)
|
|
314
|
+
os.replace(tmp_path, path)
|
|
315
|
+
except Exception:
|
|
316
|
+
try:
|
|
317
|
+
os.unlink(tmp_path)
|
|
318
|
+
except OSError:
|
|
319
|
+
pass
|
|
320
|
+
raise
|
|
321
|
+
|
|
322
|
+
# ---- freshness ----
|
|
323
|
+
|
|
324
|
+
def _within_revalidation_cooldown(self, meta: Dict[str, str]) -> bool:
|
|
325
|
+
if self._revalidate_after_seconds <= 0:
|
|
326
|
+
return False
|
|
327
|
+
fetched_at = meta.get("fetched_at")
|
|
328
|
+
if not fetched_at:
|
|
329
|
+
return False
|
|
330
|
+
try:
|
|
331
|
+
ts = datetime.datetime.fromisoformat(fetched_at)
|
|
332
|
+
except ValueError:
|
|
333
|
+
return False
|
|
334
|
+
if ts.tzinfo is None:
|
|
335
|
+
ts = ts.replace(tzinfo=datetime.timezone.utc)
|
|
336
|
+
age = datetime.datetime.now(datetime.timezone.utc) - ts
|
|
337
|
+
return age.total_seconds() < self._revalidate_after_seconds
|
|
338
|
+
|
|
339
|
+
# ---- network ----
|
|
340
|
+
|
|
341
|
+
def _conditional_download(
|
|
342
|
+
self, meta: Dict[str, str]
|
|
343
|
+
) -> "tuple[Optional[str], Dict[str, str]]":
|
|
344
|
+
"""Issue a conditional GET. Returns ``(csv_text_or_None, new_meta)``.
|
|
345
|
+
|
|
346
|
+
``csv_text`` is ``None`` iff the server returned ``304 Not Modified``
|
|
347
|
+
(in which case ``new_meta`` only carries an updated ``fetched_at``).
|
|
348
|
+
"""
|
|
349
|
+
headers = {"Accept-Encoding": "gzip"}
|
|
350
|
+
if meta.get("etag"):
|
|
351
|
+
headers["If-None-Match"] = meta["etag"]
|
|
352
|
+
if meta.get("last_modified"):
|
|
353
|
+
headers["If-Modified-Since"] = meta["last_modified"]
|
|
354
|
+
|
|
355
|
+
logger.debug("Conditional GET master.csv (etag=%s)", meta.get("etag"))
|
|
356
|
+
resp = requests.get(self._master_url, headers=headers, timeout=self._request_timeout)
|
|
357
|
+
|
|
358
|
+
if resp.status_code == 304:
|
|
359
|
+
logger.debug("master.csv unchanged (304); reusing disk cache")
|
|
360
|
+
return None, {"fetched_at": _utc_now_iso()}
|
|
361
|
+
|
|
362
|
+
resp.raise_for_status()
|
|
363
|
+
return _decode_body(resp.content), _meta_from_response(resp)
|
|
364
|
+
|
|
365
|
+
def _unconditional_download(self) -> "tuple[str, Dict[str, str]]":
|
|
366
|
+
logger.debug("Downloading master.csv from %s", self._master_url)
|
|
367
|
+
resp = requests.get(
|
|
368
|
+
self._master_url,
|
|
369
|
+
headers={"Accept-Encoding": "gzip"},
|
|
370
|
+
timeout=self._request_timeout,
|
|
371
|
+
)
|
|
372
|
+
resp.raise_for_status()
|
|
373
|
+
return _decode_body(resp.content), _meta_from_response(resp)
|
|
374
|
+
|
|
375
|
+
def _parse_and_index(self, csv_text: str) -> None:
|
|
376
|
+
reader = csv.DictReader(io.StringIO(csv_text))
|
|
377
|
+
by_ticker: Dict[str, Instrument] = {}
|
|
378
|
+
by_exchange_token: Dict[str, Instrument] = {}
|
|
379
|
+
by_isin: Dict[str, List[Instrument]] = {}
|
|
380
|
+
|
|
381
|
+
for row in reader:
|
|
382
|
+
inst = _row_to_instrument(row)
|
|
383
|
+
if inst.ticker:
|
|
384
|
+
by_ticker[inst.ticker] = inst
|
|
385
|
+
if inst.exchange and inst.token:
|
|
386
|
+
by_exchange_token[inst.exchange_token_key] = inst
|
|
387
|
+
if inst.isin:
|
|
388
|
+
by_isin.setdefault(inst.isin, []).append(inst)
|
|
389
|
+
|
|
390
|
+
# Atomic swap
|
|
391
|
+
self._by_ticker = by_ticker
|
|
392
|
+
self._by_exchange_token = by_exchange_token
|
|
393
|
+
self._by_isin = by_isin
|
|
394
|
+
logger.debug("Indexed %d instruments", len(by_exchange_token))
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def _utc_now_iso() -> str:
|
|
398
|
+
return datetime.datetime.now(datetime.timezone.utc).isoformat()
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def _decode_body(content: bytes) -> str:
|
|
402
|
+
"""Decode a response body that may be gzipped at the application layer.
|
|
403
|
+
|
|
404
|
+
``requests`` already strips ``Content-Encoding: gzip`` transparently, but
|
|
405
|
+
the static asset is sometimes served as ``application/octet-stream`` with
|
|
406
|
+
the gzip magic bytes intact — handle both.
|
|
407
|
+
"""
|
|
408
|
+
if content[:2] == b"\x1f\x8b":
|
|
409
|
+
content = gzip.decompress(content)
|
|
410
|
+
return content.decode("utf-8")
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def _meta_from_response(resp: requests.Response) -> Dict[str, str]:
|
|
414
|
+
meta: Dict[str, str] = {"fetched_at": _utc_now_iso()}
|
|
415
|
+
etag = resp.headers.get("ETag")
|
|
416
|
+
if etag:
|
|
417
|
+
meta["etag"] = etag
|
|
418
|
+
last_modified = resp.headers.get("Last-Modified")
|
|
419
|
+
if last_modified:
|
|
420
|
+
meta["last_modified"] = last_modified
|
|
421
|
+
return meta
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def _safe_int(v: str) -> int:
|
|
425
|
+
try:
|
|
426
|
+
return int(v)
|
|
427
|
+
except (TypeError, ValueError):
|
|
428
|
+
return 0
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
def _safe_float(v: str) -> float:
|
|
432
|
+
try:
|
|
433
|
+
return float(v)
|
|
434
|
+
except (TypeError, ValueError):
|
|
435
|
+
return 0.0
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
def _row_to_instrument(row: Dict[str, str]) -> Instrument:
|
|
439
|
+
last_trading_date: Optional[datetime.datetime] = None
|
|
440
|
+
raw_ltd = row.get("last_trading_date") or ""
|
|
441
|
+
if raw_ltd:
|
|
442
|
+
try:
|
|
443
|
+
# Master file stores epoch seconds shifted by IST offset
|
|
444
|
+
# (see Go parser: `time.Unix(int64(t)-19800, 0)`).
|
|
445
|
+
secs = float(raw_ltd) - 19800
|
|
446
|
+
last_trading_date = datetime.datetime.fromtimestamp(
|
|
447
|
+
secs, tz=datetime.timezone(_IST_OFFSET)
|
|
448
|
+
)
|
|
449
|
+
except (TypeError, ValueError):
|
|
450
|
+
last_trading_date = None
|
|
451
|
+
|
|
452
|
+
known = {
|
|
453
|
+
"ticker", "symbol", "token", "exchange", "instrument_name", "series",
|
|
454
|
+
"expiry_date", "isin_code", "option_type", "strike_price", "tick",
|
|
455
|
+
"lot_size", "eligibility", "security_desc", "last_trading_date",
|
|
456
|
+
"asm_gsm_stage",
|
|
457
|
+
}
|
|
458
|
+
extras = {k: v for k, v in row.items() if k not in known and v}
|
|
459
|
+
|
|
460
|
+
return Instrument(
|
|
461
|
+
ticker=row.get("ticker", "") or "",
|
|
462
|
+
token=_safe_int(row.get("token", "")),
|
|
463
|
+
exchange=row.get("exchange", "") or "",
|
|
464
|
+
instrument_name=row.get("instrument_name", "") or "",
|
|
465
|
+
symbol=row.get("symbol", "") or "",
|
|
466
|
+
series=row.get("series", "") or "",
|
|
467
|
+
expiry_date=row.get("expiry_date", "") or "",
|
|
468
|
+
isin=row.get("isin_code", "") or "",
|
|
469
|
+
option_type=row.get("option_type", "") or "",
|
|
470
|
+
strike_price=_safe_float(row.get("strike_price", "")),
|
|
471
|
+
tick=_safe_float(row.get("tick", "")),
|
|
472
|
+
lot_size=_safe_int(row.get("lot_size", "")),
|
|
473
|
+
eligibility=_safe_int(row.get("eligibility", "")) > 0,
|
|
474
|
+
name=row.get("security_desc", "") or "",
|
|
475
|
+
last_trading_date=last_trading_date,
|
|
476
|
+
asm_gsm_stage=row.get("asm_gsm_stage", "") or "",
|
|
477
|
+
extras=extras,
|
|
478
|
+
)
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "2.1.8b1"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|