bursakit 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bursakit/__init__.py +78 -0
- bursakit/_util.py +64 -0
- bursakit/cache.py +382 -0
- bursakit/cli.py +114 -0
- bursakit/errors.py +69 -0
- bursakit/macro.py +59 -0
- bursakit/policy.py +197 -0
- bursakit/py.typed +0 -0
- bursakit/router.py +116 -0
- bursakit/schema.py +195 -0
- bursakit/sources/__init__.py +8 -0
- bursakit/sources/_yf.py +97 -0
- bursakit/sources/base.py +53 -0
- bursakit/sources/bnm.py +107 -0
- bursakit/sources/datagovmy.py +63 -0
- bursakit/sources/yahoo.py +164 -0
- bursakit/tickers.py +77 -0
- bursakit-0.1.1.dist-info/METADATA +206 -0
- bursakit-0.1.1.dist-info/RECORD +22 -0
- bursakit-0.1.1.dist-info/WHEEL +4 -0
- bursakit-0.1.1.dist-info/entry_points.txt +2 -0
- bursakit-0.1.1.dist-info/licenses/LICENSE +21 -0
bursakit/__init__.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""bursakit: Bursa Malaysia stock data and Malaysian macro series as DataFrames.
|
|
2
|
+
|
|
3
|
+
Every public function follows the same path: check the cache; on a miss ask
|
|
4
|
+
the router, which tries each source in order; validate the answer against the
|
|
5
|
+
schema; store it; return it.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from datetime import timedelta
|
|
11
|
+
|
|
12
|
+
import pandas as pd
|
|
13
|
+
|
|
14
|
+
from . import _util, cache, macro, router
|
|
15
|
+
from ._util import DateLike
|
|
16
|
+
from .errors import BursakitError, RateLimited, SchemaError, SourceUnavailable, TickerNotFound
|
|
17
|
+
from .tickers import Ticker, resolve
|
|
18
|
+
|
|
19
|
+
__version__ = "0.1.1"
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"BursakitError",
|
|
23
|
+
"RateLimited",
|
|
24
|
+
"SchemaError",
|
|
25
|
+
"SourceUnavailable",
|
|
26
|
+
"Ticker",
|
|
27
|
+
"TickerNotFound",
|
|
28
|
+
"__version__",
|
|
29
|
+
"cache",
|
|
30
|
+
"dividends",
|
|
31
|
+
"listings",
|
|
32
|
+
"macro",
|
|
33
|
+
"prices",
|
|
34
|
+
"resolve",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def prices(
|
|
39
|
+
ticker: str | int | Ticker,
|
|
40
|
+
start: DateLike | None = None,
|
|
41
|
+
end: DateLike | None = None,
|
|
42
|
+
*,
|
|
43
|
+
refresh: bool = False,
|
|
44
|
+
) -> pd.DataFrame:
|
|
45
|
+
"""Daily open, high, low, close and volume for a Bursa-listed stock.
|
|
46
|
+
|
|
47
|
+
``ticker`` may be a stock code (``1155``), a ``.KL`` symbol or a stock name
|
|
48
|
+
(``"MAYBANK"``). ``start`` and ``end`` are inclusive; the default is the
|
|
49
|
+
last 365 days. Prices are in MYR, adjusted for splits and bonus issues.
|
|
50
|
+
"""
|
|
51
|
+
resolved = resolve(ticker)
|
|
52
|
+
first, last = _util.span(start, end, default=timedelta(days=365))
|
|
53
|
+
df, source = cache.series(
|
|
54
|
+
"prices",
|
|
55
|
+
resolved.code,
|
|
56
|
+
first,
|
|
57
|
+
last,
|
|
58
|
+
lambda s, e: router.fetch("prices", resolved, s, e),
|
|
59
|
+
verify="close",
|
|
60
|
+
overlap=7,
|
|
61
|
+
refresh=refresh,
|
|
62
|
+
)
|
|
63
|
+
return _util.tag(df, "prices", source, ticker=resolved.code)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def dividends(ticker: str | int | Ticker, *, refresh: bool = False) -> pd.DataFrame:
|
|
67
|
+
"""Dividend history: MYR per share, indexed by ex-date."""
|
|
68
|
+
resolved = resolve(ticker)
|
|
69
|
+
df, source = cache.frame(
|
|
70
|
+
"dividends", resolved.code, lambda: router.fetch("dividends", resolved), refresh=refresh
|
|
71
|
+
)
|
|
72
|
+
return _util.tag(df, "dividends", source, ticker=resolved.code)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def listings(*, refresh: bool = False) -> pd.DataFrame:
|
|
76
|
+
"""Companies listed on Bursa Malaysia: stock code, name, company, sector, market."""
|
|
77
|
+
df, source = cache.frame("listings", "all", lambda: router.fetch("listings"), refresh=refresh)
|
|
78
|
+
return _util.tag(df, "listings", source)
|
bursakit/_util.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""Date helpers shared across the package."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from datetime import date, datetime, timedelta, timezone
|
|
6
|
+
|
|
7
|
+
import pandas as pd
|
|
8
|
+
|
|
9
|
+
#: Malaysia Time. UTC+8 all year, so no tz database is needed.
|
|
10
|
+
MYT = timezone(timedelta(hours=8), "MYT")
|
|
11
|
+
|
|
12
|
+
DateLike = str | date | datetime | pd.Timestamp
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def today() -> date:
|
|
16
|
+
"""Today's calendar date in Malaysia."""
|
|
17
|
+
return datetime.now(MYT).date()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def span(start: DateLike | None, end: DateLike | None, *, default: timedelta) -> tuple[date, date]:
|
|
21
|
+
"""An inclusive date range: ``end`` defaults to today, ``start`` to ``end - default``."""
|
|
22
|
+
last = to_date(end) if end is not None else today()
|
|
23
|
+
first = to_date(start) if start is not None else last - default
|
|
24
|
+
if first > last:
|
|
25
|
+
raise ValueError(f"start {first} is after end {last}")
|
|
26
|
+
return first, last
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def tag(df: pd.DataFrame, kind: str, source: str, **extra: str) -> pd.DataFrame:
|
|
30
|
+
"""Record where a result came from in ``df.attrs``."""
|
|
31
|
+
from . import router
|
|
32
|
+
|
|
33
|
+
attrs: dict[str, str] = {"kind": kind}
|
|
34
|
+
attrs.update(extra)
|
|
35
|
+
attrs.update(
|
|
36
|
+
currency="MYR",
|
|
37
|
+
source=source,
|
|
38
|
+
attribution=router.attribution(source),
|
|
39
|
+
terms=router.terms(source),
|
|
40
|
+
)
|
|
41
|
+
df.attrs.clear()
|
|
42
|
+
df.attrs.update(attrs)
|
|
43
|
+
return df
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def to_date(value: DateLike) -> date:
|
|
47
|
+
"""Turn a string, date, datetime or Timestamp into a plain date."""
|
|
48
|
+
if isinstance(value, datetime): # also covers pd.Timestamp
|
|
49
|
+
return value.date()
|
|
50
|
+
if isinstance(value, date):
|
|
51
|
+
return value
|
|
52
|
+
if isinstance(value, str):
|
|
53
|
+
try:
|
|
54
|
+
return date.fromisoformat(value.strip())
|
|
55
|
+
except ValueError:
|
|
56
|
+
pass
|
|
57
|
+
try:
|
|
58
|
+
parsed = pd.Timestamp(value)
|
|
59
|
+
except (ValueError, TypeError) as exc:
|
|
60
|
+
raise ValueError(f"not a date: {value!r}") from exc
|
|
61
|
+
if pd.isna(parsed):
|
|
62
|
+
raise ValueError(f"not a date: {value!r}")
|
|
63
|
+
return parsed.date()
|
|
64
|
+
raise TypeError(f"expected a date or ISO date string, got {type(value).__name__}")
|
bursakit/cache.py
ADDED
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
"""Local SQLite cache and its expiry rules.
|
|
2
|
+
|
|
3
|
+
Two kinds of entry:
|
|
4
|
+
|
|
5
|
+
* **Series** (prices, fx, opr) are stored row by row with a coverage record.
|
|
6
|
+
Rows dated before the day they were fetched are settled and kept for good;
|
|
7
|
+
only the missing head or tail of a requested range is fetched.
|
|
8
|
+
* **Values** (dividends, listings, name lookups) are stored whole with a TTL.
|
|
9
|
+
|
|
10
|
+
Nothing is written without passing :func:`bursakit.schema.validate` first.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import os
|
|
17
|
+
import sqlite3
|
|
18
|
+
import sys
|
|
19
|
+
import time
|
|
20
|
+
from collections.abc import Callable, Iterator
|
|
21
|
+
from contextlib import closing, contextmanager
|
|
22
|
+
from dataclasses import dataclass
|
|
23
|
+
from datetime import date, timedelta
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from typing import Any
|
|
26
|
+
|
|
27
|
+
import pandas as pd
|
|
28
|
+
|
|
29
|
+
from . import _util
|
|
30
|
+
from .schema import SCHEMAS, validate
|
|
31
|
+
|
|
32
|
+
__all__ = ["TTL", "clear", "frame", "get_value", "info", "path", "put_value", "series"]
|
|
33
|
+
|
|
34
|
+
MINUTE, HOUR, DAY = 60.0, 3600.0, 86400.0
|
|
35
|
+
|
|
36
|
+
#: Seconds before an entry is refetched. For series this applies only to rows
|
|
37
|
+
#: dated today; older rows never expire.
|
|
38
|
+
TTL: dict[str, float] = {
|
|
39
|
+
"prices": 15 * MINUTE,
|
|
40
|
+
"fx": 6 * HOUR,
|
|
41
|
+
"opr": 6 * HOUR,
|
|
42
|
+
"dividends": 24 * HOUR,
|
|
43
|
+
"listings": 7 * DAY,
|
|
44
|
+
"alias": 30 * DAY,
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
#: Relative difference in an overlapping value that means history was restated.
|
|
48
|
+
RESTATED = 0.01
|
|
49
|
+
|
|
50
|
+
_DDL = """
|
|
51
|
+
CREATE TABLE IF NOT EXISTS series_rows (
|
|
52
|
+
kind TEXT NOT NULL, key TEXT NOT NULL, date TEXT NOT NULL, data TEXT NOT NULL,
|
|
53
|
+
PRIMARY KEY (kind, key, date)
|
|
54
|
+
);
|
|
55
|
+
CREATE TABLE IF NOT EXISTS coverage (
|
|
56
|
+
kind TEXT NOT NULL, key TEXT NOT NULL,
|
|
57
|
+
start TEXT NOT NULL, covered_through TEXT NOT NULL, settled_through TEXT NOT NULL,
|
|
58
|
+
fetched_at REAL NOT NULL, source TEXT NOT NULL,
|
|
59
|
+
PRIMARY KEY (kind, key)
|
|
60
|
+
);
|
|
61
|
+
CREATE TABLE IF NOT EXISTS vals (
|
|
62
|
+
kind TEXT NOT NULL, key TEXT NOT NULL, payload TEXT NOT NULL,
|
|
63
|
+
source TEXT NOT NULL, fetched_at REAL NOT NULL, expires_at REAL NOT NULL,
|
|
64
|
+
PRIMARY KEY (kind, key)
|
|
65
|
+
);
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
FetchRange = Callable[[date, date], "tuple[pd.DataFrame, str]"]
|
|
69
|
+
FetchFrame = Callable[[], "tuple[pd.DataFrame, str]"]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _now() -> float:
|
|
73
|
+
return time.time()
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def path() -> Path:
|
|
77
|
+
"""Directory holding the cache database."""
|
|
78
|
+
override = os.environ.get("BURSAKIT_CACHE_DIR")
|
|
79
|
+
if override:
|
|
80
|
+
return Path(override)
|
|
81
|
+
if sys.platform == "win32":
|
|
82
|
+
base = os.environ.get("LOCALAPPDATA") or str(Path.home() / "AppData" / "Local")
|
|
83
|
+
return Path(base) / "bursakit" / "Cache"
|
|
84
|
+
base = os.environ.get("XDG_CACHE_HOME") or str(Path.home() / ".cache")
|
|
85
|
+
return Path(base) / "bursakit"
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@contextmanager
|
|
89
|
+
def _db() -> Iterator[sqlite3.Connection]:
|
|
90
|
+
folder = path()
|
|
91
|
+
folder.mkdir(parents=True, exist_ok=True)
|
|
92
|
+
with closing(sqlite3.connect(folder / "bursakit.sqlite3", timeout=30)) as con:
|
|
93
|
+
con.executescript(_DDL)
|
|
94
|
+
with con: # one transaction per block
|
|
95
|
+
yield con
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def clear(kind: str | None = None) -> None:
|
|
99
|
+
"""Delete everything, or only one data type (``"prices"``, ``"fx"``, ...)."""
|
|
100
|
+
with _db() as con:
|
|
101
|
+
for table in ("series_rows", "coverage", "vals"):
|
|
102
|
+
if kind is None:
|
|
103
|
+
con.execute(f"DELETE FROM {table}")
|
|
104
|
+
else:
|
|
105
|
+
con.execute(f"DELETE FROM {table} WHERE kind = ?", (kind,))
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def info() -> dict[str, Any]:
|
|
109
|
+
"""Where the cache lives and how much is in it."""
|
|
110
|
+
with _db() as con:
|
|
111
|
+
rows = dict(con.execute("SELECT kind, COUNT(*) FROM series_rows GROUP BY kind").fetchall())
|
|
112
|
+
keys = dict(con.execute("SELECT kind, COUNT(*) FROM coverage GROUP BY kind").fetchall())
|
|
113
|
+
vals = dict(con.execute("SELECT kind, COUNT(*) FROM vals GROUP BY kind").fetchall())
|
|
114
|
+
file = path() / "bursakit.sqlite3"
|
|
115
|
+
return {
|
|
116
|
+
"path": str(file),
|
|
117
|
+
"bytes": file.stat().st_size if file.exists() else 0,
|
|
118
|
+
"series": {kind: {"keys": keys.get(kind, 0), "rows": n} for kind, n in rows.items()},
|
|
119
|
+
"entries": vals,
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
# --------------------------------------------------------------------- values
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def get_value(kind: str, key: str) -> Any | None:
|
|
127
|
+
"""A cached JSON value, or ``None`` if absent or expired."""
|
|
128
|
+
hit = _get_entry(kind, key)
|
|
129
|
+
return hit[0] if hit else None
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _get_entry(kind: str, key: str) -> tuple[Any, str] | None:
|
|
133
|
+
with _db() as con:
|
|
134
|
+
row = con.execute(
|
|
135
|
+
"SELECT payload, source, expires_at FROM vals WHERE kind = ? AND key = ?", (kind, key)
|
|
136
|
+
).fetchone()
|
|
137
|
+
if row is None or row[2] <= _now():
|
|
138
|
+
return None
|
|
139
|
+
return json.loads(row[0]), row[1]
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def put_value(kind: str, key: str, value: Any, *, source: str, ttl: float | None = None) -> None:
|
|
143
|
+
now = _now()
|
|
144
|
+
lifetime = TTL[kind] if ttl is None else ttl
|
|
145
|
+
with _db() as con:
|
|
146
|
+
con.execute(
|
|
147
|
+
"INSERT OR REPLACE INTO vals VALUES (?, ?, ?, ?, ?, ?)",
|
|
148
|
+
(kind, key, json.dumps(value), source, now, now + lifetime),
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def frame(
|
|
153
|
+
kind: str, key: str, fetch: FetchFrame, *, refresh: bool = False
|
|
154
|
+
) -> tuple[pd.DataFrame, str]:
|
|
155
|
+
"""A whole-frame entry: served from cache until its TTL passes."""
|
|
156
|
+
schema = SCHEMAS[kind]
|
|
157
|
+
if not refresh:
|
|
158
|
+
hit = _get_entry(kind, key)
|
|
159
|
+
if hit is not None:
|
|
160
|
+
payload, source = hit
|
|
161
|
+
cached = pd.DataFrame.from_records(
|
|
162
|
+
payload, columns=[schema.index, *schema.column_names]
|
|
163
|
+
)
|
|
164
|
+
return validate(cached.set_index(schema.index), schema, source="cache"), source
|
|
165
|
+
fetched, source = fetch()
|
|
166
|
+
df = validate(fetched, schema, source=source)
|
|
167
|
+
put_value(kind, key, _records(df, index=True), source=source)
|
|
168
|
+
return df, source
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
# --------------------------------------------------------------------- series
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
@dataclass(frozen=True)
|
|
175
|
+
class Coverage:
|
|
176
|
+
"""What is stored for one series: an unbroken range of dates."""
|
|
177
|
+
|
|
178
|
+
start: date
|
|
179
|
+
covered_through: date
|
|
180
|
+
#: Last date fetched after that day had ended. Rows up to here never expire.
|
|
181
|
+
settled_through: date
|
|
182
|
+
fetched_at: float
|
|
183
|
+
source: str
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def coverage(kind: str, key: str) -> Coverage | None:
|
|
187
|
+
with _db() as con:
|
|
188
|
+
row = con.execute(
|
|
189
|
+
"SELECT start, covered_through, settled_through, fetched_at, source "
|
|
190
|
+
"FROM coverage WHERE kind = ? AND key = ?",
|
|
191
|
+
(kind, key),
|
|
192
|
+
).fetchone()
|
|
193
|
+
if row is None:
|
|
194
|
+
return None
|
|
195
|
+
return Coverage(
|
|
196
|
+
date.fromisoformat(row[0]), date.fromisoformat(row[1]), date.fromisoformat(row[2]),
|
|
197
|
+
row[3], row[4],
|
|
198
|
+
) # fmt: skip
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def plan(
|
|
202
|
+
cov: Coverage | None, start: date, end: date, *, now: float, ttl: float, overlap: int = 0
|
|
203
|
+
) -> list[tuple[str, date, date]]:
|
|
204
|
+
"""Which ranges must be fetched to answer ``start..end`` from the cache.
|
|
205
|
+
|
|
206
|
+
Returns ``(part, from, to)`` tuples, where part is ``"all"``, ``"tail"`` or
|
|
207
|
+
``"head"``. An empty list means the cache can answer on its own.
|
|
208
|
+
"""
|
|
209
|
+
if cov is None:
|
|
210
|
+
return [("all", start, end)]
|
|
211
|
+
ranges: list[tuple[str, date, date]] = []
|
|
212
|
+
stale = now - cov.fetched_at > ttl
|
|
213
|
+
if end > cov.covered_through or (end > cov.settled_through and stale):
|
|
214
|
+
tail_from = cov.settled_through + timedelta(days=1 - overlap)
|
|
215
|
+
ranges.append(("tail", max(tail_from, cov.start), end))
|
|
216
|
+
if start < cov.start:
|
|
217
|
+
ranges.append(("head", start, cov.start - timedelta(days=1)))
|
|
218
|
+
return ranges
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def series(
|
|
222
|
+
kind: str,
|
|
223
|
+
key: str,
|
|
224
|
+
start: date,
|
|
225
|
+
end: date,
|
|
226
|
+
fetch: FetchRange,
|
|
227
|
+
*,
|
|
228
|
+
verify: str | None = None,
|
|
229
|
+
overlap: int = 0,
|
|
230
|
+
refresh: bool = False,
|
|
231
|
+
) -> tuple[pd.DataFrame, str]:
|
|
232
|
+
"""Answer ``start..end`` (inclusive), fetching only what the cache lacks.
|
|
233
|
+
|
|
234
|
+
``verify`` names a column to compare where a tail fetch overlaps stored
|
|
235
|
+
rows. If the values moved (a split restated the history), the whole series
|
|
236
|
+
is refetched.
|
|
237
|
+
|
|
238
|
+
``refresh`` swaps everything stored for the series for a new fetch of
|
|
239
|
+
``start..end``. Stored rows are only removed once that fetch has succeeded.
|
|
240
|
+
"""
|
|
241
|
+
schema = SCHEMAS[kind]
|
|
242
|
+
today = _util.today()
|
|
243
|
+
now = _now()
|
|
244
|
+
end = min(end, today)
|
|
245
|
+
if start > end:
|
|
246
|
+
empty = pd.DataFrame(columns=schema.column_names, index=pd.Index([], dtype="object"))
|
|
247
|
+
return validate(empty, schema), "cache"
|
|
248
|
+
if refresh:
|
|
249
|
+
# Fetch before touching the cache: a failed refresh leaves the stored rows alone.
|
|
250
|
+
df, source = fetch(start, end)
|
|
251
|
+
_store(kind, key, df, start, end, source=source, today=today, now=now, replace=True)
|
|
252
|
+
return _read(kind, key, start, end), source
|
|
253
|
+
|
|
254
|
+
cov = coverage(kind, key)
|
|
255
|
+
source = cov.source if cov else ""
|
|
256
|
+
for part, part_start, part_end in plan(
|
|
257
|
+
cov, start, end, now=now, ttl=TTL[kind], overlap=overlap if verify else 0
|
|
258
|
+
):
|
|
259
|
+
df, source = fetch(part_start, part_end)
|
|
260
|
+
df = validate(df, schema, source=source)
|
|
261
|
+
if (
|
|
262
|
+
part == "tail"
|
|
263
|
+
and verify
|
|
264
|
+
and cov
|
|
265
|
+
and not _agrees(kind, key, df, cov, part_start, verify)
|
|
266
|
+
):
|
|
267
|
+
full_start = min(start, cov.start)
|
|
268
|
+
df, source = fetch(full_start, end) # the old history stays if this fails
|
|
269
|
+
_store(
|
|
270
|
+
kind, key, df, full_start, end, source=source, today=today, now=now, replace=True
|
|
271
|
+
)
|
|
272
|
+
break
|
|
273
|
+
_store(kind, key, df, part_start, part_end, source=source, today=today, now=now)
|
|
274
|
+
return _read(kind, key, start, end), source
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _agrees(
|
|
278
|
+
kind: str, key: str, new: pd.DataFrame, cov: Coverage, start: date, column: str
|
|
279
|
+
) -> bool:
|
|
280
|
+
"""Do freshly fetched rows match what is stored for the settled overlap?"""
|
|
281
|
+
stored = _read(kind, key, start, cov.settled_through)[column]
|
|
282
|
+
fresh = new.loc[new.index <= pd.Timestamp(cov.settled_through), column]
|
|
283
|
+
if not stored.index.equals(fresh.index):
|
|
284
|
+
return False
|
|
285
|
+
if stored.empty:
|
|
286
|
+
return True
|
|
287
|
+
drift = ((stored - fresh).abs() / stored.abs()).max()
|
|
288
|
+
return bool(drift <= RESTATED)
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def _store(
|
|
292
|
+
kind: str,
|
|
293
|
+
key: str,
|
|
294
|
+
df: pd.DataFrame,
|
|
295
|
+
start: date,
|
|
296
|
+
end: date,
|
|
297
|
+
*,
|
|
298
|
+
source: str,
|
|
299
|
+
today: date,
|
|
300
|
+
now: float,
|
|
301
|
+
replace: bool = False,
|
|
302
|
+
) -> None:
|
|
303
|
+
"""Replace the stored rows for ``start..end`` and extend the coverage record.
|
|
304
|
+
|
|
305
|
+
With ``replace``, everything stored for the series is swapped for these
|
|
306
|
+
rows in the same transaction, so a reader never sees it half done.
|
|
307
|
+
"""
|
|
308
|
+
df = validate(df, SCHEMAS[kind], source=source) # the only door into the cache
|
|
309
|
+
df = df.loc[(df.index >= pd.Timestamp(start)) & (df.index <= pd.Timestamp(end))]
|
|
310
|
+
rows = [(kind, key, r.pop("date"), json.dumps(r)) for r in _records(df, index=True)]
|
|
311
|
+
reached = min(end, today)
|
|
312
|
+
settled = min(end, today - timedelta(days=1))
|
|
313
|
+
with _db() as con:
|
|
314
|
+
if replace:
|
|
315
|
+
con.execute("DELETE FROM series_rows WHERE kind = ? AND key = ?", (kind, key))
|
|
316
|
+
con.execute("DELETE FROM coverage WHERE kind = ? AND key = ?", (kind, key))
|
|
317
|
+
else:
|
|
318
|
+
con.execute(
|
|
319
|
+
"DELETE FROM series_rows WHERE kind = ? AND key = ? AND date BETWEEN ? AND ?",
|
|
320
|
+
(kind, key, start.isoformat(), end.isoformat()),
|
|
321
|
+
)
|
|
322
|
+
con.executemany("INSERT INTO series_rows VALUES (?, ?, ?, ?)", rows)
|
|
323
|
+
old = con.execute(
|
|
324
|
+
"SELECT start, covered_through, settled_through, fetched_at FROM coverage "
|
|
325
|
+
"WHERE kind = ? AND key = ?",
|
|
326
|
+
(kind, key),
|
|
327
|
+
).fetchone()
|
|
328
|
+
if old is not None:
|
|
329
|
+
new_start = min(start, date.fromisoformat(old[0]))
|
|
330
|
+
reached = max(reached, date.fromisoformat(old[1]))
|
|
331
|
+
settled = max(settled, date.fromisoformat(old[2]))
|
|
332
|
+
# A head fetch says nothing about how fresh today's rows are.
|
|
333
|
+
fetched_at = now if end >= today else old[3]
|
|
334
|
+
else:
|
|
335
|
+
new_start, fetched_at = start, now
|
|
336
|
+
con.execute(
|
|
337
|
+
"INSERT OR REPLACE INTO coverage VALUES (?, ?, ?, ?, ?, ?, ?)",
|
|
338
|
+
(
|
|
339
|
+
kind,
|
|
340
|
+
key,
|
|
341
|
+
new_start.isoformat(),
|
|
342
|
+
reached.isoformat(),
|
|
343
|
+
settled.isoformat(),
|
|
344
|
+
fetched_at,
|
|
345
|
+
source,
|
|
346
|
+
),
|
|
347
|
+
)
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def _read(kind: str, key: str, start: date, end: date) -> pd.DataFrame:
|
|
351
|
+
schema = SCHEMAS[kind]
|
|
352
|
+
with _db() as con:
|
|
353
|
+
rows = con.execute(
|
|
354
|
+
"SELECT date, data FROM series_rows "
|
|
355
|
+
"WHERE kind = ? AND key = ? AND date BETWEEN ? AND ? ORDER BY date",
|
|
356
|
+
(kind, key, start.isoformat(), end.isoformat()),
|
|
357
|
+
).fetchall()
|
|
358
|
+
df = pd.DataFrame( # not from_records: pandas 2.x rejects an empty index there
|
|
359
|
+
[json.loads(data) for _, data in rows],
|
|
360
|
+
index=pd.Index([day for day, _ in rows], dtype="object"),
|
|
361
|
+
columns=schema.column_names,
|
|
362
|
+
)
|
|
363
|
+
return validate(df, schema, source="cache")
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _records(df: pd.DataFrame, *, index: bool) -> list[dict[str, Any]]:
|
|
367
|
+
"""Rows as JSON-ready dicts: dates as ISO strings, missing values as ``None``."""
|
|
368
|
+
flat = df.reset_index() if index else df
|
|
369
|
+
out: list[dict[str, Any]] = []
|
|
370
|
+
for row in flat.to_dict("records"):
|
|
371
|
+
clean: dict[str, Any] = {}
|
|
372
|
+
for name, value in row.items():
|
|
373
|
+
if isinstance(value, pd.Timestamp):
|
|
374
|
+
clean[str(name)] = value.strftime("%Y-%m-%d")
|
|
375
|
+
elif value is None or value is pd.NA or (isinstance(value, float) and value != value):
|
|
376
|
+
clean[str(name)] = None
|
|
377
|
+
elif hasattr(value, "item"):
|
|
378
|
+
clean[str(name)] = value.item()
|
|
379
|
+
else:
|
|
380
|
+
clean[str(name)] = value
|
|
381
|
+
out.append(clean)
|
|
382
|
+
return out
|
bursakit/cli.py
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""Command line interface: ``bursakit prices 1155 --start 2024-01-01 --csv``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
from collections.abc import Sequence
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
import pandas as pd
|
|
12
|
+
|
|
13
|
+
import bursakit
|
|
14
|
+
|
|
15
|
+
from .errors import BursakitError
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _parser() -> argparse.ArgumentParser:
|
|
19
|
+
parser = argparse.ArgumentParser(
|
|
20
|
+
prog="bursakit", description="Bursa Malaysia stock data and Malaysian macro series."
|
|
21
|
+
)
|
|
22
|
+
parser.add_argument("--version", action="version", version=f"bursakit {bursakit.__version__}")
|
|
23
|
+
commands = parser.add_subparsers(dest="command", required=True, metavar="command")
|
|
24
|
+
|
|
25
|
+
def output(sub: argparse.ArgumentParser) -> None:
|
|
26
|
+
sub.add_argument("--csv", action="store_true", help="write CSV instead of a table")
|
|
27
|
+
sub.add_argument("--out", metavar="FILE", help="write to FILE instead of the screen")
|
|
28
|
+
sub.add_argument("--refresh", action="store_true", help="ignore the local cache")
|
|
29
|
+
|
|
30
|
+
def dates(sub: argparse.ArgumentParser) -> None:
|
|
31
|
+
sub.add_argument("--start", metavar="YYYY-MM-DD", help="first date, inclusive")
|
|
32
|
+
sub.add_argument("--end", metavar="YYYY-MM-DD", help="last date, inclusive")
|
|
33
|
+
|
|
34
|
+
sub = commands.add_parser("prices", help="daily open, high, low, close, volume")
|
|
35
|
+
sub.add_argument("ticker", help="stock code, .KL symbol or stock name")
|
|
36
|
+
dates(sub)
|
|
37
|
+
output(sub)
|
|
38
|
+
|
|
39
|
+
sub = commands.add_parser("dividends", help="dividend history")
|
|
40
|
+
sub.add_argument("ticker", help="stock code, .KL symbol or stock name")
|
|
41
|
+
output(sub)
|
|
42
|
+
|
|
43
|
+
sub = commands.add_parser("listings", help="companies listed on Bursa Malaysia")
|
|
44
|
+
output(sub)
|
|
45
|
+
|
|
46
|
+
sub = commands.add_parser("opr", help="Overnight Policy Rate decisions")
|
|
47
|
+
dates(sub)
|
|
48
|
+
output(sub)
|
|
49
|
+
|
|
50
|
+
sub = commands.add_parser("fx", help="ringgit exchange rates, MYR per 1 unit")
|
|
51
|
+
sub.add_argument("currency", help="currency code, e.g. USD")
|
|
52
|
+
dates(sub)
|
|
53
|
+
output(sub)
|
|
54
|
+
|
|
55
|
+
sub = commands.add_parser("resolve", help="show which ticker an input resolves to")
|
|
56
|
+
sub.add_argument("ticker")
|
|
57
|
+
|
|
58
|
+
sub = commands.add_parser("cache", help="inspect or empty the local cache")
|
|
59
|
+
sub.add_argument("action", choices=["info", "clear"])
|
|
60
|
+
sub.add_argument("kind", nargs="?", help="with clear: only this data type, e.g. prices")
|
|
61
|
+
return parser
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _emit(df: pd.DataFrame, args: argparse.Namespace) -> None:
|
|
65
|
+
as_csv = args.csv or bool(args.out and str(args.out).lower().endswith(".csv"))
|
|
66
|
+
if as_csv:
|
|
67
|
+
text = df.to_csv(lineterminator="\n", date_format="%Y-%m-%d")
|
|
68
|
+
elif df.empty:
|
|
69
|
+
text = "(no rows)\n"
|
|
70
|
+
else:
|
|
71
|
+
shown = df.copy()
|
|
72
|
+
if isinstance(shown.index, pd.DatetimeIndex):
|
|
73
|
+
shown.index = pd.Index(shown.index.strftime("%Y-%m-%d"), name=shown.index.name)
|
|
74
|
+
text = shown.to_string(max_rows=None, na_rep="") + "\n"
|
|
75
|
+
if args.out:
|
|
76
|
+
Path(args.out).write_text(text, encoding="utf-8", newline="")
|
|
77
|
+
print(f"wrote {len(df)} rows to {args.out}", file=sys.stderr)
|
|
78
|
+
else:
|
|
79
|
+
sys.stdout.write(text)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _run(args: argparse.Namespace) -> None:
|
|
83
|
+
if args.command == "prices":
|
|
84
|
+
_emit(bursakit.prices(args.ticker, args.start, args.end, refresh=args.refresh), args)
|
|
85
|
+
elif args.command == "dividends":
|
|
86
|
+
_emit(bursakit.dividends(args.ticker, refresh=args.refresh), args)
|
|
87
|
+
elif args.command == "listings":
|
|
88
|
+
_emit(bursakit.listings(refresh=args.refresh), args)
|
|
89
|
+
elif args.command == "opr":
|
|
90
|
+
_emit(bursakit.macro.opr(args.start, args.end, refresh=args.refresh), args)
|
|
91
|
+
elif args.command == "fx":
|
|
92
|
+
_emit(bursakit.macro.fx(args.currency, args.start, args.end, refresh=args.refresh), args)
|
|
93
|
+
elif args.command == "resolve":
|
|
94
|
+
ticker = bursakit.resolve(args.ticker)
|
|
95
|
+
print(f"{ticker.code}\t{ticker.symbol}\t{ticker.name or ''}".rstrip())
|
|
96
|
+
elif args.action == "info":
|
|
97
|
+
print(json.dumps(bursakit.cache.info(), indent=2))
|
|
98
|
+
else:
|
|
99
|
+
bursakit.cache.clear(args.kind)
|
|
100
|
+
print(f"cleared {args.kind or 'everything'}", file=sys.stderr)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
104
|
+
args = _parser().parse_args(argv)
|
|
105
|
+
try:
|
|
106
|
+
_run(args)
|
|
107
|
+
except (BursakitError, ValueError) as exc:
|
|
108
|
+
print(f"bursakit: {exc}", file=sys.stderr)
|
|
109
|
+
return 1
|
|
110
|
+
return 0
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
if __name__ == "__main__":
|
|
114
|
+
raise SystemExit(main())
|