qlib-sync 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qlib_sync/__init__.py +13 -0
- qlib_sync/api.py +67 -0
- qlib_sync/cli.py +81 -0
- qlib_sync/cn/__init__.py +1 -0
- qlib_sync/cn/market.py +381 -0
- qlib_sync/cn/update.py +609 -0
- qlib_sync/mcp_server.py +249 -0
- qlib_sync/txn.py +71 -0
- qlib_sync-0.1.0.dist-info/METADATA +219 -0
- qlib_sync-0.1.0.dist-info/RECORD +13 -0
- qlib_sync-0.1.0.dist-info/WHEEL +4 -0
- qlib_sync-0.1.0.dist-info/entry_points.txt +2 -0
- qlib_sync-0.1.0.dist-info/licenses/LICENSE +21 -0
qlib_sync/mcp_server.py
ADDED
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
"""MCP server: let AI assistants check, update, and read the local A-share dataset.
|
|
2
|
+
|
|
3
|
+
Run with `qlib-sync mcp` (stdio), or without installing: `uvx qlib-sync mcp`.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import dataclasses
|
|
9
|
+
import datetime as dt
|
|
10
|
+
import os
|
|
11
|
+
import re
|
|
12
|
+
import threading
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any, Literal
|
|
15
|
+
|
|
16
|
+
import anyio
|
|
17
|
+
from mcp.server.mcpserver import MCPServer
|
|
18
|
+
from mcp.server.mcpserver.exceptions import ToolError
|
|
19
|
+
from mcp.types import ToolAnnotations
|
|
20
|
+
|
|
21
|
+
from .cn import market
|
|
22
|
+
from .cn.update import DEFAULT_SOURCE_URL, UpdateError, UpdateOptions, run_update
|
|
23
|
+
|
|
24
|
+
DATA_DIR_ENV = "QLIB_SYNC_DATA_DIR"
|
|
25
|
+
PRICE_FIELDS = ("open", "high", "low", "close", "vwap")
|
|
26
|
+
ALL_FIELDS = (*PRICE_FIELDS, "volume", "amount", "change", "factor")
|
|
27
|
+
DEFAULT_FIELDS = ("open", "high", "low", "close", "volume", "change")
|
|
28
|
+
MAX_SYMBOLS = 20
|
|
29
|
+
MAX_ROWS_LIMIT = 5000
|
|
30
|
+
|
|
31
|
+
INSTRUCTIONS = """\
|
|
32
|
+
Local China A-share daily data (Qlib format), installed and kept current by qlib-sync.
|
|
33
|
+
Call cn_status first to see how recent the data is. If the user needs recent prices and
|
|
34
|
+
an update is available, call cn_update (it takes about a minute and is safe: on any
|
|
35
|
+
failure the previous data is restored). Symbols accept 600519, SH600519 or 600519.SH.
|
|
36
|
+
cn_get_prices returns real traded prices by default; pass price_basis="adjusted" for the
|
|
37
|
+
split- and dividend-adjusted series used in backtests. Data covers trading days only;
|
|
38
|
+
the latest date is the last close, never intraday.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
_SYMBOL = re.compile(r"^(?:(SH|SZ|BJ)(\d{6})|(\d{6})(?:\.(SH|SZ|BJ))?)$")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def normalize_symbol(value: str) -> str:
|
|
45
|
+
"""Map 600519, sh600519, or 600519.SH to Qlib's SH600519."""
|
|
46
|
+
match = _SYMBOL.match(value.strip().upper())
|
|
47
|
+
if not match:
|
|
48
|
+
raise ToolError(f"unrecognized symbol {value!r}; use a 6-digit code such as 600519 or SH600519")
|
|
49
|
+
prefix, code = (match.group(1), match.group(2)) if match.group(1) else (match.group(4), match.group(3))
|
|
50
|
+
if prefix:
|
|
51
|
+
return f"{prefix}{code}"
|
|
52
|
+
if code.startswith("92") or code[0] in "48":
|
|
53
|
+
return f"BJ{code}"
|
|
54
|
+
if code[0] in "69":
|
|
55
|
+
return f"SH{code}"
|
|
56
|
+
return f"SZ{code}"
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class QlibReader:
|
|
60
|
+
"""Initialize Qlib once per process and re-initialize after the data is replaced."""
|
|
61
|
+
|
|
62
|
+
def __init__(self, data_dir: Path):
|
|
63
|
+
self.data_dir = data_dir
|
|
64
|
+
self._ready = False
|
|
65
|
+
self._lock = threading.Lock()
|
|
66
|
+
|
|
67
|
+
def invalidate(self) -> None:
|
|
68
|
+
with self._lock:
|
|
69
|
+
self._ready = False
|
|
70
|
+
|
|
71
|
+
def _ensure(self) -> None:
|
|
72
|
+
# pylint: disable=import-outside-toplevel
|
|
73
|
+
import qlib
|
|
74
|
+
from qlib.config import REG_CN
|
|
75
|
+
|
|
76
|
+
if not (self.data_dir / "calendars" / "day.txt").is_file():
|
|
77
|
+
raise ToolError(f"no A-share data at {self.data_dir}; call cn_update to install it")
|
|
78
|
+
if not self._ready:
|
|
79
|
+
# kernels=1 keeps reads in-process; spawning workers per request is far slower.
|
|
80
|
+
qlib.init(provider_uri=str(self.data_dir), region=REG_CN, logging_level="ERROR", kernels=1)
|
|
81
|
+
self._ready = True
|
|
82
|
+
|
|
83
|
+
def prices(self, symbols: list[str], start: str, end: str, fields: list[str], adjusted: bool):
|
|
84
|
+
from qlib.data import D # pylint: disable=import-outside-toplevel
|
|
85
|
+
|
|
86
|
+
with self._lock:
|
|
87
|
+
self._ensure()
|
|
88
|
+
columns = sorted(set(fields) | ({"factor"} if not adjusted else set()))
|
|
89
|
+
frame = D.features(symbols, [f"${name}" for name in columns], start_time=start, end_time=end)
|
|
90
|
+
frame.columns = [column.lstrip("$") for column in frame.columns]
|
|
91
|
+
if not adjusted:
|
|
92
|
+
# Qlib stores adjusted prices: raw = adjusted / factor, raw volume = adjusted * factor.
|
|
93
|
+
for name in PRICE_FIELDS:
|
|
94
|
+
if name in frame:
|
|
95
|
+
frame[name] = frame[name] / frame["factor"]
|
|
96
|
+
if "volume" in frame:
|
|
97
|
+
frame["volume"] = frame["volume"] * frame["factor"]
|
|
98
|
+
frame = frame[[name for name in fields if name in frame]].dropna(how="all").astype("float64")
|
|
99
|
+
# Qlib stores float32; round away the noise so 1452.9998779 reads as 1453.0.
|
|
100
|
+
return frame.round({**{name: 3 for name in PRICE_FIELDS}, "volume": 0, "amount": 2, "change": 6, "factor": 6})
|
|
101
|
+
|
|
102
|
+
def pool(self, name: str, date: str | None) -> list[str]:
|
|
103
|
+
from qlib.data import D # pylint: disable=import-outside-toplevel
|
|
104
|
+
|
|
105
|
+
with self._lock:
|
|
106
|
+
self._ensure()
|
|
107
|
+
day = date or market.read_latest_date(self.data_dir)
|
|
108
|
+
return sorted(D.list_instruments(D.instruments(name), start_time=day, end_time=day, as_list=True))
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def create_server(data_dir: Path | None = None, source_url: str = DEFAULT_SOURCE_URL) -> MCPServer:
|
|
112
|
+
# An empty variable (an unset option in an MCP client's config) means the default.
|
|
113
|
+
data_dir = (data_dir or Path(os.environ.get(DATA_DIR_ENV) or market.DEFAULT_DATA_DIR)).expanduser().resolve()
|
|
114
|
+
reader = QlibReader(data_dir)
|
|
115
|
+
server = MCPServer("qlib-sync", instructions=INSTRUCTIONS)
|
|
116
|
+
|
|
117
|
+
@server.tool(
|
|
118
|
+
name="cn_status",
|
|
119
|
+
annotations=ToolAnnotations(title="A-share data status", read_only_hint=True, open_world_hint=True),
|
|
120
|
+
)
|
|
121
|
+
def cn_status(offline: bool = False) -> dict[str, Any]:
|
|
122
|
+
"""Report the local A-share data's latest trading date, size, and stock pools, and whether
|
|
123
|
+
the community release has newer data (set offline=True to skip the network check)."""
|
|
124
|
+
result = market.collect_status(data_dir, market.DEFAULT_PROTECTED_DIR, offline, 15)
|
|
125
|
+
result.pop("related_datasets", None)
|
|
126
|
+
result["pools"] = [pool["name"] for pool in result.pop("instrument_pools")]
|
|
127
|
+
result["today"] = dt.date.today().isoformat()
|
|
128
|
+
return result
|
|
129
|
+
|
|
130
|
+
@server.tool(
|
|
131
|
+
name="cn_update",
|
|
132
|
+
annotations=ToolAnnotations(
|
|
133
|
+
title="Update A-share data",
|
|
134
|
+
read_only_hint=False,
|
|
135
|
+
destructive_hint=False,
|
|
136
|
+
idempotent_hint=True,
|
|
137
|
+
open_world_hint=True,
|
|
138
|
+
),
|
|
139
|
+
)
|
|
140
|
+
async def cn_update(force: bool = False) -> dict[str, Any]:
|
|
141
|
+
"""Install or update the local A-share data from the community release.
|
|
142
|
+
|
|
143
|
+
Returns quickly when already current. Otherwise downloads about 550 MB, verifies its
|
|
144
|
+
checksum, and swaps it in; this takes roughly a minute. On any failure the previous
|
|
145
|
+
data is restored. The result lists new trading days and listed/delisted symbols.
|
|
146
|
+
"""
|
|
147
|
+
options = UpdateOptions(
|
|
148
|
+
data_dir=data_dir, protected_dir=market.DEFAULT_PROTECTED_DIR, source_url=source_url, force=force
|
|
149
|
+
)
|
|
150
|
+
try:
|
|
151
|
+
result = await anyio.to_thread.run_sync(run_update, options)
|
|
152
|
+
except UpdateError as exc:
|
|
153
|
+
raise ToolError(f"update failed and the existing data is unchanged: {exc}") from exc
|
|
154
|
+
if result.status != "up_to_date":
|
|
155
|
+
reader.invalidate()
|
|
156
|
+
payload = dataclasses.asdict(result)
|
|
157
|
+
payload["summary"] = result.summary_lines()
|
|
158
|
+
return payload
|
|
159
|
+
|
|
160
|
+
@server.tool(
|
|
161
|
+
name="cn_health_check",
|
|
162
|
+
annotations=ToolAnnotations(title="Check A-share data", read_only_hint=True, open_world_hint=False),
|
|
163
|
+
)
|
|
164
|
+
async def cn_health_check() -> dict[str, Any]:
|
|
165
|
+
"""Validate the local data's structure and perform a real Qlib read; returns ok, errors, checks."""
|
|
166
|
+
return await anyio.to_thread.run_sync(market.health_check, data_dir, market.DEFAULT_PROTECTED_DIR)
|
|
167
|
+
|
|
168
|
+
@server.tool(
|
|
169
|
+
name="cn_get_prices",
|
|
170
|
+
annotations=ToolAnnotations(title="Get A-share daily prices", read_only_hint=True, open_world_hint=False),
|
|
171
|
+
)
|
|
172
|
+
async def cn_get_prices(
|
|
173
|
+
symbols: list[str],
|
|
174
|
+
start: str,
|
|
175
|
+
end: str | None = None,
|
|
176
|
+
fields: list[str] | None = None,
|
|
177
|
+
price_basis: Literal["raw", "adjusted"] = "raw",
|
|
178
|
+
max_rows: int = 500,
|
|
179
|
+
) -> dict[str, Any]:
|
|
180
|
+
"""Daily bars for up to 20 A-share symbols between start and end (YYYY-MM-DD, inclusive;
|
|
181
|
+
end defaults to the latest date).
|
|
182
|
+
|
|
183
|
+
fields: any of open, high, low, close, vwap, volume, amount, change, factor
|
|
184
|
+
(default open, high, low, close, volume, change). change is the daily return as a
|
|
185
|
+
fraction (0.01 = +1%). price_basis="raw" gives real traded prices and share volume;
|
|
186
|
+
"adjusted" gives Qlib's split- and dividend-adjusted series. Rows are capped by max_rows.
|
|
187
|
+
"""
|
|
188
|
+
if not symbols:
|
|
189
|
+
raise ToolError("pass at least one symbol, for example ['600519']")
|
|
190
|
+
if len(symbols) > MAX_SYMBOLS:
|
|
191
|
+
raise ToolError(f"at most {MAX_SYMBOLS} symbols per call; split the request")
|
|
192
|
+
wanted = list(fields or DEFAULT_FIELDS)
|
|
193
|
+
unknown = sorted(set(wanted) - set(ALL_FIELDS))
|
|
194
|
+
if unknown:
|
|
195
|
+
raise ToolError(f"unknown fields {unknown}; choose from {list(ALL_FIELDS)}")
|
|
196
|
+
codes = [normalize_symbol(symbol) for symbol in symbols]
|
|
197
|
+
for value in (start, end):
|
|
198
|
+
if value is not None and not re.fullmatch(r"\d{4}-\d{2}-\d{2}", value):
|
|
199
|
+
raise ToolError(f"dates must be YYYY-MM-DD, got {value!r}")
|
|
200
|
+
limit = max(1, min(max_rows, MAX_ROWS_LIMIT))
|
|
201
|
+
latest = market.read_latest_date(data_dir)
|
|
202
|
+
frame = await anyio.to_thread.run_sync(
|
|
203
|
+
reader.prices, codes, start, end or latest, wanted, price_basis == "adjusted"
|
|
204
|
+
)
|
|
205
|
+
rows = [
|
|
206
|
+
{"symbol": symbol, "date": day.date().isoformat(), **{k: _clean(v) for k, v in values.items()}}
|
|
207
|
+
for (symbol, day), values in frame.to_dict(orient="index").items()
|
|
208
|
+
]
|
|
209
|
+
missing = sorted(set(codes) - {row["symbol"] for row in rows})
|
|
210
|
+
return {
|
|
211
|
+
"price_basis": price_basis,
|
|
212
|
+
"data_latest_date": latest,
|
|
213
|
+
"rows": rows[:limit],
|
|
214
|
+
"total_rows": len(rows),
|
|
215
|
+
"truncated": len(rows) > limit,
|
|
216
|
+
"symbols_without_data": missing,
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
@server.tool(
|
|
220
|
+
name="cn_list_pool",
|
|
221
|
+
annotations=ToolAnnotations(title="List stock pool members", read_only_hint=True, open_world_hint=False),
|
|
222
|
+
)
|
|
223
|
+
async def cn_list_pool(pool: str = "csi300", date: str | None = None) -> dict[str, Any]:
|
|
224
|
+
"""Members of a stock pool on a date (default: the latest date). Pools: all, csi100, csi300,
|
|
225
|
+
csi500, csi800, csi1000, csiall, plus any local pools; cn_status lists what exists."""
|
|
226
|
+
available = sorted(path.stem for path in (data_dir / "instruments").glob("*.txt"))
|
|
227
|
+
if pool not in available:
|
|
228
|
+
raise ToolError(f"unknown pool {pool!r}; available: {available}")
|
|
229
|
+
members = await anyio.to_thread.run_sync(reader.pool, pool, date)
|
|
230
|
+
return {"pool": pool, "date": date or market.read_latest_date(data_dir), "count": len(members), "symbols": members}
|
|
231
|
+
|
|
232
|
+
return server
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _clean(value: float) -> float | None:
|
|
236
|
+
return None if value != value else value # NaN -> null in JSON
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def main(argv: list[str] | None = None) -> None:
|
|
240
|
+
import argparse # pylint: disable=import-outside-toplevel
|
|
241
|
+
|
|
242
|
+
parser = argparse.ArgumentParser(prog="qlib-sync mcp", description="Serve the A-share dataset over MCP (stdio).")
|
|
243
|
+
parser.add_argument("--data-dir", type=Path, help=f"Dataset directory (default: ${DATA_DIR_ENV} or ~/.qlib/qlib_data/cn_data)")
|
|
244
|
+
args = parser.parse_args(argv)
|
|
245
|
+
create_server(args.data_dir).run("stdio")
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
if __name__ == "__main__":
|
|
249
|
+
main()
|
qlib_sync/txn.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""Transactional replacement of a dataset directory.
|
|
2
|
+
|
|
3
|
+
A dataset is replaced with two renames: the active directory moves aside to a
|
|
4
|
+
backup path, then the staged directory moves into place. A verification step
|
|
5
|
+
then runs against the new data. If anything fails after the first rename, the
|
|
6
|
+
previous directory is put back, so the active path ends up holding either the
|
|
7
|
+
complete old dataset or the complete, verified new one.
|
|
8
|
+
|
|
9
|
+
Renames are atomic only within one filesystem, so callers stage new data next
|
|
10
|
+
to the active directory rather than in a system temp directory.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import shutil
|
|
16
|
+
from contextlib import contextmanager
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Callable, Iterator
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class LockedError(RuntimeError):
|
|
22
|
+
"""Another operation already holds the lock."""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@contextmanager
|
|
26
|
+
def exclusive_lock(lock_dir: Path) -> Iterator[None]:
|
|
27
|
+
"""Hold a lock for the duration of the block.
|
|
28
|
+
|
|
29
|
+
`mkdir` either creates the directory or fails, atomically, on every platform,
|
|
30
|
+
which makes a directory a dependable cross-process lock without extra packages.
|
|
31
|
+
"""
|
|
32
|
+
try:
|
|
33
|
+
lock_dir.mkdir()
|
|
34
|
+
except FileExistsError as exc:
|
|
35
|
+
raise LockedError(f"another operation is already running (lock: {lock_dir})") from exc
|
|
36
|
+
try:
|
|
37
|
+
yield
|
|
38
|
+
finally:
|
|
39
|
+
lock_dir.rmdir()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def replace_directory(
|
|
43
|
+
active: Path,
|
|
44
|
+
staged: Path,
|
|
45
|
+
backup: Path,
|
|
46
|
+
verify: Callable[[Path], None] | None = None,
|
|
47
|
+
) -> Path | None:
|
|
48
|
+
"""Swap `staged` into the `active` path and verify it.
|
|
49
|
+
|
|
50
|
+
Returns the backup path holding the previous directory, or None when there was
|
|
51
|
+
no previous directory. The caller decides whether to keep or delete the backup.
|
|
52
|
+
On any failure the previous state is restored and the exception propagates.
|
|
53
|
+
"""
|
|
54
|
+
if backup.exists():
|
|
55
|
+
raise FileExistsError(f"backup path already exists: {backup}")
|
|
56
|
+
had_previous = active.exists()
|
|
57
|
+
if had_previous:
|
|
58
|
+
active.rename(backup)
|
|
59
|
+
try:
|
|
60
|
+
staged.rename(active)
|
|
61
|
+
if verify is not None:
|
|
62
|
+
verify(active)
|
|
63
|
+
except BaseException:
|
|
64
|
+
# Whatever now sits at the active path is the new data that failed; the
|
|
65
|
+
# staged copy is disposable, so remove it before moving the old data back.
|
|
66
|
+
if active.exists():
|
|
67
|
+
shutil.rmtree(active)
|
|
68
|
+
if had_previous:
|
|
69
|
+
backup.rename(active)
|
|
70
|
+
raise
|
|
71
|
+
return backup if had_previous else None
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: qlib-sync
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Safely install and update a local Qlib China A-share dataset, with an MCP server for AI assistants.
|
|
5
|
+
Project-URL: Homepage, https://github.com/randyy179/qlib-sync
|
|
6
|
+
Project-URL: Issues, https://github.com/randyy179/qlib-sync/issues
|
|
7
|
+
Project-URL: Changelog, https://github.com/randyy179/qlib-sync/releases
|
|
8
|
+
Author: randyy179
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: a-share,china,market-data,mcp,qlib,quant,stock
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Financial and Insurance Industry
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Operating System :: MacOS
|
|
17
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Topic :: Office/Business :: Financial :: Investment
|
|
23
|
+
Requires-Python: <3.13,>=3.10
|
|
24
|
+
Requires-Dist: mcp<3,>=2.3
|
|
25
|
+
Requires-Dist: pyqlib>=0.9.6
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: pytest; extra == 'dev'
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
|
|
30
|
+
# qlib-sync
|
|
31
|
+
|
|
32
|
+
[](https://github.com/randyy179/qlib-sync/actions/workflows/ci.yml)
|
|
33
|
+
[](https://pypi.org/project/qlib-sync/)
|
|
34
|
+
[](https://github.com/randyy179/qlib-sync/blob/main/LICENSE)
|
|
35
|
+
|
|
36
|
+
[中文说明](https://github.com/randyy179/qlib-sync/blob/main/README.zh-CN.md)
|
|
37
|
+
|
|
38
|
+
**Website:** [randyy179.github.io/qlib-sync](https://randyy179.github.io/qlib-sync/)
|
|
39
|
+
|
|
40
|
+
<!-- mcp-name: io.github.randyy179/qlib-sync -->
|
|
41
|
+
|
|
42
|
+
**One command to install and keep current a local [Qlib](https://github.com/microsoft/qlib) China A-share dataset, and an MCP server so AI assistants can read it.**
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pip install qlib-sync
|
|
46
|
+
qlib-sync cn update
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Or without installing anything first: `uvx qlib-sync cn update`.
|
|
50
|
+
|
|
51
|
+

|
|
52
|
+
|
|
53
|
+
That installs the daily community release from
|
|
54
|
+
[chenditc/investment_data](https://github.com/chenditc/investment_data) into
|
|
55
|
+
`~/.qlib/qlib_data/cn_data`, the location Qlib reads by default. Run the same command again
|
|
56
|
+
any time; when the data is already current it finishes in about a second.
|
|
57
|
+
|
|
58
|
+
## Why not just `wget` and `tar`?
|
|
59
|
+
|
|
60
|
+
<img src="https://raw.githubusercontent.com/randyy179/qlib-sync/main/docs/assets/comic.jpg" width="560" alt="Four-panel comic: a download dies at 99% at night; the next day 550 MB arrive and nothing changed; the qlib-sync robot checks the data and stamps it verified, keeping the old copy safe; the researcher sips tea and asks an AI for Moutai's recent closes">
|
|
61
|
+
|
|
62
|
+
Qlib's README points A-share users to the community release with a `wget` and `tar` one-liner.
|
|
63
|
+
That works until it doesn't:
|
|
64
|
+
|
|
65
|
+
- **An interrupted or truncated download** extracts into a half-written dataset, or fails
|
|
66
|
+
after the old one has already been overwritten.
|
|
67
|
+
- **Upstream republishes releases.** In November 2026 a data error was found and corrected
|
|
68
|
+
([investment_data#163](https://github.com/chenditc/investment_data/issues/163)) and releases
|
|
69
|
+
were republished under the same date. A date check alone never sees the fix.
|
|
70
|
+
- **Every check downloads 550 MB**, even when nothing changed.
|
|
71
|
+
|
|
72
|
+
The investment_data README itself describes a careful manual process: download the release
|
|
73
|
+
manifest, check the archive with its `validate_archive.py`, then extract. qlib-sync runs the
|
|
74
|
+
same checks automatically on every update and adds what a manual process cannot: it skips the
|
|
75
|
+
download when nothing changed, extracts beside the live data and swaps it in with rollback
|
|
76
|
+
(rather than extracting on top of the existing folder), and picks up republished releases.
|
|
77
|
+
|
|
78
|
+
qlib-sync treats the dataset like a production system:
|
|
79
|
+
|
|
80
|
+
<picture>
|
|
81
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/randyy179/qlib-sync/main/docs/assets/update-flow-dark.svg">
|
|
82
|
+
<img alt="Update flow: check the manifest, download, verify, stage, swap; any failure puts the previous dataset back" src="https://raw.githubusercontent.com/randyy179/qlib-sync/main/docs/assets/update-flow-light.svg">
|
|
83
|
+
</picture>
|
|
84
|
+
|
|
85
|
+
| Step | What happens |
|
|
86
|
+
|---|---|
|
|
87
|
+
| Check | Reads the release's 500-byte manifest. Already current: done, nothing downloaded. |
|
|
88
|
+
| Download | Resumes interrupted downloads; shows progress; optional mirror for slow links to GitHub. |
|
|
89
|
+
| Verify | Size and sha256 from the manifest; the same release checks as upstream's `validate_archive.py` (every index pool and the future calendar reach the manifest's dates, release marked publishable); archive paths and links are checked before extraction. |
|
|
90
|
+
| Stage | Extracts next to the active dataset; refuses downgrades. |
|
|
91
|
+
| Swap | Renames the new dataset into place, then runs a real Qlib read. |
|
|
92
|
+
| Roll back | Any failure after the swap moves the previous dataset back. |
|
|
93
|
+
| Report | New trading days, newly listed and delisted symbols. |
|
|
94
|
+
|
|
95
|
+
The checks catch broken, truncated, or tampered packages. They cannot catch wrong values inside a
|
|
96
|
+
correctly published release (like the one in #163); for that, keep a rollback copy with
|
|
97
|
+
`--keep-backup`.
|
|
98
|
+
|
|
99
|
+
## Commands
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
qlib-sync status # every local Qlib dataset at a glance
|
|
103
|
+
qlib-sync cn status # local date, pools, and whether a newer release exists
|
|
104
|
+
qlib-sync cn update # install or update (add --json for machine-readable output)
|
|
105
|
+
qlib-sync cn health-check # structure checks plus a real Qlib read
|
|
106
|
+
qlib-sync cn restore --list # rollback backups; restore --yes switches back
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Useful options for `cn update`:
|
|
110
|
+
|
|
111
|
+
| Option | Effect |
|
|
112
|
+
|---|---|
|
|
113
|
+
| `--data-dir PATH` | Install somewhere other than `~/.qlib/qlib_data/cn_data` |
|
|
114
|
+
| `--mirror URL` | Download through a prefix-style GitHub proxy such as `https://ghfast.top` (or set `QLIB_SYNC_MIRROR`) |
|
|
115
|
+
| `--keep-backup` | Keep the previous dataset for one rollback |
|
|
116
|
+
| `--protect FILE` | Carry a custom stock pool in `instruments/` across updates (repeatable) |
|
|
117
|
+
| `--force` | Reinstall even when the dates match |
|
|
118
|
+
|
|
119
|
+
**The update replaces the whole `cn_data` folder.** Anything you added there is lost unless you
|
|
120
|
+
pass it with `--protect`.
|
|
121
|
+
|
|
122
|
+
### Mirrors
|
|
123
|
+
|
|
124
|
+
From mainland China, GitHub downloads can be slow or fail. Pass a mirror and qlib-sync downloads
|
|
125
|
+
the package through it while still reading the manifest from GitHub directly when it can, so the
|
|
126
|
+
checksum does not depend on the mirror. Mirrors are third-party services; qlib-sync does not
|
|
127
|
+
endorse any of them.
|
|
128
|
+
|
|
129
|
+
## Python
|
|
130
|
+
|
|
131
|
+
```python
|
|
132
|
+
from qlib_sync import update_cn, status_cn, health_check_cn
|
|
133
|
+
|
|
134
|
+
result = update_cn() # raises qlib_sync.UpdateError on failure; data unchanged
|
|
135
|
+
print("\n".join(result.summary_lines()))
|
|
136
|
+
print(result.new_trading_days, result.added_symbols)
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
## Use it from your AI assistant
|
|
140
|
+
|
|
141
|
+
qlib-sync includes an [MCP](https://modelcontextprotocol.io) server, the open standard most AI
|
|
142
|
+
tools use to call local tools. Any MCP client that can start a local command works, whichever
|
|
143
|
+
model it runs: Claude, GPT, Gemini, DeepSeek, Qwen, Kimi, GLM, or a local model. The server
|
|
144
|
+
runs on your machine and reads the dataset there; there is no service to host and nothing
|
|
145
|
+
leaves your computer. It is listed in the [MCP Registry](https://registry.modelcontextprotocol.io)
|
|
146
|
+
as `io.github.randyy179/qlib-sync`.
|
|
147
|
+
|
|
148
|
+
Every client needs the same two things: the command `uvx` and the arguments `qlib-sync mcp`.
|
|
149
|
+
|
|
150
|
+
| Client | How to add qlib-sync |
|
|
151
|
+
|---|---|
|
|
152
|
+
| Claude Desktop | Download `qlib-sync.mcpb` from the [latest release](https://github.com/randyy179/qlib-sync/releases/latest) and open it |
|
|
153
|
+
| Claude Code | `claude mcp add qlib-sync -- uvx qlib-sync mcp` |
|
|
154
|
+
| OpenAI Codex CLI | `codex mcp add qlib-sync -- uvx qlib-sync mcp` |
|
|
155
|
+
| Cursor | The JSON below, in `~/.cursor/mcp.json` |
|
|
156
|
+
| Gemini CLI | The JSON below, in `~/.gemini/settings.json` |
|
|
157
|
+
| Qwen Code | The JSON below, in `~/.qwen/settings.json` |
|
|
158
|
+
| Tencent CodeBuddy, ByteDance Trae, Alibaba Tongyi Lingma | MCP settings, add a server manually, paste the JSON below |
|
|
159
|
+
| Cherry Studio | Settings, MCP servers, add: type STDIO, command `uvx`, arguments `qlib-sync` and `mcp` on separate lines |
|
|
160
|
+
| VS Code (Copilot agent mode) | `.vscode/mcp.json`: `{ "servers": { "qlib-sync": { "type": "stdio", "command": "uvx", "args": ["qlib-sync", "mcp"] } } }` |
|
|
161
|
+
|
|
162
|
+
```json
|
|
163
|
+
{ "mcpServers": { "qlib-sync": { "command": "uvx", "args": ["qlib-sync", "mcp"] } } }
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
Menu names and file locations change between versions; check your client's MCP documentation
|
|
167
|
+
if a path above has moved. Tested with Claude Code and OpenAI Codex CLI.
|
|
168
|
+
|
|
169
|
+
Web chat apps such as ChatGPT connect only to remote MCP servers over the internet, so they
|
|
170
|
+
cannot start this local server.
|
|
171
|
+
|
|
172
|
+
Set `QLIB_SYNC_DATA_DIR` to use a dataset outside `~/.qlib/qlib_data/cn_data`, and
|
|
173
|
+
`QLIB_SYNC_MIRROR` to download through a mirror.
|
|
174
|
+
|
|
175
|
+
| Tool | Does |
|
|
176
|
+
|---|---|
|
|
177
|
+
| `cn_status` | Local latest date, pools, and whether an update is available |
|
|
178
|
+
| `cn_update` | Install or update (safe to call: failures restore the previous data) |
|
|
179
|
+
| `cn_health_check` | Structure checks plus a real Qlib read |
|
|
180
|
+
| `cn_get_prices` | Daily bars for up to 20 symbols; real traded prices by default, adjusted on request |
|
|
181
|
+
| `cn_list_pool` | Members of csi300, csi500, and other pools on a date |
|
|
182
|
+
|
|
183
|
+
Then ask things like *"Is my A-share data current? What did Moutai close at over the last
|
|
184
|
+
week?"* Symbols accept `600519`, `SH600519`, or `600519.SH`.
|
|
185
|
+
|
|
186
|
+

|
|
187
|
+
|
|
188
|
+
[`skills/qlib-sync/SKILL.md`](https://github.com/randyy179/qlib-sync/blob/main/skills/qlib-sync/SKILL.md) teaches an assistant when to update,
|
|
189
|
+
how to read each failure, and the price conventions. It uses the open `SKILL.md` format that
|
|
190
|
+
Claude, Codex, and other agents load from their skills folder; any assistant can also simply
|
|
191
|
+
be pointed at the file.
|
|
192
|
+
|
|
193
|
+
## Daily updates
|
|
194
|
+
|
|
195
|
+
A run that finds nothing new costs one small request, so schedule it daily. See
|
|
196
|
+
[docs/scheduling.md](https://github.com/randyy179/qlib-sync/blob/main/docs/scheduling.md) for cron and launchd examples.
|
|
197
|
+
|
|
198
|
+
## Data and credits
|
|
199
|
+
|
|
200
|
+
All data comes from [chenditc/investment_data](https://github.com/chenditc/investment_data)
|
|
201
|
+
(Apache-2.0). qlib-sync only downloads the public release; it does not host or redistribute
|
|
202
|
+
data. Prices follow Qlib's convention: `$close` is adjusted and the real price is
|
|
203
|
+
`$close / $factor`.
|
|
204
|
+
|
|
205
|
+
qlib-sync is an independent project, not affiliated with Microsoft or the Qlib team.
|
|
206
|
+
|
|
207
|
+
Supported: macOS and Linux, Python 3.10 to 3.12 (pyqlib does not publish wheels for newer
|
|
208
|
+
versions yet; `uvx` picks a compatible Python automatically).
|
|
209
|
+
|
|
210
|
+
## Development
|
|
211
|
+
|
|
212
|
+
```bash
|
|
213
|
+
uv venv && uv pip install -e ".[dev]"
|
|
214
|
+
.venv/bin/python -m pytest
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
## License
|
|
218
|
+
|
|
219
|
+
MIT
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
qlib_sync/__init__.py,sha256=u_sxeWDHrbhaOepjl9AxrRzYMTkVP0xQVOxsJ6l5T3o,463
|
|
2
|
+
qlib_sync/api.py,sha256=E-lkRm6Lp9d2i7qDoZvEuojo7tFN8mhJBQlRb37Zuyg,2126
|
|
3
|
+
qlib_sync/cli.py,sha256=oia8VSrnAEn2jDMSb4PrMwhgmGC5nrp0_a-5RhnmrmE,2798
|
|
4
|
+
qlib_sync/mcp_server.py,sha256=yehmniP5qJ97Hk0MOf6I35dzyXDDZt6gPI5SWJNay-0,11301
|
|
5
|
+
qlib_sync/txn.py,sha256=FzkhuuTkYH29MkjQGyeSIc3MvA5djlZCInkDruLRvpE,2437
|
|
6
|
+
qlib_sync/cn/__init__.py,sha256=xzDCR9NFrXYKL_NuPpzYLQs6DKodtuwudqxk5KTqPj8,46
|
|
7
|
+
qlib_sync/cn/market.py,sha256=2w2rPorJ0AUaD_vHqva400DOnSNHmw2fkj8dPgMYf1E,15630
|
|
8
|
+
qlib_sync/cn/update.py,sha256=cpG1GohhEDyRno5uzl07u8qw_1OuS0JGFzsDQ1Yp61M,27113
|
|
9
|
+
qlib_sync-0.1.0.dist-info/METADATA,sha256=ZtIJSEZep_gc0tLjvgjmg-NBQKc9lbe1hcwML7-egMw,11193
|
|
10
|
+
qlib_sync-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
11
|
+
qlib_sync-0.1.0.dist-info/entry_points.txt,sha256=VRZjrWdLOfOJoIEdpdzfYh6Jitg_CZCp3YEh5oWlE1Q,49
|
|
12
|
+
qlib_sync-0.1.0.dist-info/licenses/LICENSE,sha256=GiKNiTcrDIfdccuBMJKFy9Ovr5COxDM4tDggWmZL-5s,1066
|
|
13
|
+
qlib_sync-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 randyy179
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|