ohlcvault 0.2.1__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/PKG-INFO +12 -1
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/README.md +11 -0
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/ohlcvault/client.py +13 -1
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/ohlcvault/config.py +9 -6
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/pyproject.toml +1 -1
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/tests/conftest.py +34 -3
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/.gitignore +0 -0
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/LICENSE +0 -0
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/SPEC.md +0 -0
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/ohlcvault/__init__.py +0 -0
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/ohlcvault/adjust.py +0 -0
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/ohlcvault/codes.py +0 -0
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/ohlcvault/errors.py +0 -0
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/ohlcvault/net.py +0 -0
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/ohlcvault/sql.py +0 -0
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/ohlcvault/store.py +0 -0
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/schema/example-calendar.json +0 -0
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/schema/example-idx-monthly.json +0 -0
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/schema/example-latest.json +0 -0
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/schema/example-manifest.json +0 -0
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/schema/example-monthly.json +0 -0
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/schema/example-snapshot.json +0 -0
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/schema/example-symbols.json +0 -0
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/tests/test_adjust.py +0 -0
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/tests/test_client.py +0 -0
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/tests/test_perf.py +0 -0
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/tests/test_store.py +0 -0
- {ohlcvault-0.2.1 → ohlcvault-0.2.2}/tests/test_symbols.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: ohlcvault
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Reproducible, checksummed daily OHLCV data for A-share, HK and US markets — static files, content-addressed snapshots, zero API keys.
|
|
5
5
|
Project-URL: Homepage, https://helpquant.com
|
|
6
6
|
Project-URL: Documentation, https://github.com/helpquant/ohlcvault#readme
|
|
@@ -132,6 +132,17 @@ A mirror can be an HTTP(S) URL **or a local directory**:
|
|
|
132
132
|
ov.connect(mirrors=["/path/to/data"])
|
|
133
133
|
```
|
|
134
134
|
|
|
135
|
+
The built-in chain has two entries whose **failure domains are disjoint** — a
|
|
136
|
+
commercial CDN and a self-hosted origin:
|
|
137
|
+
|
|
138
|
+
| Mirror | Role | Independent failure domain |
|
|
139
|
+
|---|---|---|
|
|
140
|
+
| `https://data.ohlcvault.com` | primary | commercial CDN: account suspension, billing, regional outage |
|
|
141
|
+
| `https://data.helpquant.com` | fallback | own server + own DNS |
|
|
142
|
+
|
|
143
|
+
Override with `OHLCVAULT_MIRRORS=https://a,https://b`, or pass `mirrors=` / set
|
|
144
|
+
`OHLCVAULT_CACHE` to keep everything local.
|
|
145
|
+
|
|
135
146
|
Mirrors are tried in order; whichever one succeeds is promoted to first place. Every
|
|
136
147
|
file is checked against the `sha256` in the snapshot manifest, and anything that
|
|
137
148
|
fails is **discarded and the next mirror is tried** — bad bytes are never handed to
|
|
@@ -100,6 +100,17 @@ A mirror can be an HTTP(S) URL **or a local directory**:
|
|
|
100
100
|
ov.connect(mirrors=["/path/to/data"])
|
|
101
101
|
```
|
|
102
102
|
|
|
103
|
+
The built-in chain has two entries whose **failure domains are disjoint** — a
|
|
104
|
+
commercial CDN and a self-hosted origin:
|
|
105
|
+
|
|
106
|
+
| Mirror | Role | Independent failure domain |
|
|
107
|
+
|---|---|---|
|
|
108
|
+
| `https://data.ohlcvault.com` | primary | commercial CDN: account suspension, billing, regional outage |
|
|
109
|
+
| `https://data.helpquant.com` | fallback | own server + own DNS |
|
|
110
|
+
|
|
111
|
+
Override with `OHLCVAULT_MIRRORS=https://a,https://b`, or pass `mirrors=` / set
|
|
112
|
+
`OHLCVAULT_CACHE` to keep everything local.
|
|
113
|
+
|
|
103
114
|
Mirrors are tried in order; whichever one succeeds is promoted to first place. Every
|
|
104
115
|
file is checked against the `sha256` in the snapshot manifest, and anything that
|
|
105
116
|
fails is **discarded and the next mirror is tried** — bad bytes are never handed to
|
|
@@ -71,7 +71,19 @@ class MarketClient:
|
|
|
71
71
|
|
|
72
72
|
@staticmethod
|
|
73
73
|
def _label(m: str) -> str:
|
|
74
|
-
|
|
74
|
+
"""镜像的短标签,用于 trace 与 `stats["served_by"]`。
|
|
75
|
+
|
|
76
|
+
必须对**裸主机 URL** 也给出可区分的名字:默认镜像链全是
|
|
77
|
+
`https://<host>` 这种无路径形态,而按「最后一个路径段」取名会得到
|
|
78
|
+
空串 —— 于是两个不同的镜像在统计里混成同一个 key,
|
|
79
|
+
「到底命中了主源还是备用源」这个问题就再也答不出来了(实测踩到)。
|
|
80
|
+
"""
|
|
81
|
+
s = str(m).rstrip("/")
|
|
82
|
+
if "://" in s:
|
|
83
|
+
rest = s.split("://", 1)[1]
|
|
84
|
+
host, _, path = rest.partition("/")
|
|
85
|
+
return f"{host}/{path}" if path else host
|
|
86
|
+
return Path(s).name
|
|
75
87
|
|
|
76
88
|
def _fetch_raw(self, m: str, rel: str) -> bytes:
|
|
77
89
|
if m.startswith("http"):
|
|
@@ -5,9 +5,13 @@
|
|
|
5
5
|
|
|
6
6
|
| 镜像 | 角色 | 故障域 |
|
|
7
7
|
|---|---|---|
|
|
8
|
-
| Cloudflare R2 + CDN | 主源 | 商业 CDN:账号封禁、账单异常、区域故障 |
|
|
9
|
-
|
|
|
10
|
-
|
|
8
|
+
| `data.ohlcvault.com`(Cloudflare R2 + CDN) | 主源 | 商业 CDN:账号封禁、账单异常、区域故障 |
|
|
9
|
+
| `data.helpquant.com`(自有服务器 HTTPS) | 备用 1 | 自有机器 + 自有 DNS(DNSPod):唯一完全可控的兜底 |
|
|
10
|
+
|
|
11
|
+
历史上这里还有第三个源(`cdn.jsdelivr.net/gh/…` 与 `raw.githubusercontent.com/…`),
|
|
12
|
+
指向数据镜像仓库 `helpquant/ohlcvault-data`。该仓库经 GitHub API 查证 **404,从未
|
|
13
|
+
创建**,两个 URL 都是死链,只会让客户端在冷启动时白付一次超时 —— 已移除。
|
|
14
|
+
日后若真建了 GitHub 镜像仓库,再把它们加回来即可(届时故障域才是真正独立的)。
|
|
11
15
|
|
|
12
16
|
客户端会**记住哪个镜像成功过**并把它提到首位,所以镜像顺序不是性能关键,
|
|
13
17
|
减少的是「每次请求都先付一次失败超时」。
|
|
@@ -33,14 +37,13 @@ NAMESPACES = ("stock", "index")
|
|
|
33
37
|
|
|
34
38
|
# HTTP User-Agent。必须带自定义 UA:Cloudflare 等边缘节点默认拦截
|
|
35
39
|
# Python-urllib/* 的请求(403),裸 urllib 会被当成 bot。
|
|
36
|
-
VERSION = "0.2.
|
|
40
|
+
VERSION = "0.2.2"
|
|
37
41
|
USER_AGENT = f"ohlcvault/{VERSION} (+https://github.com/helpquant/ohlcvault)"
|
|
38
42
|
|
|
39
43
|
# 默认镜像链。可用 OHLCVAULT_MIRRORS 覆盖。
|
|
40
44
|
DEFAULT_MIRRORS: list[str] = [
|
|
41
45
|
"https://data.ohlcvault.com",
|
|
42
|
-
"https://
|
|
43
|
-
"https://raw.githubusercontent.com/helpquant/ohlcvault-data/main",
|
|
46
|
+
"https://data.helpquant.com",
|
|
44
47
|
]
|
|
45
48
|
|
|
46
49
|
DEFAULT_CACHE = Path(
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "ohlcvault"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.2"
|
|
8
8
|
description = "Reproducible, checksummed daily OHLCV data for A-share, HK and US markets — static files, content-addressed snapshots, zero API keys."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -52,9 +52,39 @@ def store(data_dir, tmp_path):
|
|
|
52
52
|
return ov.connect(mirrors=[str(data_dir)], cache_dir=tmp_path / "cache", quiet=True)
|
|
53
53
|
|
|
54
54
|
|
|
55
|
+
def _copy_tree_cow(src: Path, dst: Path) -> None:
|
|
56
|
+
"""复制目录树,尽可能用「写时复制」(APFS clonefile / reflink)。
|
|
57
|
+
|
|
58
|
+
mirrors 夹具要为**每个**测试准备多份数据副本;真实数据集在 GB 量级,
|
|
59
|
+
逐字节复制三遍就是数个 GB —— 会直接吃光开发机磁盘(实测测试进程被
|
|
60
|
+
SIGKILL 打断,就是因为这里)。COW 副本几乎不额外占空间,而且**写入时
|
|
61
|
+
自动分离**,所以「篡改某个副本」的测试语义完全成立(硬链接做不到:
|
|
62
|
+
原地改写会串到其他副本)。平台不支持时回退到普通复制。
|
|
63
|
+
"""
|
|
64
|
+
import subprocess
|
|
65
|
+
|
|
66
|
+
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
67
|
+
if sys.platform == "darwin":
|
|
68
|
+
cmd = ["cp", "-Rc"]
|
|
69
|
+
elif sys.platform.startswith("linux"):
|
|
70
|
+
cmd = ["cp", "-a", "--reflink=auto"]
|
|
71
|
+
else:
|
|
72
|
+
cmd = []
|
|
73
|
+
if cmd:
|
|
74
|
+
r = subprocess.run(cmd + [str(src), str(dst)],
|
|
75
|
+
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
76
|
+
if r.returncode == 0:
|
|
77
|
+
return
|
|
78
|
+
shutil.copytree(src, dst, dirs_exist_ok=True)
|
|
79
|
+
|
|
80
|
+
|
|
55
81
|
@pytest.fixture()
|
|
56
82
|
def mirrors(data_dir, tmp_path):
|
|
57
|
-
"""
|
|
83
|
+
"""三份独立副本,模拟「主源 + 两个备用源」的镜像链。
|
|
84
|
+
|
|
85
|
+
这里的三份是**占位符**(名字沿用历史,不代表当前真实镜像链 —— 真实链见
|
|
86
|
+
`ohlcvault.config.DEFAULT_MIRRORS`,2026-10-09 起只有两个源)。保留三份是为了
|
|
87
|
+
让「N 路降级」这条逻辑本身被测到,而不是绑死在当前有几个源上。
|
|
58
88
|
|
|
59
89
|
用真实字节而不是合成数据 —— 校验、降级、篡改拒绝这些路径只有在
|
|
60
90
|
真实的 gzip 分片上跑才有意义。
|
|
@@ -62,8 +92,9 @@ def mirrors(data_dir, tmp_path):
|
|
|
62
92
|
base = tmp_path / "remote"
|
|
63
93
|
names = ["cloudflare_r2", "jsdelivr", "github_raw"]
|
|
64
94
|
for n in names:
|
|
65
|
-
|
|
66
|
-
|
|
95
|
+
_copy_tree_cow(data_dir, base / n)
|
|
96
|
+
for skip in (".cache", ".months"):
|
|
97
|
+
shutil.rmtree(base / n / skip, ignore_errors=True)
|
|
67
98
|
return [str(base / n) for n in names], base
|
|
68
99
|
|
|
69
100
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|