ashare-data-immunity 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ashare_data_immunity-0.1.1/LICENSE +21 -0
- ashare_data_immunity-0.1.1/PKG-INFO +166 -0
- ashare_data_immunity-0.1.1/README.md +146 -0
- ashare_data_immunity-0.1.1/pyproject.toml +34 -0
- ashare_data_immunity-0.1.1/setup.cfg +4 -0
- ashare_data_immunity-0.1.1/src/ashare_data_immunity/__init__.py +36 -0
- ashare_data_immunity-0.1.1/src/ashare_data_immunity/__main__.py +4 -0
- ashare_data_immunity-0.1.1/src/ashare_data_immunity/audit.py +191 -0
- ashare_data_immunity-0.1.1/src/ashare_data_immunity/cleaning.py +113 -0
- ashare_data_immunity-0.1.1/src/ashare_data_immunity/cli.py +152 -0
- ashare_data_immunity-0.1.1/src/ashare_data_immunity/limits.py +147 -0
- ashare_data_immunity-0.1.1/src/ashare_data_immunity/snapshot.py +89 -0
- ashare_data_immunity-0.1.1/src/ashare_data_immunity.egg-info/PKG-INFO +166 -0
- ashare_data_immunity-0.1.1/src/ashare_data_immunity.egg-info/SOURCES.txt +18 -0
- ashare_data_immunity-0.1.1/src/ashare_data_immunity.egg-info/dependency_links.txt +1 -0
- ashare_data_immunity-0.1.1/src/ashare_data_immunity.egg-info/entry_points.txt +3 -0
- ashare_data_immunity-0.1.1/src/ashare_data_immunity.egg-info/top_level.txt +1 -0
- ashare_data_immunity-0.1.1/tests/test_audit_snapshot.py +116 -0
- ashare_data_immunity-0.1.1/tests/test_cleaning.py +56 -0
- ashare_data_immunity-0.1.1/tests/test_limits.py +115 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ashare Data Immunity contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ashare-data-immunity
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Data immunity for A-share daily bars: cleaning, board-aware price-limit and suspension detection, quality audit and snapshot versioning.
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Keywords: a-share,data-quality,price-limit,suspension,cleaning,snapshot,quant
|
|
7
|
+
Classifier: Development Status :: 3 - Alpha
|
|
8
|
+
Classifier: Environment :: Console
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Intended Audience :: Financial and Insurance Industry
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Topic :: Office/Business :: Financial :: Investment
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# ashare-data-immunity
|
|
22
|
+
|
|
23
|
+
**Data immunity for A-share daily bars**: cleaning (NaN / OHLCV
|
|
24
|
+
validation), board-aware price-limit and suspension detection, quality
|
|
25
|
+
audit (listing / coverage / continuity) and snapshot versioning (sha256
|
|
26
|
+
manifests). Python 3.11+, **zero dependencies**, Windows / Linux / macOS.
|
|
27
|
+
|
|
28
|
+
**Status:** v0.1 鈥?alpha. The audit structure is distilled from a
|
|
29
|
+
production A-share pipeline; board rules follow the current exchange
|
|
30
|
+
conventions and should be re-checked against the exchanges' rule
|
|
31
|
+
documents before you rely on them.
|
|
32
|
+
|
|
33
|
+
## Why this exists
|
|
34
|
+
|
|
35
|
+
A-share daily data is not born clean. Vendors ship NaN closes, negative
|
|
36
|
+
opens, volume in lots or shares depending on the board, silent suspensions
|
|
37
|
+
that look like flat prices, and limit-up days that look like "huge moves"
|
|
38
|
+
unless you know the board's 10/20/30% rule. Every one of these corrupts a
|
|
39
|
+
factor pipeline differently, and most corrupt it *quietly*.
|
|
40
|
+
|
|
41
|
+
`ashare-data-immunity` is the immune system: it does not fetch data and it
|
|
42
|
+
does not trade 鈥?it makes the data you already have **honest**:
|
|
43
|
+
|
|
44
|
+
- **clean** 鈥?flag or sanitize non-finite values, non-positive prices,
|
|
45
|
+
OHLC inconsistencies (high below max(open, close), low above
|
|
46
|
+
min(open, close)), negative volume;
|
|
47
|
+
- **limits** 鈥?board-aware price-limit detection (main 卤10%, STAR and
|
|
48
|
+
ChiNext 卤20%, BSE 卤30%, ST 卤5% on the main board) against the previous
|
|
49
|
+
close with tick rounding tolerance, plus a documented suspension
|
|
50
|
+
heuristic (zero volume, or no prices on a dated row);
|
|
51
|
+
- **audit** 鈥?daily quality audit: are watched codes still listed, does
|
|
52
|
+
history coverage meet the threshold, are there calendar gaps? All data
|
|
53
|
+
sources injectable, results append-only;
|
|
54
|
+
- **snapshot** 鈥?sha256 manifests with cutoffs, so "which data did this
|
|
55
|
+
backtest actually see" is a file you can compare and prove.
|
|
56
|
+
|
|
57
|
+
## Philosophy
|
|
58
|
+
|
|
59
|
+
**Data is an asset; immunity is a discipline.**
|
|
60
|
+
|
|
61
|
+
Most data tooling optimizes for *getting* data. This tool optimizes for
|
|
62
|
+
*trusting* the data you have 鈥?and it refuses to guess: board rules are
|
|
63
|
+
explicit tables, the suspension detector is documented as a heuristic
|
|
64
|
+
(vendor conventions differ), and the audit reports "universe unavailable"
|
|
65
|
+
instead of pretending the listing check ran. Read-only by design; every
|
|
66
|
+
function either returns a report or writes an append-only record. Nothing
|
|
67
|
+
here trades, prices, or decides.
|
|
68
|
+
|
|
69
|
+
## Quick start
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
# install from PyPI (once published)
|
|
73
|
+
pip install ashare-data-immunity
|
|
74
|
+
|
|
75
|
+
# or run without installing anything:
|
|
76
|
+
# PYTHONPATH=src python -m ashare_data_immunity --help
|
|
77
|
+
|
|
78
|
+
python examples/demo.py # clean + limits + audit + snapshot on synthetic data
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Your own data:
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
# 1. validate / clean a bars file
|
|
85
|
+
imm clean --bars bars.json # exit 1 when problems found
|
|
86
|
+
imm clean --bars bars.json --drop-non-positive --out clean.json
|
|
87
|
+
|
|
88
|
+
# 2. board-aware limits + suspensions
|
|
89
|
+
imm limits --bars bars.json --code 600000
|
|
90
|
+
imm limits --bars bars.json --code 688001 --st
|
|
91
|
+
|
|
92
|
+
# 3. daily quality audit (watchlist + history dir + append-only audit dir)
|
|
93
|
+
imm audit --watchlist watchlist.json \
|
|
94
|
+
--history-root data/daily --audit-root data/audits
|
|
95
|
+
|
|
96
|
+
# 4. snapshot versioning
|
|
97
|
+
imm snapshot --name v2026-08-01 --cutoff 2026-08-01 \
|
|
98
|
+
--files data/daily/*.json --root data --out manifests/v1.json
|
|
99
|
+
imm snapshot-compare --before manifests/v1.json --after manifests/v2.json
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
## Commands
|
|
103
|
+
|
|
104
|
+
| Command | What it does |
|
|
105
|
+
| --- | --- |
|
|
106
|
+
| `clean` | Validate bars (missing/non-finite/non-positive fields, OHLC consistency, negative volume); optionally sanitize (non-finite and non-positive prices 鈫?`None`, volume kept 鈮?0) and optionally drop non-positive rows |
|
|
107
|
+
| `limits` | Board classification, price-limit events (up/down with ratio and limit price) and suspension days for one code |
|
|
108
|
+
| `audit` | Listing (codes not in the injected universe), history coverage, calendar continuity; appends a JSONL record per day |
|
|
109
|
+
| `snapshot` | sha256 manifest of a file list with name + cutoff |
|
|
110
|
+
| `snapshot-compare` | added / removed / changed files between two manifests |
|
|
111
|
+
| `version` | Print version |
|
|
112
|
+
|
|
113
|
+
## Board rules (v0.1)
|
|
114
|
+
|
|
115
|
+
| Board | Prefixes | Limit |
|
|
116
|
+
| --- | --- | --- |
|
|
117
|
+
| main | 60xxxx / 00xxxx | 卤10% (ST: 卤5%) |
|
|
118
|
+
| STAR | 688 / 689 | 卤20% |
|
|
119
|
+
| ChiNext | 300 / 301 | 卤20% |
|
|
120
|
+
| Beijing SE | 43x / 83x / 87x / 920 | 卤30% |
|
|
121
|
+
| unknown | 鈥?| 卤10% (assumed main) |
|
|
122
|
+
|
|
123
|
+
Limit detection compares `close` against `round(prev_close 脳 (1 卤 ratio), 2)`
|
|
124
|
+
with a default tolerance of 0.001 for vendor rounding conventions. The
|
|
125
|
+
first bar has no reference and is never flagged. **Verify the tables
|
|
126
|
+
against the current exchange rule documents before production use** 鈥?the
|
|
127
|
+
tool's job is to make the rules explicit, not to invent them.
|
|
128
|
+
|
|
129
|
+
Suspension heuristic: a dated row with zero volume, or with no prices at
|
|
130
|
+
all, is a suspension day. Documented, not hidden 鈥?and toggleable
|
|
131
|
+
(`zero_volume_means_suspended`).
|
|
132
|
+
|
|
133
|
+
## Development
|
|
134
|
+
|
|
135
|
+
```bash
|
|
136
|
+
python -m pip install -e . pytest
|
|
137
|
+
python -m pytest
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
CI runs the full test suite on Ubuntu, Windows and macOS with Python 3.11
|
|
141
|
+
and 3.12. Issues are handled on weekends; pull requests are welcome.
|
|
142
|
+
|
|
143
|
+
## Related work
|
|
144
|
+
|
|
145
|
+
This tool makes no claims to novelty of its own: it is the engineering
|
|
146
|
+
layer under the data-quality principles that the industry is converging on
|
|
147
|
+
鈥?point-in-time discipline ([Kelly et al., NBER w35247](https://www.nber.org/papers/w35247)),
|
|
148
|
+
look-ahead awareness ([Fonseca 2026, arXiv:2607.04958](https://econpapers.repec.org/paper/arxpapers/2607.04958.htm))
|
|
149
|
+
and reproducible snapshots. The pieces that *are* worth citing live in the
|
|
150
|
+
sibling repos of this project family; this one just keeps the data honest.
|
|
151
|
+
|
|
152
|
+
## Project family
|
|
153
|
+
|
|
154
|
+
Part of [Foolproof Labs](https://github.com/foolproof-labs) — a toolchain
|
|
155
|
+
against self-deception in quantitative research:
|
|
156
|
+
|
|
157
|
+
- [pit-adjuster](https://github.com/foolproof-labs/pit-adjuster) — PIT back-adjustment with static forward-adjustment drift detection
|
|
158
|
+
- [falsification-ledger](https://github.com/foolproof-labs/falsification-ledger) — pre-registration and falsification ledger
|
|
159
|
+
- [factor-qc](https://github.com/foolproof-labs/factor-qc) — fail-closed backtest quality gate
|
|
160
|
+
- [lesson-book](https://github.com/foolproof-labs/lesson-book) — tuition memory for traders
|
|
161
|
+
- [lookahead-free](https://github.com/foolproof-labs/lookahead-free) — verifiable look-ahead-freedom checks
|
|
162
|
+
- [ashare-data-immunity](https://github.com/foolproof-labs/ashare-data-immunity) — data immunity for A-share daily bars
|
|
163
|
+
|
|
164
|
+
## License
|
|
165
|
+
|
|
166
|
+
MIT
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
# ashare-data-immunity
|
|
2
|
+
|
|
3
|
+
**Data immunity for A-share daily bars**: cleaning (NaN / OHLCV
|
|
4
|
+
validation), board-aware price-limit and suspension detection, quality
|
|
5
|
+
audit (listing / coverage / continuity) and snapshot versioning (sha256
|
|
6
|
+
manifests). Python 3.11+, **zero dependencies**, Windows / Linux / macOS.
|
|
7
|
+
|
|
8
|
+
**Status:** v0.1 鈥?alpha. The audit structure is distilled from a
|
|
9
|
+
production A-share pipeline; board rules follow the current exchange
|
|
10
|
+
conventions and should be re-checked against the exchanges' rule
|
|
11
|
+
documents before you rely on them.
|
|
12
|
+
|
|
13
|
+
## Why this exists
|
|
14
|
+
|
|
15
|
+
A-share daily data is not born clean. Vendors ship NaN closes, negative
|
|
16
|
+
opens, volume in lots or shares depending on the board, silent suspensions
|
|
17
|
+
that look like flat prices, and limit-up days that look like "huge moves"
|
|
18
|
+
unless you know the board's 10/20/30% rule. Every one of these corrupts a
|
|
19
|
+
factor pipeline differently, and most corrupt it *quietly*.
|
|
20
|
+
|
|
21
|
+
`ashare-data-immunity` is the immune system: it does not fetch data and it
|
|
22
|
+
does not trade 鈥?it makes the data you already have **honest**:
|
|
23
|
+
|
|
24
|
+
- **clean** 鈥?flag or sanitize non-finite values, non-positive prices,
|
|
25
|
+
OHLC inconsistencies (high below max(open, close), low above
|
|
26
|
+
min(open, close)), negative volume;
|
|
27
|
+
- **limits** 鈥?board-aware price-limit detection (main 卤10%, STAR and
|
|
28
|
+
ChiNext 卤20%, BSE 卤30%, ST 卤5% on the main board) against the previous
|
|
29
|
+
close with tick rounding tolerance, plus a documented suspension
|
|
30
|
+
heuristic (zero volume, or no prices on a dated row);
|
|
31
|
+
- **audit** 鈥?daily quality audit: are watched codes still listed, does
|
|
32
|
+
history coverage meet the threshold, are there calendar gaps? All data
|
|
33
|
+
sources injectable, results append-only;
|
|
34
|
+
- **snapshot** 鈥?sha256 manifests with cutoffs, so "which data did this
|
|
35
|
+
backtest actually see" is a file you can compare and prove.
|
|
36
|
+
|
|
37
|
+
## Philosophy
|
|
38
|
+
|
|
39
|
+
**Data is an asset; immunity is a discipline.**
|
|
40
|
+
|
|
41
|
+
Most data tooling optimizes for *getting* data. This tool optimizes for
|
|
42
|
+
*trusting* the data you have 鈥?and it refuses to guess: board rules are
|
|
43
|
+
explicit tables, the suspension detector is documented as a heuristic
|
|
44
|
+
(vendor conventions differ), and the audit reports "universe unavailable"
|
|
45
|
+
instead of pretending the listing check ran. Read-only by design; every
|
|
46
|
+
function either returns a report or writes an append-only record. Nothing
|
|
47
|
+
here trades, prices, or decides.
|
|
48
|
+
|
|
49
|
+
## Quick start
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
# install from PyPI (once published)
|
|
53
|
+
pip install ashare-data-immunity
|
|
54
|
+
|
|
55
|
+
# or run without installing anything:
|
|
56
|
+
# PYTHONPATH=src python -m ashare_data_immunity --help
|
|
57
|
+
|
|
58
|
+
python examples/demo.py # clean + limits + audit + snapshot on synthetic data
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Your own data:
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
# 1. validate / clean a bars file
|
|
65
|
+
imm clean --bars bars.json # exit 1 when problems found
|
|
66
|
+
imm clean --bars bars.json --drop-non-positive --out clean.json
|
|
67
|
+
|
|
68
|
+
# 2. board-aware limits + suspensions
|
|
69
|
+
imm limits --bars bars.json --code 600000
|
|
70
|
+
imm limits --bars bars.json --code 688001 --st
|
|
71
|
+
|
|
72
|
+
# 3. daily quality audit (watchlist + history dir + append-only audit dir)
|
|
73
|
+
imm audit --watchlist watchlist.json \
|
|
74
|
+
--history-root data/daily --audit-root data/audits
|
|
75
|
+
|
|
76
|
+
# 4. snapshot versioning
|
|
77
|
+
imm snapshot --name v2026-08-01 --cutoff 2026-08-01 \
|
|
78
|
+
--files data/daily/*.json --root data --out manifests/v1.json
|
|
79
|
+
imm snapshot-compare --before manifests/v1.json --after manifests/v2.json
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
## Commands
|
|
83
|
+
|
|
84
|
+
| Command | What it does |
|
|
85
|
+
| --- | --- |
|
|
86
|
+
| `clean` | Validate bars (missing/non-finite/non-positive fields, OHLC consistency, negative volume); optionally sanitize (non-finite and non-positive prices 鈫?`None`, volume kept 鈮?0) and optionally drop non-positive rows |
|
|
87
|
+
| `limits` | Board classification, price-limit events (up/down with ratio and limit price) and suspension days for one code |
|
|
88
|
+
| `audit` | Listing (codes not in the injected universe), history coverage, calendar continuity; appends a JSONL record per day |
|
|
89
|
+
| `snapshot` | sha256 manifest of a file list with name + cutoff |
|
|
90
|
+
| `snapshot-compare` | added / removed / changed files between two manifests |
|
|
91
|
+
| `version` | Print version |
|
|
92
|
+
|
|
93
|
+
## Board rules (v0.1)
|
|
94
|
+
|
|
95
|
+
| Board | Prefixes | Limit |
|
|
96
|
+
| --- | --- | --- |
|
|
97
|
+
| main | 60xxxx / 00xxxx | 卤10% (ST: 卤5%) |
|
|
98
|
+
| STAR | 688 / 689 | 卤20% |
|
|
99
|
+
| ChiNext | 300 / 301 | 卤20% |
|
|
100
|
+
| Beijing SE | 43x / 83x / 87x / 920 | 卤30% |
|
|
101
|
+
| unknown | 鈥?| 卤10% (assumed main) |
|
|
102
|
+
|
|
103
|
+
Limit detection compares `close` against `round(prev_close 脳 (1 卤 ratio), 2)`
|
|
104
|
+
with a default tolerance of 0.001 for vendor rounding conventions. The
|
|
105
|
+
first bar has no reference and is never flagged. **Verify the tables
|
|
106
|
+
against the current exchange rule documents before production use** 鈥?the
|
|
107
|
+
tool's job is to make the rules explicit, not to invent them.
|
|
108
|
+
|
|
109
|
+
Suspension heuristic: a dated row with zero volume, or with no prices at
|
|
110
|
+
all, is a suspension day. Documented, not hidden 鈥?and toggleable
|
|
111
|
+
(`zero_volume_means_suspended`).
|
|
112
|
+
|
|
113
|
+
## Development
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
python -m pip install -e . pytest
|
|
117
|
+
python -m pytest
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
CI runs the full test suite on Ubuntu, Windows and macOS with Python 3.11
|
|
121
|
+
and 3.12. Issues are handled on weekends; pull requests are welcome.
|
|
122
|
+
|
|
123
|
+
## Related work
|
|
124
|
+
|
|
125
|
+
This tool makes no claims to novelty of its own: it is the engineering
|
|
126
|
+
layer under the data-quality principles that the industry is converging on
|
|
127
|
+
鈥?point-in-time discipline ([Kelly et al., NBER w35247](https://www.nber.org/papers/w35247)),
|
|
128
|
+
look-ahead awareness ([Fonseca 2026, arXiv:2607.04958](https://econpapers.repec.org/paper/arxpapers/2607.04958.htm))
|
|
129
|
+
and reproducible snapshots. The pieces that *are* worth citing live in the
|
|
130
|
+
sibling repos of this project family; this one just keeps the data honest.
|
|
131
|
+
|
|
132
|
+
## Project family
|
|
133
|
+
|
|
134
|
+
Part of [Foolproof Labs](https://github.com/foolproof-labs) — a toolchain
|
|
135
|
+
against self-deception in quantitative research:
|
|
136
|
+
|
|
137
|
+
- [pit-adjuster](https://github.com/foolproof-labs/pit-adjuster) — PIT back-adjustment with static forward-adjustment drift detection
|
|
138
|
+
- [falsification-ledger](https://github.com/foolproof-labs/falsification-ledger) — pre-registration and falsification ledger
|
|
139
|
+
- [factor-qc](https://github.com/foolproof-labs/factor-qc) — fail-closed backtest quality gate
|
|
140
|
+
- [lesson-book](https://github.com/foolproof-labs/lesson-book) — tuition memory for traders
|
|
141
|
+
- [lookahead-free](https://github.com/foolproof-labs/lookahead-free) — verifiable look-ahead-freedom checks
|
|
142
|
+
- [ashare-data-immunity](https://github.com/foolproof-labs/ashare-data-immunity) — data immunity for A-share daily bars
|
|
143
|
+
|
|
144
|
+
## License
|
|
145
|
+
|
|
146
|
+
MIT
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=64"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "ashare-data-immunity"
|
|
7
|
+
version = "0.1.1"
|
|
8
|
+
description = "Data immunity for A-share daily bars: cleaning, board-aware price-limit and suspension detection, quality audit and snapshot versioning."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
keywords = ["a-share", "data-quality", "price-limit", "suspension", "cleaning", "snapshot", "quant"]
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Development Status :: 3 - Alpha",
|
|
15
|
+
"Environment :: Console",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"Intended Audience :: Financial and Insurance Industry",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Programming Language :: Python :: 3.11",
|
|
21
|
+
"Programming Language :: Python :: 3.12",
|
|
22
|
+
"Topic :: Office/Business :: Financial :: Investment",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
[project.scripts]
|
|
26
|
+
imm = "ashare_data_immunity.cli:main"
|
|
27
|
+
ashare-data-immunity = "ashare_data_immunity.cli:main"
|
|
28
|
+
|
|
29
|
+
[tool.setuptools.packages.find]
|
|
30
|
+
where = ["src"]
|
|
31
|
+
|
|
32
|
+
[tool.pytest.ini_options]
|
|
33
|
+
testpaths = ["tests"]
|
|
34
|
+
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""ashare-data-immunity: data immunity for A-share daily bars.
|
|
2
|
+
|
|
3
|
+
Cleaning (NaN/OHLCV), board-aware price-limit and suspension detection,
|
|
4
|
+
quality audit (listing / coverage / continuity, with injectable data
|
|
5
|
+
sources) and snapshot versioning (sha256 manifests). Read-only by design:
|
|
6
|
+
nothing here trades, prices, or decides 鈥?it keeps the data you feed it
|
|
7
|
+
honest.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from .audit import (
|
|
11
|
+
continuity_problems,
|
|
12
|
+
history_coverage,
|
|
13
|
+
listing_problems,
|
|
14
|
+
run_quality_audit,
|
|
15
|
+
)
|
|
16
|
+
from .cleaning import clean_bars, validate_bars
|
|
17
|
+
from .limits import board_of, detect_limits, price_limit_ratio, suspension_days
|
|
18
|
+
from .snapshot import build_snapshot, compare_snapshots
|
|
19
|
+
|
|
20
|
+
__version__ = "0.1.1"
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"board_of",
|
|
24
|
+
"build_snapshot",
|
|
25
|
+
"clean_bars",
|
|
26
|
+
"compare_snapshots",
|
|
27
|
+
"continuity_problems",
|
|
28
|
+
"detect_limits",
|
|
29
|
+
"history_coverage",
|
|
30
|
+
"listing_problems",
|
|
31
|
+
"price_limit_ratio",
|
|
32
|
+
"run_quality_audit",
|
|
33
|
+
"suspension_days",
|
|
34
|
+
"validate_bars",
|
|
35
|
+
]
|
|
36
|
+
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"""Quality audit with injectable data sources.
|
|
2
|
+
|
|
3
|
+
The audit answers three daily questions about a watchlist:
|
|
4
|
+
|
|
5
|
+
1. **listing** — are the watched codes still in the current universe?
|
|
6
|
+
(delisting / suspension check; the universe source is injectable —
|
|
7
|
+
network fetchers are the caller's business)
|
|
8
|
+
2. **coverage** — what fraction of the watchlist has daily history?
|
|
9
|
+
3. **continuity** — which codes have calendar gaps beyond a threshold?
|
|
10
|
+
|
|
11
|
+
Everything is read-only; results are appended to a JSONL audit file.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
from datetime import date, datetime, timedelta
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any, Callable
|
|
20
|
+
|
|
21
|
+
MIN_HISTORY_COVERAGE = 0.8
|
|
22
|
+
MAX_CALENDAR_GAP_DAYS = 15
|
|
23
|
+
|
|
24
|
+
UniverseFetcher = Callable[[], list[dict[str, Any]]]
|
|
25
|
+
WatchlistLoader = Callable[[Path], dict[str, Any]]
|
|
26
|
+
BarsLoader = Callable[[Path, str], list[dict[str, Any]]]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _watchlist_codes(payload: dict[str, Any]) -> list[str]:
|
|
30
|
+
eligible = payload.get("eligible_codes")
|
|
31
|
+
if isinstance(eligible, list):
|
|
32
|
+
return sorted({str(code).strip() for code in eligible if str(code).strip()})
|
|
33
|
+
members = payload.get("members") or []
|
|
34
|
+
return sorted(
|
|
35
|
+
{
|
|
36
|
+
str(item.get("code") or "").strip()
|
|
37
|
+
for item in members
|
|
38
|
+
if str(item.get("code") or "").strip()
|
|
39
|
+
}
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def default_watchlist_loader(path: Path) -> dict[str, Any]:
|
|
44
|
+
try:
|
|
45
|
+
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
46
|
+
except (OSError, json.JSONDecodeError):
|
|
47
|
+
return {}
|
|
48
|
+
return payload if isinstance(payload, dict) else {}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def default_bars_loader(history_root: Path, code: str) -> list[dict[str, Any]]:
|
|
52
|
+
"""Load ``<history_root>/<code>.json`` with a ``bars`` list."""
|
|
53
|
+
try:
|
|
54
|
+
payload = json.loads((history_root / f"{code}.json").read_text(encoding="utf-8"))
|
|
55
|
+
except (OSError, json.JSONDecodeError):
|
|
56
|
+
return []
|
|
57
|
+
bars = payload.get("bars") or []
|
|
58
|
+
return bars if isinstance(bars, list) else []
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def listing_problems(
|
|
62
|
+
codes: list[str],
|
|
63
|
+
universe_rows: list[dict[str, Any]],
|
|
64
|
+
) -> dict[str, Any]:
|
|
65
|
+
"""Codes not present in the current universe (suspected delisted/suspended)."""
|
|
66
|
+
universe = {
|
|
67
|
+
str(row.get("code") or "").strip()
|
|
68
|
+
for row in universe_rows
|
|
69
|
+
if str(row.get("code") or "").strip()
|
|
70
|
+
}
|
|
71
|
+
missing = sorted(code for code in codes if code not in universe)
|
|
72
|
+
return {
|
|
73
|
+
"checked": True,
|
|
74
|
+
"universe_count": len(universe),
|
|
75
|
+
"not_in_current_universe": missing,
|
|
76
|
+
"problem": bool(missing),
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def history_coverage(
|
|
81
|
+
codes: list[str],
|
|
82
|
+
bars_loader: BarsLoader,
|
|
83
|
+
history_root: Path | str,
|
|
84
|
+
) -> dict[str, Any]:
|
|
85
|
+
"""Fraction of the watchlist with non-empty daily history."""
|
|
86
|
+
if not codes:
|
|
87
|
+
return {"checked": True, "covered": 0, "total": 0, "missing_codes": [], "problem": False}
|
|
88
|
+
covered: list[str] = []
|
|
89
|
+
missing: list[str] = []
|
|
90
|
+
for code in codes:
|
|
91
|
+
bars = bars_loader(Path(history_root), code)
|
|
92
|
+
(covered if bars else missing).append(code)
|
|
93
|
+
ratio = len(covered) / len(codes)
|
|
94
|
+
return {
|
|
95
|
+
"checked": True,
|
|
96
|
+
"covered": len(covered),
|
|
97
|
+
"total": len(codes),
|
|
98
|
+
"coverage": round(ratio, 4),
|
|
99
|
+
"missing_codes": missing,
|
|
100
|
+
"problem": ratio < MIN_HISTORY_COVERAGE,
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def continuity_problems(
|
|
105
|
+
codes: list[str],
|
|
106
|
+
bars_loader: BarsLoader,
|
|
107
|
+
history_root: Path | str,
|
|
108
|
+
*,
|
|
109
|
+
max_gap_days: int = MAX_CALENDAR_GAP_DAYS,
|
|
110
|
+
) -> list[dict[str, Any]]:
|
|
111
|
+
"""Codes whose daily bars have calendar gaps beyond the threshold."""
|
|
112
|
+
problems: list[dict[str, Any]] = []
|
|
113
|
+
for code in codes:
|
|
114
|
+
bars = bars_loader(Path(history_root), code)
|
|
115
|
+
dates = sorted(
|
|
116
|
+
{str(row.get("date") or "")[:10] for row in bars if row.get("date")}
|
|
117
|
+
)
|
|
118
|
+
if len(dates) < 2:
|
|
119
|
+
continue
|
|
120
|
+
gaps: list[dict[str, Any]] = []
|
|
121
|
+
for previous, current in zip(dates, dates[1:]):
|
|
122
|
+
try:
|
|
123
|
+
gap_days = (date.fromisoformat(current) - date.fromisoformat(previous)).days
|
|
124
|
+
except ValueError:
|
|
125
|
+
continue
|
|
126
|
+
if gap_days > max_gap_days:
|
|
127
|
+
gaps.append({"from": previous, "to": current, "gap_days": gap_days})
|
|
128
|
+
if gaps:
|
|
129
|
+
problems.append({"code": code, "gaps": gaps})
|
|
130
|
+
return problems
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def run_quality_audit(
|
|
134
|
+
*,
|
|
135
|
+
watchlist_path: Path | str,
|
|
136
|
+
history_root: Path | str,
|
|
137
|
+
audit_root: Path | str,
|
|
138
|
+
universe_fetcher: UniverseFetcher | None = None,
|
|
139
|
+
watchlist_loader: WatchlistLoader | None = None,
|
|
140
|
+
bars_loader: BarsLoader | None = None,
|
|
141
|
+
now: datetime | None = None,
|
|
142
|
+
) -> dict[str, Any]:
|
|
143
|
+
"""Run all checks and append the result record (JSONL, one per day)."""
|
|
144
|
+
watchlist_path = Path(watchlist_path)
|
|
145
|
+
watchlist_loader = watchlist_loader or default_watchlist_loader
|
|
146
|
+
bars_loader = bars_loader or default_bars_loader
|
|
147
|
+
current = now or datetime.now()
|
|
148
|
+
codes = _watchlist_codes(watchlist_loader(watchlist_path))
|
|
149
|
+
|
|
150
|
+
listing: dict[str, Any] = {"checked": False, "problem": False, "reason": "universe_unavailable"}
|
|
151
|
+
if universe_fetcher is not None:
|
|
152
|
+
try:
|
|
153
|
+
listing = listing_problems(codes, universe_fetcher())
|
|
154
|
+
except Exception as exc: # noqa: BLE001 - tolerant of upstream failure
|
|
155
|
+
listing = {
|
|
156
|
+
"checked": False,
|
|
157
|
+
"problem": False,
|
|
158
|
+
"reason": f"{type(exc).__name__}: {str(exc)[:120]}",
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
coverage = history_coverage(codes, bars_loader, history_root)
|
|
162
|
+
continuity = continuity_problems(codes, bars_loader, history_root)
|
|
163
|
+
|
|
164
|
+
problems: list[str] = []
|
|
165
|
+
if listing.get("problem"):
|
|
166
|
+
problems.append(
|
|
167
|
+
"suspected delisted/suspended (not in current universe): "
|
|
168
|
+
+ ",".join(listing["not_in_current_universe"][:10])
|
|
169
|
+
)
|
|
170
|
+
if coverage.get("problem"):
|
|
171
|
+
problems.append(f"history coverage {coverage['coverage']} < {MIN_HISTORY_COVERAGE}")
|
|
172
|
+
if continuity:
|
|
173
|
+
problems.append(f"codes with calendar gaps: {len(continuity)}")
|
|
174
|
+
|
|
175
|
+
record = {
|
|
176
|
+
"schema_version": "ashare_data_immunity.audit.v1",
|
|
177
|
+
"audit_date": current.date().isoformat(),
|
|
178
|
+
"checked_at": current.isoformat(timespec="seconds"),
|
|
179
|
+
"watchlist_code_count": len(codes),
|
|
180
|
+
"listing": listing,
|
|
181
|
+
"history_coverage": coverage,
|
|
182
|
+
"continuity": continuity,
|
|
183
|
+
"problems": problems,
|
|
184
|
+
"passed": not problems,
|
|
185
|
+
}
|
|
186
|
+
audit_root = Path(audit_root)
|
|
187
|
+
audit_root.mkdir(parents=True, exist_ok=True)
|
|
188
|
+
path = audit_root / f"{record['audit_date']}.jsonl"
|
|
189
|
+
with path.open("a", encoding="utf-8") as handle:
|
|
190
|
+
handle.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n")
|
|
191
|
+
return record
|