quantmine 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- quantmine/__init__.py +97 -0
- quantmine/back_testing.py +381 -0
- quantmine/config.py +115 -0
- quantmine/data_acquisition.py +326 -0
- quantmine/datareader.py +119 -0
- quantmine/factor_attribution.py +70 -0
- quantmine/factor_mining.py +192 -0
- quantmine/factor_register.py +73 -0
- quantmine/ic_calculator.py +543 -0
- quantmine/load_config.py +16 -0
- quantmine-0.2.0.dist-info/METADATA +222 -0
- quantmine-0.2.0.dist-info/RECORD +14 -0
- quantmine-0.2.0.dist-info/WHEEL +4 -0
- quantmine-0.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,543 @@
|
|
|
1
|
+
import pandas as pd
|
|
2
|
+
import os
|
|
3
|
+
from scipy import stats
|
|
4
|
+
import numpy as np
|
|
5
|
+
from statsmodels.stats.multitest import multipletests
|
|
6
|
+
|
|
7
|
+
def forward_return(close:pd.DataFrame, tickers: list = None, periods: list[int] | int = None)->dict:
|
|
8
|
+
"""Build forward holding-period returns for each ticker.
|
|
9
|
+
|
|
10
|
+
Args:
|
|
11
|
+
close: Close price dataframe indexed by date with one column per ticker.
|
|
12
|
+
tickers: Ticker list to process.
|
|
13
|
+
periods: Holding periods in trading days. Can be a list or a single int.
|
|
14
|
+
|
|
15
|
+
Returns:
|
|
16
|
+
A tuple of ``(holding_period_dataframe, periods)``. The dataframe
|
|
17
|
+
contains columns named like ``{period}DaysHoldingPeriod_TICKER``.
|
|
18
|
+
|
|
19
|
+
Notes:
|
|
20
|
+
The return series are shifted forward, so the value at date ``t`` is the
|
|
21
|
+
return realized over the next ``period`` trading days.
|
|
22
|
+
"""
|
|
23
|
+
if isinstance(periods, int):
|
|
24
|
+
periods = [periods]
|
|
25
|
+
if periods is None:
|
|
26
|
+
periods = [1,5,20] #将默认参数放到函数中可以防止默认变量在函数内被更改
|
|
27
|
+
if tickers is None:
|
|
28
|
+
cols = close.columns.tolist()
|
|
29
|
+
else:
|
|
30
|
+
cols = tickers
|
|
31
|
+
forward_return = {}
|
|
32
|
+
for period in periods:
|
|
33
|
+
# 整个frame一次pct_change, 替代逐列插入(逐列插入会触发DataFrame碎片化, 570*3列时极慢)
|
|
34
|
+
forward_return[period] = close[cols].pct_change(period).shift(-period)
|
|
35
|
+
return forward_return
|
|
36
|
+
|
|
37
|
+
def data_standarization(df:pd.DataFrame)->pd.DataFrame:
|
|
38
|
+
"""Cross-sectionally standardize factor columns to the range [-1, 1].
|
|
39
|
+
|
|
40
|
+
Args:
|
|
41
|
+
df: Factor dataframe with columns named as ``factor_ticker``.
|
|
42
|
+
|
|
43
|
+
Returns:
|
|
44
|
+
A dataframe with the same shape, where each factor group is ranked by
|
|
45
|
+
row and scaled to the interval ``[-1, 1]``.
|
|
46
|
+
|
|
47
|
+
Notes:
|
|
48
|
+
This function assumes the column naming convention is consistent and
|
|
49
|
+
uses rank-based scaling, which is less sensitive to outliers than raw
|
|
50
|
+
value normalization.
|
|
51
|
+
"""
|
|
52
|
+
return {
|
|
53
|
+
factor_name: factor_df.rank(axis = 1, pct=True)*2-1
|
|
54
|
+
for factor_name, factor_df in df.items()
|
|
55
|
+
} #一次concat替代逐块列插入, 避免碎片化
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def TM_Information_correlation(factors: dict[str, pd.DataFrame], forward_returns: dict[int, pd.DataFrame], output_path: str)->pd.DataFrame:
|
|
59
|
+
"""Compute time-series information correlation for each ticker.
|
|
60
|
+
|
|
61
|
+
Args:
|
|
62
|
+
tickers: Tickers to evaluate.
|
|
63
|
+
factors: Factor dataframe with ``factor_ticker`` column names.
|
|
64
|
+
different_holding_period: Forward return dataframe with matching naming
|
|
65
|
+
convention.
|
|
66
|
+
output_path: Relative or absolute path used to save the parquet output.
|
|
67
|
+
|
|
68
|
+
Returns:
|
|
69
|
+
A dataframe of time-series IC values, saved to ``output_path``.
|
|
70
|
+
|
|
71
|
+
Notes:
|
|
72
|
+
The function groups columns by ticker first and then computes the
|
|
73
|
+
correlation between factor series and holding-period return series.
|
|
74
|
+
"""
|
|
75
|
+
result = {}
|
|
76
|
+
for factor_name ,factor_df in factors.items():
|
|
77
|
+
factor_ticker = list(factor_df.columns)
|
|
78
|
+
for period, return_df in forward_returns.items():
|
|
79
|
+
forward_returns_ticker = list(return_df.columns)
|
|
80
|
+
overlap = list(set(factor_ticker) & set(forward_returns_ticker))
|
|
81
|
+
ic_series = factor_df[overlap].corrwith(return_df[overlap], method= 'pearson',axis=0)
|
|
82
|
+
result[(factor_name, period)] = ic_series
|
|
83
|
+
TM_IC_matrix=pd.DataFrame(result)
|
|
84
|
+
TM_IC_matrix.columns.names = ['factor', 'period']
|
|
85
|
+
TM_IC_matrix.to_parquet(os.path.join(os.getcwd(), output_path))
|
|
86
|
+
print("time Series information correlation computation complete")
|
|
87
|
+
|
|
88
|
+
def CS_Information_Correlation(factors: dict[str, pd.DataFrame], forward_returns: dict[int, pd.DataFrame], output_path: str)-> pd.DataFrame:
|
|
89
|
+
"""Compute cross-sectional information correlation across dates.
|
|
90
|
+
|
|
91
|
+
Args:
|
|
92
|
+
factors: Factor dataframe with ``factor_ticker`` column names.
|
|
93
|
+
different_holding_period: Forward return dataframe with matching naming
|
|
94
|
+
convention.
|
|
95
|
+
output_path: Relative or absolute path used to save the parquet output.
|
|
96
|
+
|
|
97
|
+
Returns:
|
|
98
|
+
A dataframe of cross-sectional IC values, saved to ``output_path``.
|
|
99
|
+
|
|
100
|
+
Notes:
|
|
101
|
+
This function correlates factor values with same-date forward returns
|
|
102
|
+
across the cross section of tickers.
|
|
103
|
+
"""
|
|
104
|
+
result = {}
|
|
105
|
+
for factor_name ,factor_df in factors.items():
|
|
106
|
+
factor_ticker = list(factor_df.columns)
|
|
107
|
+
for period, return_df in forward_returns.items():
|
|
108
|
+
forward_returns_ticker = list(return_df.columns)
|
|
109
|
+
overlap = list(set(factor_ticker) & set(forward_returns_ticker))
|
|
110
|
+
ic_series = factor_df[overlap].corrwith(return_df[overlap], method= 'pearson',axis=1)
|
|
111
|
+
result[(factor_name, period)] = ic_series
|
|
112
|
+
CS_IC_matrix=pd.DataFrame(result)
|
|
113
|
+
CS_IC_matrix.columns.names = ['factor', 'period']
|
|
114
|
+
CS_IC_matrix.to_parquet(os.path.join(os.getcwd(), output_path))
|
|
115
|
+
print("Cross-sectional information correlation computation complete")
|
|
116
|
+
|
|
117
|
+
return CS_IC_matrix
|
|
118
|
+
|
|
119
|
+
def summary(cross_section_IC_matrix:pd.DataFrame)->pd.DataFrame:
|
|
120
|
+
"""Summarize an IC dataframe with mean, std, IR, sign ratio, and count.
|
|
121
|
+
|
|
122
|
+
Args:
|
|
123
|
+
cross_section_IC_matrix: IC dataframe where each column is a factor or
|
|
124
|
+
factor-holding-period combination.
|
|
125
|
+
|
|
126
|
+
Returns:
|
|
127
|
+
A summary dataframe indexed by column name.
|
|
128
|
+
|
|
129
|
+
Input : ic_df, MultiIndex column(factor, holding_period)
|
|
130
|
+
Output : DataFrame, MultiIndex index (factor, holding_period)
|
|
131
|
+
"""
|
|
132
|
+
return pd.DataFrame({
|
|
133
|
+
'IC_mean' : cross_section_IC_matrix.mean(),
|
|
134
|
+
'IC_std': cross_section_IC_matrix.std(),
|
|
135
|
+
'IR': cross_section_IC_matrix.mean()/cross_section_IC_matrix.std(),
|
|
136
|
+
'IC>0 pct': (cross_section_IC_matrix>0).mean(),
|
|
137
|
+
'n': cross_section_IC_matrix.count(),
|
|
138
|
+
})
|
|
139
|
+
|
|
140
|
+
def resample_summary(cross_section_IC: pd.DataFrame, periods:list|int)-> pd.DataFrame:
|
|
141
|
+
"""Build IC summaries from down-sampled, approximately independent samples.
|
|
142
|
+
|
|
143
|
+
Args:
|
|
144
|
+
cross_section_IC: Cross-sectional IC dataframe.
|
|
145
|
+
periods: Holding periods in trading days.
|
|
146
|
+
|
|
147
|
+
Returns:
|
|
148
|
+
A concatenated dataframe of summary statistics for each holding period.
|
|
149
|
+
|
|
150
|
+
Notes:
|
|
151
|
+
The dataframe is sub-sampled using ``iloc[::period]`` to reduce overlap
|
|
152
|
+
dependence when the holding period is longer than one day.
|
|
153
|
+
"""
|
|
154
|
+
if isinstance(periods, int):
|
|
155
|
+
periods = [periods]
|
|
156
|
+
result = {}
|
|
157
|
+
for period in periods:
|
|
158
|
+
period_df = cross_section_IC.xs(period, level = 'period', axis = 1) #xs筛选出的就是df
|
|
159
|
+
summary_df = period_df.iloc[::period]
|
|
160
|
+
result[f'{period}HoldingPeriodSummary']=summary(summary_df)
|
|
161
|
+
result_df = pd.concat(result.values())
|
|
162
|
+
return result_df
|
|
163
|
+
|
|
164
|
+
def newey_west_summary(cross_section_IC: pd.DataFrame, lag_multiplier: int = 2)-> pd.DataFrame:
|
|
165
|
+
"""Compute Newey-West adjusted IC statistics for overlapping returns.
|
|
166
|
+
|
|
167
|
+
Args:
|
|
168
|
+
cross_section_IC: Cross-sectional IC dataframe.
|
|
169
|
+
lag_multiplier: Multiplier used to set the Newey-West lag length.
|
|
170
|
+
|
|
171
|
+
Returns:
|
|
172
|
+
A dataframe containing IC mean, IC std, IR, sample size, lag, Newey-West
|
|
173
|
+
t-statistic, and p-value.
|
|
174
|
+
|
|
175
|
+
Notes:
|
|
176
|
+
Overlapping holding periods create autocorrelation, so a plain IID t-test
|
|
177
|
+
can overstate significance. When ``period == 1``, the lag becomes zero and
|
|
178
|
+
the method collapses to a standard t-test.
|
|
179
|
+
"""
|
|
180
|
+
rows = {}
|
|
181
|
+
for col in cross_section_IC.columns:
|
|
182
|
+
factor, period = col
|
|
183
|
+
lag = lag_multiplier * max(period - 1, 0)
|
|
184
|
+
s = cross_section_IC[col].dropna()
|
|
185
|
+
n = len(s)
|
|
186
|
+
mu = s.mean()
|
|
187
|
+
e = (s - mu).values
|
|
188
|
+
var = e @ e / n #lag=0时就是普通样本方差
|
|
189
|
+
for l in range(1, min(lag, n - 1) + 1):
|
|
190
|
+
w = 1 - l / (lag + 1) #Bartlett核权重, 保证方差估计非负
|
|
191
|
+
var += 2 * w * (e[:-l] @ e[l:]) / n
|
|
192
|
+
se = np.sqrt(var / n)
|
|
193
|
+
rows[col] = {'IC_mean': mu, 'IC_std': s.std(), 'IR': mu / s.std(),
|
|
194
|
+
'n': n, 'lag': lag, 'NW_t': mu / se}
|
|
195
|
+
nw_df = pd.DataFrame(rows).T
|
|
196
|
+
nw_df.index =nw_df.index.set_names(['factor', 'period'])
|
|
197
|
+
nw_df['p_value'] = stats.t.sf(nw_df['NW_t'].abs(), df=nw_df['n'] - 1) * 2
|
|
198
|
+
return nw_df
|
|
199
|
+
|
|
200
|
+
def multiple_testing(summary_df:pd.DataFrame)->pd.DataFrame:
|
|
201
|
+
"""Apply multiple-testing corrections to IC significance results.
|
|
202
|
+
|
|
203
|
+
Args:
|
|
204
|
+
summary_df: Summary dataframe from either ``newey_west_summary`` or
|
|
205
|
+
``resample_summary``.
|
|
206
|
+
|
|
207
|
+
Returns:
|
|
208
|
+
A dataframe with raw p-values and significance flags under several
|
|
209
|
+
correction schemes.
|
|
210
|
+
|
|
211
|
+
Notes:
|
|
212
|
+
If ``NW_t`` exists, the function uses the Newey-West corrected path.
|
|
213
|
+
Otherwise it falls back to the IID resampled path.
|
|
214
|
+
"""
|
|
215
|
+
if 'NW_t' in summary_df.columns: #NW路径: 用自相关修正后的t和p(newey_west_summary的输出)
|
|
216
|
+
significant_t = pd.DataFrame({
|
|
217
|
+
"t": summary_df['NW_t'],
|
|
218
|
+
"p_value": summary_df['p_value']
|
|
219
|
+
})
|
|
220
|
+
else: #iid路径: 抽样降频后的独立样本t检验(resample_summary的输出)
|
|
221
|
+
significant_t = pd.DataFrame({
|
|
222
|
+
"t":summary_df["IR"]*np.sqrt(summary_df["n"]),
|
|
223
|
+
"p_value":stats.t.sf(x=abs(summary_df["IR"]*np.sqrt(summary_df["n"])), df=summary_df['n']-1)*2
|
|
224
|
+
})
|
|
225
|
+
significant_t['significant'] = significant_t["p_value"] < 0.05
|
|
226
|
+
significant_t['Bonferroni_significant'] = significant_t["p_value"] < 0.05/len(summary_df)
|
|
227
|
+
significant_t['Rank'] = significant_t["p_value"].rank(ascending=1,method='max')
|
|
228
|
+
rej_bonf, _, _, _ = multipletests(significant_t['p_value'], alpha = 0.05, method='fdr_bh')
|
|
229
|
+
significant_t['BH_significant'] = rej_bonf
|
|
230
|
+
return significant_t
|
|
231
|
+
|
|
232
|
+
def orthogonal_analysis(factors: dict[str, pd.DataFrame]):
|
|
233
|
+
"""Compute average factor correlation and identify highly correlated pairs.
|
|
234
|
+
|
|
235
|
+
Args:
|
|
236
|
+
factors_ticker: Factor dataframe with ``factor_ticker`` columns.
|
|
237
|
+
|
|
238
|
+
Returns:
|
|
239
|
+
A tuple of ``(average_correlation_matrix, high_correlation_dict)``.
|
|
240
|
+
|
|
241
|
+
Notes:
|
|
242
|
+
Dates with missing factor columns are skipped. Factors whose absolute
|
|
243
|
+
average correlation exceeds 0.5 are treated as highly correlated.
|
|
244
|
+
"""
|
|
245
|
+
factor = sorted(factors.keys())
|
|
246
|
+
ticker = sorted(set().union(*[tickers.columns.tolist() for tickers in factors.values()])) #*是“解包”操作,把列表拆成多个独立参数传给union
|
|
247
|
+
|
|
248
|
+
#每个因子重排成一张 dates*tickers 的宽表, 列顺序统一, 后续全部是frame级向量化操作
|
|
249
|
+
frames = { f: factors[f].reindex(columns = ticker) for f in factor}
|
|
250
|
+
|
|
251
|
+
#有效日: 每个因子当天至少有一个非NaN值(与原逐日实现的跳过条件一致)
|
|
252
|
+
valid = pd.concat({f: frames[f].notna().any(axis=1) for f in factors}, axis=1).all(axis=1)
|
|
253
|
+
valid_days = int(valid.sum())
|
|
254
|
+
|
|
255
|
+
avg_corr = pd.DataFrame(1.0, index=factors, columns=factors) #对角线恒为1
|
|
256
|
+
for i, fa in enumerate(factors):
|
|
257
|
+
a = frames[fa]
|
|
258
|
+
for fb in factor[i+1:]:
|
|
259
|
+
b = frames[fb]
|
|
260
|
+
#逐日截面Pearson相关的向量化展开: 每天在两因子共同非NaN的ticker上算相关
|
|
261
|
+
mask = a.notna() & b.notna()
|
|
262
|
+
xa, xb = a.where(mask), b.where(mask)
|
|
263
|
+
n = mask.sum(axis=1)
|
|
264
|
+
sa, sb = xa.sum(axis=1), xb.sum(axis=1)
|
|
265
|
+
cov = (xa*xb).sum(axis=1) - sa*sb/n
|
|
266
|
+
var_a = (xa*xa).sum(axis=1) - sa*sa/n
|
|
267
|
+
var_b = (xb*xb).sum(axis=1) - sb*sb/n
|
|
268
|
+
corr_t = cov/np.sqrt(var_a*var_b)
|
|
269
|
+
#与原实现一致: 对有效日求和(若某日相关无法计算则整体为NaN)后除以有效日数
|
|
270
|
+
pair_avg = corr_t[valid].to_numpy().sum()/valid_days
|
|
271
|
+
avg_corr.loc[fa, fb] = pair_avg
|
|
272
|
+
avg_corr.loc[fb, fa] = pair_avg
|
|
273
|
+
|
|
274
|
+
high_corr_dict={
|
|
275
|
+
column: avg_corr.index[(avg_corr[column].abs()>0.5) & (avg_corr.index!=column)].tolist() for column in avg_corr.columns
|
|
276
|
+
}
|
|
277
|
+
return avg_corr, high_corr_dict
|
|
278
|
+
|
|
279
|
+
def orthogonalize(factors: dict[str, pd.DataFrame], high_corr_dict: dict, ic_summary:pd.DataFrame, threshold: float = 0.03, min_period: int = 60)->dict:
|
|
280
|
+
"""Orthogonalize highly correlated factors using expanding regression.
|
|
281
|
+
|
|
282
|
+
Args:
|
|
283
|
+
factor_df: Factor dataframe with ``factor_ticker`` columns.
|
|
284
|
+
high_corr_dict: Dictionary of highly correlated factor names produced by
|
|
285
|
+
``orthogonal_analysis``.
|
|
286
|
+
ic_summary: IC summary dataframe used to estimate factor quality.
|
|
287
|
+
threshold: Minimum average IR required to keep a factor. Default is 0.03.
|
|
288
|
+
min_period: Minimum number of observations used in the expanding beta
|
|
289
|
+
calculation.
|
|
290
|
+
|
|
291
|
+
Returns:
|
|
292
|
+
A dataframe where some factor columns may be dropped or replaced by
|
|
293
|
+
orthogonalized residuals.
|
|
294
|
+
|
|
295
|
+
Notes:
|
|
296
|
+
When both factors in a correlated pair have low IR, both are dropped.
|
|
297
|
+
Otherwise the weaker factor is residualized against the stronger one.
|
|
298
|
+
"""
|
|
299
|
+
pairs=set()
|
|
300
|
+
for factor, factors_corr in high_corr_dict.items():
|
|
301
|
+
for factor_corr in factors_corr:
|
|
302
|
+
pair = tuple(sorted([factor, factor_corr]))
|
|
303
|
+
pairs.add(pair)
|
|
304
|
+
|
|
305
|
+
factor_ir = ic_summary['IR'].abs().groupby(level = 'factor').mean()
|
|
306
|
+
|
|
307
|
+
print("Average IR:")
|
|
308
|
+
print({k: round(v,4) for k,v in sorted(factor_ir.items())})
|
|
309
|
+
print(f"threshold={threshold}, will be dropped because it is below the threshold")
|
|
310
|
+
|
|
311
|
+
result = factors.copy()
|
|
312
|
+
drop = set()
|
|
313
|
+
orthogonalized = set()
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
for factor_a, factor_b in pairs:
|
|
317
|
+
if factor_a in drop or factor_b in drop:
|
|
318
|
+
continue
|
|
319
|
+
ir_a = factor_ir.get(factor_a, 0) #pd.Series有get操作,但不是字典
|
|
320
|
+
ir_b = factor_ir.get(factor_b, 0)
|
|
321
|
+
|
|
322
|
+
if ir_a < threshold and ir_b < threshold:
|
|
323
|
+
result.pop(factor_a)
|
|
324
|
+
result.pop(factor_b)
|
|
325
|
+
drop.update([factor_a, factor_b])
|
|
326
|
+
continue
|
|
327
|
+
|
|
328
|
+
keeper = factor_a if ir_a > ir_b else factor_b
|
|
329
|
+
to_orthogonalize = factor_b if ir_a > ir_b else factor_a
|
|
330
|
+
|
|
331
|
+
if ir_a < threshold:
|
|
332
|
+
result.pop(factor_a)
|
|
333
|
+
drop.add(factor_a)
|
|
334
|
+
|
|
335
|
+
elif ir_b < threshold:
|
|
336
|
+
|
|
337
|
+
result.pop(factor_b)
|
|
338
|
+
drop.add(factor_b)
|
|
339
|
+
|
|
340
|
+
if ir_a >= threshold and ir_b >= threshold:
|
|
341
|
+
if to_orthogonalize in orthogonalized:
|
|
342
|
+
continue
|
|
343
|
+
if keeper not in result.keys() or to_orthogonalize not in result.keys():
|
|
344
|
+
continue
|
|
345
|
+
x = result[keeper]
|
|
346
|
+
y = result[to_orthogonalize]
|
|
347
|
+
|
|
348
|
+
expanding_cov = x.expanding(min_periods= min_period).cov(y)
|
|
349
|
+
expanding_var = x.expanding(min_periods= min_period).var()
|
|
350
|
+
beta_series = expanding_cov/expanding_var
|
|
351
|
+
|
|
352
|
+
residuals = y - beta_series * x
|
|
353
|
+
result[to_orthogonalize] = residuals
|
|
354
|
+
orthogonalized.add(to_orthogonalize)
|
|
355
|
+
return result
|
|
356
|
+
|
|
357
|
+
def time_series_stationary_test(CS_IC_matrix:pd.DataFrame, rolling_period:int =126, periods:list = None)-> pd.DataFrame:
|
|
358
|
+
"""Compute rolling IC, autocorrelation, and yearly IC summaries.
|
|
359
|
+
|
|
360
|
+
Args:
|
|
361
|
+
CS_IC_matrix: Cross-sectional IC dataframe indexed by date.
|
|
362
|
+
rolling_period: Window size for rolling mean IC. Default is 126.
|
|
363
|
+
periods: Lags used when computing autocorrelation.
|
|
364
|
+
|
|
365
|
+
Returns:
|
|
366
|
+
A tuple of ``(rolling_ic_df, acf_df, yearly_df)``.
|
|
367
|
+
|
|
368
|
+
Notes:
|
|
369
|
+
The index is converted to ``datetime64`` before grouping and rolling.
|
|
370
|
+
"""
|
|
371
|
+
if periods is None:
|
|
372
|
+
periods = [1,5,20]
|
|
373
|
+
|
|
374
|
+
rolling_window_IC={}
|
|
375
|
+
acf_ic={}
|
|
376
|
+
|
|
377
|
+
CS_IC_matrix.index = pd.to_datetime(CS_IC_matrix.index)
|
|
378
|
+
for col in CS_IC_matrix.columns:
|
|
379
|
+
# rolling ic
|
|
380
|
+
rolling_window_IC[col]=CS_IC_matrix[col].rolling(rolling_period).mean()
|
|
381
|
+
rolling_ic_df = pd.DataFrame(rolling_window_IC, index = CS_IC_matrix.index)
|
|
382
|
+
|
|
383
|
+
for col in CS_IC_matrix.columns:
|
|
384
|
+
for period in periods:
|
|
385
|
+
# acf_ic
|
|
386
|
+
acf_ic[(col,period)] = CS_IC_matrix[col].corr(CS_IC_matrix[col].shift(period), method="pearson") #corr是对series, corrwith是对dataframe
|
|
387
|
+
acf_df = pd.Series(acf_ic).to_frame(name = 'ACF') #字典的每个value是一个标量,不能直接用pd.DataFrame,需要先转成Series
|
|
388
|
+
yearly_summary={}
|
|
389
|
+
# 分段IC
|
|
390
|
+
for year, group in CS_IC_matrix.groupby(CS_IC_matrix.index.year):
|
|
391
|
+
yearly_summary[year] = summary(group)
|
|
392
|
+
yearly_df = pd.concat(yearly_summary, axis= 0)
|
|
393
|
+
|
|
394
|
+
return rolling_ic_df, acf_df, yearly_df
|
|
395
|
+
|
|
396
|
+
def get_constitunents_at_date(historical_df: pd.DataFrame, date: pd.Timestamp)->set:
|
|
397
|
+
"""Get the set of active constituents on a specific date.
|
|
398
|
+
|
|
399
|
+
Args:
|
|
400
|
+
historical_df: Historical constituent table with ``start_date``,
|
|
401
|
+
``end_date``, and ``ticker`` columns.
|
|
402
|
+
date: Target date to query.
|
|
403
|
+
|
|
404
|
+
Returns:
|
|
405
|
+
A set of tickers that are active on the given date.
|
|
406
|
+
|
|
407
|
+
Notes:
|
|
408
|
+
The date comparison is inclusive. Missing ``end_date`` values are treated
|
|
409
|
+
as open-ended membership.
|
|
410
|
+
"""
|
|
411
|
+
mask = (historical_df['start_date'] <= date) & (historical_df['end_date'].isnull() | (historical_df['end_date'] >= date))
|
|
412
|
+
return set(historical_df.loc[mask, 'ticker'].str.replace('.','-',regax = False))
|
|
413
|
+
|
|
414
|
+
def train_test_analysis(cs_df: pd.DataFrame, factors: dict[str, pd.DataFrame], close: pd.DataFrame , train_end: str, test_start: str, periods: list|int = None):
|
|
415
|
+
"""Run the full train/test IC workflow and orthogonalization pipeline.
|
|
416
|
+
|
|
417
|
+
Args:
|
|
418
|
+
cs_df: Cross-sectional IC dataframe for all available dates.
|
|
419
|
+
factor_ticker: Full factor dataframe with ``factor_ticker`` columns.
|
|
420
|
+
close: Close price dataframe used to generate holding-period returns.
|
|
421
|
+
train_end: Last date included in the training sample.
|
|
422
|
+
test_start: First date included in the test sample.
|
|
423
|
+
periods: Holding periods in trading days.
|
|
424
|
+
|
|
425
|
+
Returns:
|
|
426
|
+
A dictionary containing training diagnostics, orthogonalized factor
|
|
427
|
+
outputs, test splits, and summary statistics.
|
|
428
|
+
|
|
429
|
+
Notes:
|
|
430
|
+
The training sample is used to select significant factors and build the
|
|
431
|
+
orthogonalization mapping before evaluating the test period.
|
|
432
|
+
"""
|
|
433
|
+
if isinstance(periods, int):
|
|
434
|
+
periods = [periods]
|
|
435
|
+
|
|
436
|
+
if periods is None:
|
|
437
|
+
periods = [1,5,20]
|
|
438
|
+
|
|
439
|
+
train_cs_df = cs_df[cs_df.index <= train_end]
|
|
440
|
+
test_cs_df = cs_df[cs_df.index >= test_start]
|
|
441
|
+
|
|
442
|
+
forward_return_train = forward_return(close[close.index<=train_end], close.columns, periods)
|
|
443
|
+
forward_return_train_stand = data_standarization(forward_return_train)
|
|
444
|
+
forward_return_test = forward_return(close[close.index>=test_start], close.columns, periods)
|
|
445
|
+
forward_return_test_stand = data_standarization(forward_return_test)
|
|
446
|
+
|
|
447
|
+
train_factor_ticker = {factor_name : train_factor_ticker.loc[:train_end] for factor_name, train_factor_ticker in factors.items()}
|
|
448
|
+
test_factor_ticker = {factor_name : test_factor_ticker.loc[test_start:] for factor_name, test_factor_ticker in factors.items()}
|
|
449
|
+
|
|
450
|
+
resample_summary_train = resample_summary(train_cs_df, periods)
|
|
451
|
+
print(resample_summary_train)
|
|
452
|
+
print(resample_summary_train.index.tolist()[:5])
|
|
453
|
+
|
|
454
|
+
orth_analysis, high_corr_dict= orthogonal_analysis(train_factor_ticker)
|
|
455
|
+
orthogonalize_result = orthogonalize(factors, high_corr_dict, resample_summary_train)
|
|
456
|
+
orthogonalize_result.pop('excess_return')
|
|
457
|
+
|
|
458
|
+
orth_train = {factor_name : orth_result.loc[:train_end] for factor_name, orth_result in orthogonalize_result.items()}
|
|
459
|
+
orth_test = {factor_name : orth_result.loc[test_start: ] for factor_name, orth_result in orthogonalize_result.items()}
|
|
460
|
+
|
|
461
|
+
cs_df_orth_train = CS_Information_Correlation(factors = orth_train,
|
|
462
|
+
forward_returns=forward_return_train_stand,
|
|
463
|
+
output_path = 'tmp/ic_test/cs_df_orth_train.parquet')
|
|
464
|
+
#主检验: Newey-West(全日频IC, 修正重叠持有期自相关)
|
|
465
|
+
nw_summary_orth_train = newey_west_summary(cs_df_orth_train)
|
|
466
|
+
multiple_testing_train = multiple_testing(nw_summary_orth_train)
|
|
467
|
+
print("=== Newey-West test ===")
|
|
468
|
+
print(multiple_testing_train)
|
|
469
|
+
print("Number of True values in BH_significant:", multiple_testing_train['BH_significant'].sum())
|
|
470
|
+
|
|
471
|
+
#稳健性对照: 抽样降频(iloc[::period], 保守但有相位依赖)
|
|
472
|
+
resample_summary_orth_train = resample_summary(cs_df_orth_train, periods)
|
|
473
|
+
multiple_testing_resample = multiple_testing(resample_summary_orth_train)
|
|
474
|
+
print("=== Down-sampling control ===")
|
|
475
|
+
print("Number of True values in BH_significant:", multiple_testing_resample['BH_significant'].sum())
|
|
476
|
+
significant_factor = (
|
|
477
|
+
multiple_testing_resample[multiple_testing_resample['BH_significant']]
|
|
478
|
+
.index.get_level_values('factor')
|
|
479
|
+
.unique()
|
|
480
|
+
.tolist()
|
|
481
|
+
)
|
|
482
|
+
significant_factor_nw = (
|
|
483
|
+
multiple_testing_train[multiple_testing_train['BH_significant']]
|
|
484
|
+
.index.get_level_values('factor')
|
|
485
|
+
.unique()
|
|
486
|
+
.tolist()
|
|
487
|
+
)
|
|
488
|
+
rolling_ic_train , acf_train, yearly_train = time_series_stationary_test(cs_df_orth_train)
|
|
489
|
+
|
|
490
|
+
return {"resample_summary_train":resample_summary_train,
|
|
491
|
+
"nw_summary_orth_train": nw_summary_orth_train,
|
|
492
|
+
"multiple_testing_train": multiple_testing_train,
|
|
493
|
+
"multiple_testing_resample": multiple_testing_resample,
|
|
494
|
+
"orth_analysis": orth_analysis,
|
|
495
|
+
"orthogonalize_result_full": orthogonalize_result,
|
|
496
|
+
"orth_factors_test": orth_test,
|
|
497
|
+
"high_corr_dict": high_corr_dict,
|
|
498
|
+
"test_cs_df": test_cs_df,
|
|
499
|
+
"test_factor_ticker": test_factor_ticker,
|
|
500
|
+
"forward_return_train" : forward_return_train,
|
|
501
|
+
"forward_return_train_stand" : forward_return_train_stand,
|
|
502
|
+
"forward_return_test" : forward_return_test,
|
|
503
|
+
"forward_return_test_stand" : forward_return_test_stand,
|
|
504
|
+
"significant_factors": significant_factor,
|
|
505
|
+
"significant_factors_nw": significant_factor_nw,
|
|
506
|
+
"rolling_ic_train": rolling_ic_train,
|
|
507
|
+
"acf_train": acf_train,
|
|
508
|
+
"yearly_train" : yearly_train}
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
if __name__=='__main__':
|
|
513
|
+
|
|
514
|
+
pd.set_option('display.max_rows', None)
|
|
515
|
+
pd.set_option('display.max_columns', None)
|
|
516
|
+
pd.set_option('display.width', None)
|
|
517
|
+
|
|
518
|
+
close_data=pd.read_parquet("data/processed/processed_close.parquet")
|
|
519
|
+
factor_data=pd.read_parquet("tmp/factors/factors.parquet")
|
|
520
|
+
|
|
521
|
+
ticker_list=close_data.columns
|
|
522
|
+
periods=[1,5,20]
|
|
523
|
+
|
|
524
|
+
different_holding_period_df , _ = different_holding_period(close=close_data, tickers = close_data.columns.tolist(), periods=periods)
|
|
525
|
+
# #原始因子算IC,用于正交化
|
|
526
|
+
cs_df = CS_Information_Correlation(factors=factor_data, different_holding_period=different_holding_period_df, output_path="tmp/ic_test/cs_df.parquet")
|
|
527
|
+
|
|
528
|
+
train_end = '2023-12-31'
|
|
529
|
+
test_start = '2024-01-01'
|
|
530
|
+
|
|
531
|
+
train_test_analysis_result = train_test_analysis(cs_df= cs_df, factor_ticker=factor_data ,close = close_data, train_end=train_end, test_start= test_start)
|
|
532
|
+
|
|
533
|
+
# with pd.ExcelWriter('tmp/ic_test/stationary.xlsx') as w:
|
|
534
|
+
# train_test_analysis_result['rolling_ic_train'].to_excel(w, sheet_name="rolling_ic")
|
|
535
|
+
# train_test_analysis_result['acf_train'].to_excel(w,sheet_name="acf")
|
|
536
|
+
# train_test_analysis_result['yearly_train'].to_excel(w, sheet_name = "yearly")
|
|
537
|
+
|
|
538
|
+
# train_test_analysis_result['multiple_testing_train'].to_parquet('tmp/ic_test/significant_test.parquet')
|
|
539
|
+
# print("Done !!!")
|
|
540
|
+
# resample_summary_train = resample_summary(train_cs_df, periods)
|
|
541
|
+
# print(resample_summary_train.index.tolist()[:5])
|
|
542
|
+
|
|
543
|
+
|
quantmine/load_config.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
from .config import CONFIG_REGISTRY
|
|
2
|
+
import yaml
|
|
3
|
+
|
|
4
|
+
def load_configs(yaml_path: str) -> dict:
|
|
5
|
+
"""
|
|
6
|
+
读取YAML文件,返回 {config_name: Config实例} 的字典。
|
|
7
|
+
YAML里没有出现的顶层key,对应的Config类会用全部默认值创建。
|
|
8
|
+
"""
|
|
9
|
+
with open(yaml_path, 'r') as f:
|
|
10
|
+
raw = yaml.safe_load(f) or {} # 如果文件是空的,safe_load返回None,用 or {} 兜底
|
|
11
|
+
|
|
12
|
+
result = {}
|
|
13
|
+
for config_name, config_class in CONFIG_REGISTRY.items():
|
|
14
|
+
kwargs = raw.get(config_name, {}) # YAML里没有这个key,就用空字典(意味着全部用默认值)
|
|
15
|
+
result[config_name] = config_class(**kwargs) # 你来写这一行,回忆之前已经确认过的模式
|
|
16
|
+
return result
|