cape-aeri 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cape_aeri-0.1.0/LICENSE +21 -0
- cape_aeri-0.1.0/PKG-INFO +154 -0
- cape_aeri-0.1.0/README.md +131 -0
- cape_aeri-0.1.0/pyproject.toml +40 -0
- cape_aeri-0.1.0/setup.cfg +4 -0
- cape_aeri-0.1.0/src/aeri/__init__.py +39 -0
- cape_aeri-0.1.0/src/aeri/articles.py +63 -0
- cape_aeri-0.1.0/src/aeri/confidence.py +45 -0
- cape_aeri-0.1.0/src/aeri/config.py +126 -0
- cape_aeri-0.1.0/src/aeri/countries.py +117 -0
- cape_aeri-0.1.0/src/aeri/dictionary.py +114 -0
- cape_aeri-0.1.0/src/aeri/index.py +98 -0
- cape_aeri-0.1.0/src/aeri/match.py +106 -0
- cape_aeri-0.1.0/src/aeri/pillars.py +30 -0
- cape_aeri-0.1.0/src/aeri/relevance.py +35 -0
- cape_aeri-0.1.0/src/aeri/scoring.py +81 -0
- cape_aeri-0.1.0/src/aeri/severity.py +23 -0
- cape_aeri-0.1.0/src/aeri/text.py +34 -0
- cape_aeri-0.1.0/src/cape_aeri.egg-info/PKG-INFO +154 -0
- cape_aeri-0.1.0/src/cape_aeri.egg-info/SOURCES.txt +33 -0
- cape_aeri-0.1.0/src/cape_aeri.egg-info/dependency_links.txt +1 -0
- cape_aeri-0.1.0/src/cape_aeri.egg-info/requires.txt +4 -0
- cape_aeri-0.1.0/src/cape_aeri.egg-info/top_level.txt +1 -0
- cape_aeri-0.1.0/tests/test_articles.py +68 -0
- cape_aeri-0.1.0/tests/test_confidence.py +40 -0
- cape_aeri-0.1.0/tests/test_countries.py +21 -0
- cape_aeri-0.1.0/tests/test_dictionary.py +81 -0
- cape_aeri-0.1.0/tests/test_index.py +86 -0
- cape_aeri-0.1.0/tests/test_integration.py +55 -0
- cape_aeri-0.1.0/tests/test_match.py +42 -0
- cape_aeri-0.1.0/tests/test_pillars.py +41 -0
- cape_aeri-0.1.0/tests/test_relevance.py +38 -0
- cape_aeri-0.1.0/tests/test_scoring.py +84 -0
- cape_aeri-0.1.0/tests/test_severity.py +27 -0
- cape_aeri-0.1.0/tests/test_text.py +24 -0
cape_aeri-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 CAPE Economic Research and Consulting
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
cape_aeri-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cape-aeri
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: AERI article scoring and country-month economic-risk index library.
|
|
5
|
+
Author: Shakir Adeyemi Tewogbade
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/CAPE-ERC/aeri-library
|
|
8
|
+
Project-URL: Issues, https://github.com/CAPE-ERC/aeri-library/issues
|
|
9
|
+
Keywords: economics,news,risk-index,africa,nlp
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: pandas>=1.5
|
|
20
|
+
Provides-Extra: test
|
|
21
|
+
Requires-Dist: pytest>=7; extra == "test"
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
# aeri
|
|
25
|
+
|
|
26
|
+
AERI article scoring and country-month economic-risk index library, built
|
|
27
|
+
from the AERI/ARS work at CAPE Economic Research and Consulting.
|
|
28
|
+
|
|
29
|
+
This first release covers **Library 1** from the proposal: scoring each
|
|
30
|
+
scraped news article and rolling the scores up into the country-month
|
|
31
|
+
AERI (Africa Economic Risk Index). Pillar composite indices, the ARS
|
|
32
|
+
composite score and early-warning alerts (Libraries 2-4) are planned as
|
|
33
|
+
follow-up PRs on top of this foundation. Matching conventions, data
|
|
34
|
+
layout and output schema are kept consistent with the sibling
|
|
35
|
+
[`newsindex`](https://github.com/CAPE-ERC/newsindex-library) (EPU) library
|
|
36
|
+
and the production AERI Pillar API, so the CAPE libraries interoperate.
|
|
37
|
+
|
|
38
|
+
## Install
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
pip install cape-aeri # the import name is still `aeri`: import aeri
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
For development:
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
pip install -e ".[test]"
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## Quickstart
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
import aeri
|
|
54
|
+
|
|
55
|
+
dictionary = aeri.dictionary.load("data/aeri_country_keywords.json")
|
|
56
|
+
articles = aeri.read_articles("path/to/Shak_Scrapped_Top_Articles")
|
|
57
|
+
|
|
58
|
+
scored = aeri.score_articles(articles, dictionary)
|
|
59
|
+
|
|
60
|
+
aeri_df = aeri.compute_aeri(scored)
|
|
61
|
+
diag = aeri.diagnostics(scored)
|
|
62
|
+
aeri_df = aeri.coverage_filter(aeri_df, diag) # drop thin country-months
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
## Data format
|
|
66
|
+
|
|
67
|
+
`read_articles()` expects the on-disk layout produced by CAPE's scraper:
|
|
68
|
+
|
|
69
|
+
```
|
|
70
|
+
<root>/<Country>/<Publisher>/<YYYY-MM>/articles_<Publisher>_<YYYY-MM>.csv(.gz)
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Each CSV has columns `site`, `date`, `title`, `url`, `text`. `country` is
|
|
74
|
+
not a column -- it's the top-level folder name, added by `read_articles()`
|
|
75
|
+
along with `year_month`.
|
|
76
|
+
|
|
77
|
+
`read_articles()` does **not** auto-canonicalize country folder names
|
|
78
|
+
(matching `newsindex`'s loader). A folder like `Morocco_new` is kept
|
|
79
|
+
distinct from `Morocco` rather than silently merged -- it may be a
|
|
80
|
+
deliberately separate re-scrape. Apply `aeri.countries.canonicalize()`
|
|
81
|
+
yourself once you've decided whether/how to merge it.
|
|
82
|
+
|
|
83
|
+
## Scoring pipeline
|
|
84
|
+
|
|
85
|
+
| Function | What it does |
|
|
86
|
+
| --- | --- |
|
|
87
|
+
| `normalize_text()` | Strip HTML; standardise case, whitespace and Unicode |
|
|
88
|
+
| `is_relevant()` | Economic relevance flag `R_i`, per the article's country dictionary |
|
|
89
|
+
| `severity_tier()` | Ordinal severity `S_i` from 0 to 4; the highest tier matched wins |
|
|
90
|
+
| `confidence_weight()` | `W_i = clip(1 + 0.5*TitleHint + 0.2*LongText + 0.3*MultiPillar, 1, 2)` |
|
|
91
|
+
| `assign_pillars()` | Flags for the 8 pillars; one article can activate several |
|
|
92
|
+
| `article_score()` | `A_i = R_i * S_i * W_i` |
|
|
93
|
+
| `compute_aeri()` | `AERI = 100 * sum(A_i) / N_t` for each country and month |
|
|
94
|
+
| `diagnostics()` | Total/relevant articles, relevant share, average confidence, average severity among relevant articles |
|
|
95
|
+
| `coverage_filter()` | Keeps country-months with at least 100 articles |
|
|
96
|
+
|
|
97
|
+
`score_articles()` runs the per-article functions over an articles
|
|
98
|
+
DataFrame in one pass. Relevance, severity and pillar matching run on the
|
|
99
|
+
article's **title and body concatenated** (`x_i = title + " " + text`),
|
|
100
|
+
and matching is **word-boundary aware** (`aeri.match`), not substring
|
|
101
|
+
search -- e.g. the severity term `"war"` does not match inside `"warning"`
|
|
102
|
+
or `"forward"`. `compute_aeri()` / `diagnostics()` output `Country`,
|
|
103
|
+
`Period` (`YYYY-MM`), `AERI`, `TotalArticles`, `RelevantArticles`,
|
|
104
|
+
`RelevantSharePct`, `AvgConfidence`, `AvgSeverityAmongRelevant` -- the
|
|
105
|
+
same column names as the production `aeri_country_period.csv` /
|
|
106
|
+
`aeri_diagnostics.csv` already consumed by the AERI Pillar API.
|
|
107
|
+
|
|
108
|
+
The 8 pillars (`aeri.config.PILLARS`) are named to match the Pillar API's
|
|
109
|
+
catalog: `InflationRisk`, `FXExternalRisk`, `DebtFiscalRisk`,
|
|
110
|
+
`FinancialSectorRisk`, `GrowthRealEconomyRisk`, `CommoditySupplyRisk`,
|
|
111
|
+
`PoliticalEconomyRisk`, `ClimateDisasterRisk`.
|
|
112
|
+
|
|
113
|
+
## Dictionaries
|
|
114
|
+
|
|
115
|
+
`data/aeri_country_keywords.json` is the real, production EPTU-schema
|
|
116
|
+
keyword dictionary (54 African countries, Economic/Policy/Uncertainty
|
|
117
|
+
term lists) shared with `newsindex`. `is_relevant()` defaults to the
|
|
118
|
+
Economic component alone for `R_i` -- **this default is an assumption**;
|
|
119
|
+
no source document specified whether Policy terms should also count
|
|
120
|
+
toward relevance. Add `"policy"` via `AERIConfig.relevance_components` to
|
|
121
|
+
include them:
|
|
122
|
+
|
|
123
|
+
```python
|
|
124
|
+
from aeri import AERIConfig
|
|
125
|
+
config = AERIConfig(relevance_components=("economic", "policy"))
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
The **severity-tier and pillar keyword lists have no equivalent
|
|
129
|
+
production source** -- nothing in the underlying research materials
|
|
130
|
+
defines them as keyword lists, only conceptually ("mild concern" through
|
|
131
|
+
"crisis-level language"). `aeri.config.DEFAULT_SEVERITY_KEYWORDS` and
|
|
132
|
+
`DEFAULT_PILLAR_KEYWORDS` are a first-pass, hand-curated starting point
|
|
133
|
+
that needs calibration before being treated as final. Every scoring
|
|
134
|
+
function takes an `AERIConfig` to override them without forking the
|
|
135
|
+
library.
|
|
136
|
+
|
|
137
|
+
The `TitleHint`/`LongText`/`MultiPillar` thresholds inside
|
|
138
|
+
`confidence_weight()` (title relevance; >=300 words; >=2 active pillars)
|
|
139
|
+
are likewise placeholders -- the exact numeric cutoffs used in the
|
|
140
|
+
original research were not found in any available source and need
|
|
141
|
+
confirmation.
|
|
142
|
+
|
|
143
|
+
## Roadmap
|
|
144
|
+
|
|
145
|
+
- Pillar composite indices and PCA-combined pillar measure (Library 2)
|
|
146
|
+
- `ars` composite score (Library 3): `ARS = 100 * (0.40*Intensity + 0.25*Breadth + 0.20*Severity + 0.15*Structure)`, risk bands at 20/40/60/80
|
|
147
|
+
- `riskalert` early-warning alerts (Library 4)
|
|
148
|
+
|
|
149
|
+
## Development
|
|
150
|
+
|
|
151
|
+
```bash
|
|
152
|
+
pip install -e ".[test]"
|
|
153
|
+
pytest
|
|
154
|
+
```
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
# aeri
|
|
2
|
+
|
|
3
|
+
AERI article scoring and country-month economic-risk index library, built
|
|
4
|
+
from the AERI/ARS work at CAPE Economic Research and Consulting.
|
|
5
|
+
|
|
6
|
+
This first release covers **Library 1** from the proposal: scoring each
|
|
7
|
+
scraped news article and rolling the scores up into the country-month
|
|
8
|
+
AERI (Africa Economic Risk Index). Pillar composite indices, the ARS
|
|
9
|
+
composite score and early-warning alerts (Libraries 2-4) are planned as
|
|
10
|
+
follow-up PRs on top of this foundation. Matching conventions, data
|
|
11
|
+
layout and output schema are kept consistent with the sibling
|
|
12
|
+
[`newsindex`](https://github.com/CAPE-ERC/newsindex-library) (EPU) library
|
|
13
|
+
and the production AERI Pillar API, so the CAPE libraries interoperate.
|
|
14
|
+
|
|
15
|
+
## Install
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
pip install cape-aeri # the import name is still `aeri`: import aeri
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
For development:
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pip install -e ".[test]"
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
## Quickstart
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
import aeri
|
|
31
|
+
|
|
32
|
+
dictionary = aeri.dictionary.load("data/aeri_country_keywords.json")
|
|
33
|
+
articles = aeri.read_articles("path/to/Shak_Scrapped_Top_Articles")
|
|
34
|
+
|
|
35
|
+
scored = aeri.score_articles(articles, dictionary)
|
|
36
|
+
|
|
37
|
+
aeri_df = aeri.compute_aeri(scored)
|
|
38
|
+
diag = aeri.diagnostics(scored)
|
|
39
|
+
aeri_df = aeri.coverage_filter(aeri_df, diag) # drop thin country-months
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Data format
|
|
43
|
+
|
|
44
|
+
`read_articles()` expects the on-disk layout produced by CAPE's scraper:
|
|
45
|
+
|
|
46
|
+
```
|
|
47
|
+
<root>/<Country>/<Publisher>/<YYYY-MM>/articles_<Publisher>_<YYYY-MM>.csv(.gz)
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Each CSV has columns `site`, `date`, `title`, `url`, `text`. `country` is
|
|
51
|
+
not a column -- it's the top-level folder name, added by `read_articles()`
|
|
52
|
+
along with `year_month`.
|
|
53
|
+
|
|
54
|
+
`read_articles()` does **not** auto-canonicalize country folder names
|
|
55
|
+
(matching `newsindex`'s loader). A folder like `Morocco_new` is kept
|
|
56
|
+
distinct from `Morocco` rather than silently merged -- it may be a
|
|
57
|
+
deliberately separate re-scrape. Apply `aeri.countries.canonicalize()`
|
|
58
|
+
yourself once you've decided whether/how to merge it.
|
|
59
|
+
|
|
60
|
+
## Scoring pipeline
|
|
61
|
+
|
|
62
|
+
| Function | What it does |
|
|
63
|
+
| --- | --- |
|
|
64
|
+
| `normalize_text()` | Strip HTML; standardise case, whitespace and Unicode |
|
|
65
|
+
| `is_relevant()` | Economic relevance flag `R_i`, per the article's country dictionary |
|
|
66
|
+
| `severity_tier()` | Ordinal severity `S_i` from 0 to 4; the highest tier matched wins |
|
|
67
|
+
| `confidence_weight()` | `W_i = clip(1 + 0.5*TitleHint + 0.2*LongText + 0.3*MultiPillar, 1, 2)` |
|
|
68
|
+
| `assign_pillars()` | Flags for the 8 pillars; one article can activate several |
|
|
69
|
+
| `article_score()` | `A_i = R_i * S_i * W_i` |
|
|
70
|
+
| `compute_aeri()` | `AERI = 100 * sum(A_i) / N_t` for each country and month |
|
|
71
|
+
| `diagnostics()` | Total/relevant articles, relevant share, average confidence, average severity among relevant articles |
|
|
72
|
+
| `coverage_filter()` | Keeps country-months with at least 100 articles |
|
|
73
|
+
|
|
74
|
+
`score_articles()` runs the per-article functions over an articles
|
|
75
|
+
DataFrame in one pass. Relevance, severity and pillar matching run on the
|
|
76
|
+
article's **title and body concatenated** (`x_i = title + " " + text`),
|
|
77
|
+
and matching is **word-boundary aware** (`aeri.match`), not substring
|
|
78
|
+
search -- e.g. the severity term `"war"` does not match inside `"warning"`
|
|
79
|
+
or `"forward"`. `compute_aeri()` / `diagnostics()` output `Country`,
|
|
80
|
+
`Period` (`YYYY-MM`), `AERI`, `TotalArticles`, `RelevantArticles`,
|
|
81
|
+
`RelevantSharePct`, `AvgConfidence`, `AvgSeverityAmongRelevant` -- the
|
|
82
|
+
same column names as the production `aeri_country_period.csv` /
|
|
83
|
+
`aeri_diagnostics.csv` already consumed by the AERI Pillar API.
|
|
84
|
+
|
|
85
|
+
The 8 pillars (`aeri.config.PILLARS`) are named to match the Pillar API's
|
|
86
|
+
catalog: `InflationRisk`, `FXExternalRisk`, `DebtFiscalRisk`,
|
|
87
|
+
`FinancialSectorRisk`, `GrowthRealEconomyRisk`, `CommoditySupplyRisk`,
|
|
88
|
+
`PoliticalEconomyRisk`, `ClimateDisasterRisk`.
|
|
89
|
+
|
|
90
|
+
## Dictionaries
|
|
91
|
+
|
|
92
|
+
`data/aeri_country_keywords.json` is the real, production EPTU-schema
|
|
93
|
+
keyword dictionary (54 African countries, Economic/Policy/Uncertainty
|
|
94
|
+
term lists) shared with `newsindex`. `is_relevant()` defaults to the
|
|
95
|
+
Economic component alone for `R_i` -- **this default is an assumption**;
|
|
96
|
+
no source document specified whether Policy terms should also count
|
|
97
|
+
toward relevance. Add `"policy"` via `AERIConfig.relevance_components` to
|
|
98
|
+
include them:
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
from aeri import AERIConfig
|
|
102
|
+
config = AERIConfig(relevance_components=("economic", "policy"))
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
The **severity-tier and pillar keyword lists have no equivalent
|
|
106
|
+
production source** -- nothing in the underlying research materials
|
|
107
|
+
defines them as keyword lists, only conceptually ("mild concern" through
|
|
108
|
+
"crisis-level language"). `aeri.config.DEFAULT_SEVERITY_KEYWORDS` and
|
|
109
|
+
`DEFAULT_PILLAR_KEYWORDS` are a first-pass, hand-curated starting point
|
|
110
|
+
that needs calibration before being treated as final. Every scoring
|
|
111
|
+
function takes an `AERIConfig` to override them without forking the
|
|
112
|
+
library.
|
|
113
|
+
|
|
114
|
+
The `TitleHint`/`LongText`/`MultiPillar` thresholds inside
|
|
115
|
+
`confidence_weight()` (title relevance; >=300 words; >=2 active pillars)
|
|
116
|
+
are likewise placeholders -- the exact numeric cutoffs used in the
|
|
117
|
+
original research were not found in any available source and need
|
|
118
|
+
confirmation.
|
|
119
|
+
|
|
120
|
+
## Roadmap
|
|
121
|
+
|
|
122
|
+
- Pillar composite indices and PCA-combined pillar measure (Library 2)
|
|
123
|
+
- `ars` composite score (Library 3): `ARS = 100 * (0.40*Intensity + 0.25*Breadth + 0.20*Severity + 0.15*Structure)`, risk bands at 20/40/60/80
|
|
124
|
+
- `riskalert` early-warning alerts (Library 4)
|
|
125
|
+
|
|
126
|
+
## Development
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
pip install -e ".[test]"
|
|
130
|
+
pytest
|
|
131
|
+
```
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "cape-aeri"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "AERI article scoring and country-month economic-risk index library."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "Shakir Adeyemi Tewogbade" },
|
|
14
|
+
]
|
|
15
|
+
keywords = ["economics", "news", "risk-index", "africa", "nlp"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 3 - Alpha",
|
|
18
|
+
"Intended Audience :: Science/Research",
|
|
19
|
+
"License :: OSI Approved :: MIT License",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.10",
|
|
22
|
+
"Topic :: Scientific/Engineering",
|
|
23
|
+
]
|
|
24
|
+
dependencies = [
|
|
25
|
+
"pandas>=1.5",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[project.optional-dependencies]
|
|
29
|
+
test = ["pytest>=7"]
|
|
30
|
+
|
|
31
|
+
[project.urls]
|
|
32
|
+
Homepage = "https://github.com/CAPE-ERC/aeri-library"
|
|
33
|
+
Issues = "https://github.com/CAPE-ERC/aeri-library/issues"
|
|
34
|
+
|
|
35
|
+
[tool.setuptools.packages.find]
|
|
36
|
+
where = ["src"]
|
|
37
|
+
|
|
38
|
+
[tool.pytest.ini_options]
|
|
39
|
+
testpaths = ["tests"]
|
|
40
|
+
pythonpath = ["src"]
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""aeri: AERI article scoring and country-month economic-risk index library."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from . import config, countries, dictionary, match
|
|
6
|
+
from .articles import read_articles
|
|
7
|
+
from .config import DEFAULT_CONFIG, PILLARS, AERIConfig
|
|
8
|
+
from .confidence import confidence_weight
|
|
9
|
+
from .index import compute_aeri, coverage_filter, diagnostics
|
|
10
|
+
from .pillars import active_pillar_count, assign_pillars
|
|
11
|
+
from .relevance import is_relevant
|
|
12
|
+
from .scoring import article_score, score_articles
|
|
13
|
+
from .severity import severity_tier
|
|
14
|
+
from .text import normalize_text
|
|
15
|
+
|
|
16
|
+
__version__ = "0.1.0"
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
"config",
|
|
20
|
+
"countries",
|
|
21
|
+
"dictionary",
|
|
22
|
+
"match",
|
|
23
|
+
"AERIConfig",
|
|
24
|
+
"DEFAULT_CONFIG",
|
|
25
|
+
"PILLARS",
|
|
26
|
+
"normalize_text",
|
|
27
|
+
"is_relevant",
|
|
28
|
+
"severity_tier",
|
|
29
|
+
"confidence_weight",
|
|
30
|
+
"assign_pillars",
|
|
31
|
+
"active_pillar_count",
|
|
32
|
+
"article_score",
|
|
33
|
+
"score_articles",
|
|
34
|
+
"compute_aeri",
|
|
35
|
+
"diagnostics",
|
|
36
|
+
"coverage_filter",
|
|
37
|
+
"read_articles",
|
|
38
|
+
"__version__",
|
|
39
|
+
]
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Read scraped article archives into a single table.
|
|
2
|
+
|
|
3
|
+
Expects the on-disk layout produced by CAPE's scraper: ``<root>/<country>/
|
|
4
|
+
<newspaper>/<year-month>/articles_<newspaper>_<year-month>.csv(.gz)``, each
|
|
5
|
+
file holding the columns ``site``, ``date``, ``title``, ``url``, ``text``.
|
|
6
|
+
Matches the loader in the sibling ``newsindex`` package so both libraries
|
|
7
|
+
read the same archive the same way.
|
|
8
|
+
|
|
9
|
+
The inferred ``country`` is the raw top-level folder name -- it is **not**
|
|
10
|
+
auto-canonicalized against :mod:`aeri.countries`, because a folder like
|
|
11
|
+
``"Morocco_new"`` may be a deliberately separate corpus (a re-scrape) and
|
|
12
|
+
silently merging it into ``"Morocco"`` could conflate two different data
|
|
13
|
+
sources. Apply :func:`aeri.countries.canonicalize` yourself once you've
|
|
14
|
+
decided that's the right thing to do for your archive.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import html
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
import pandas as pd
|
|
23
|
+
|
|
24
|
+
_REQUIRED_COLUMNS = ("site", "date", "title", "url", "text")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _unescape(value: object) -> object:
|
|
28
|
+
if not isinstance(value, str):
|
|
29
|
+
return value
|
|
30
|
+
return html.unescape(value)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def read_articles(root: str | Path, *, encoding: str = "utf-8") -> pd.DataFrame:
|
|
34
|
+
"""Read every ``articles_*.csv`` / ``articles_*.csv.gz`` file under
|
|
35
|
+
``root`` into one table, adding ``country`` (the top-level folder name)
|
|
36
|
+
and ``year_month`` (the first seven characters of ``date``) columns.
|
|
37
|
+
|
|
38
|
+
Article titles and text are HTML-unescaped (the scraper stores raw
|
|
39
|
+
HTML entities such as ``’``) before they reach `aeri.match`.
|
|
40
|
+
"""
|
|
41
|
+
root = Path(root)
|
|
42
|
+
files = sorted({*root.glob("**/*.csv.gz"), *root.glob("**/*.csv")})
|
|
43
|
+
if not files:
|
|
44
|
+
raise FileNotFoundError(f"no article files found under {root}")
|
|
45
|
+
|
|
46
|
+
frames: list[pd.DataFrame] = []
|
|
47
|
+
for file in files:
|
|
48
|
+
country = file.relative_to(root).parts[0]
|
|
49
|
+
frame = pd.read_csv(file, encoding=encoding, compression="infer")
|
|
50
|
+
missing = [col for col in _REQUIRED_COLUMNS if col not in frame.columns]
|
|
51
|
+
if missing:
|
|
52
|
+
raise ValueError(f"{file}: missing column(s) {missing}")
|
|
53
|
+
frame["country"] = country
|
|
54
|
+
frame["source_file"] = str(file)
|
|
55
|
+
frames.append(frame)
|
|
56
|
+
|
|
57
|
+
combined = pd.concat(frames, ignore_index=True)
|
|
58
|
+
combined["title"] = combined["title"].map(_unescape)
|
|
59
|
+
combined["text"] = combined["text"].map(_unescape)
|
|
60
|
+
combined["year_month"] = combined["date"].astype(str).str.slice(0, 7)
|
|
61
|
+
return combined[
|
|
62
|
+
["country", "site", "year_month", "date", "title", "url", "text", "source_file"]
|
|
63
|
+
]
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""Confidence weighting for article scores."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from .config import DEFAULT_CONFIG, AERIConfig
|
|
6
|
+
from .dictionary import CountryDictionary
|
|
7
|
+
from .pillars import active_pillar_count, assign_pillars
|
|
8
|
+
from .relevance import is_relevant
|
|
9
|
+
from .text import normalize_text, word_count
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def confidence_weight(
|
|
13
|
+
title: str,
|
|
14
|
+
text: str,
|
|
15
|
+
entry: CountryDictionary,
|
|
16
|
+
pillar_flags: dict[str, bool] | None = None,
|
|
17
|
+
config: AERIConfig = DEFAULT_CONFIG,
|
|
18
|
+
) -> float:
|
|
19
|
+
"""W_i = clip(1 + 0.5*TitleHint + 0.2*LongText + 0.3*MultiPillar, 1, 2).
|
|
20
|
+
|
|
21
|
+
- TitleHint: the title itself carries an economic-relevance signal
|
|
22
|
+
(per the country's dictionary).
|
|
23
|
+
- LongText: the article body is at least `long_text_word_threshold`
|
|
24
|
+
words (normalized word count). This threshold is a placeholder --
|
|
25
|
+
no source document gave the real cutoff; see README.
|
|
26
|
+
- MultiPillar: the article activates `multi_pillar_threshold` or more
|
|
27
|
+
of the 8 pillars.
|
|
28
|
+
|
|
29
|
+
`pillar_flags` can be passed in to reuse an `assign_pillars()` result
|
|
30
|
+
already computed for this article; otherwise it is derived from `text`.
|
|
31
|
+
"""
|
|
32
|
+
title_hint = 1.0 if is_relevant(title, entry, config) else 0.0
|
|
33
|
+
|
|
34
|
+
normalized_text = normalize_text(text)
|
|
35
|
+
long_text = 1.0 if word_count(normalized_text) >= config.long_text_word_threshold else 0.0
|
|
36
|
+
|
|
37
|
+
if pillar_flags is None:
|
|
38
|
+
pillar_flags = assign_pillars(text, config)
|
|
39
|
+
multi_pillar = (
|
|
40
|
+
1.0 if active_pillar_count(pillar_flags) >= config.multi_pillar_threshold else 0.0
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
weight = 1.0 + 0.5 * title_hint + 0.2 * long_text + 0.3 * multi_pillar
|
|
44
|
+
low, high = config.confidence_bounds
|
|
45
|
+
return min(max(weight, low), high)
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""Configuration and default keyword sets for AERI scoring.
|
|
2
|
+
|
|
3
|
+
The economic-relevance dictionary is real production data loaded
|
|
4
|
+
separately via :mod:`aeri.dictionary` (country-specific, see
|
|
5
|
+
``data/aeri_country_keywords.json``). The severity-tier and pillar
|
|
6
|
+
keyword sets below have **no equivalent production source** -- no
|
|
7
|
+
dictionary for them was found anywhere in the underlying research
|
|
8
|
+
materials -- so these are a first-pass, hand-curated starting point that
|
|
9
|
+
needs calibration against real coverage before being treated as final.
|
|
10
|
+
Every scoring function accepts an `AERIConfig` so a project can supply
|
|
11
|
+
its own keyword sets without forking the library.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from typing import Mapping, Sequence
|
|
18
|
+
|
|
19
|
+
#: The 8 AERI risk pillars, named to match the production AERI Pillar API
|
|
20
|
+
#: catalog (`InflationRisk`, `FXExternalRisk`, ...) and the
|
|
21
|
+
#: `aeri_pillars_country_period.csv` column headers it publishes.
|
|
22
|
+
PILLARS: tuple[str, ...] = (
|
|
23
|
+
"InflationRisk",
|
|
24
|
+
"FXExternalRisk",
|
|
25
|
+
"DebtFiscalRisk",
|
|
26
|
+
"FinancialSectorRisk",
|
|
27
|
+
"GrowthRealEconomyRisk",
|
|
28
|
+
"CommoditySupplyRisk",
|
|
29
|
+
"PoliticalEconomyRisk",
|
|
30
|
+
"ClimateDisasterRisk",
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
DEFAULT_SEVERITY_KEYWORDS: dict[int, tuple[str, ...]] = {
|
|
34
|
+
1: (
|
|
35
|
+
"concern", "warning", "pressure", "risk", "slowdown", "softening",
|
|
36
|
+
"worry", "caution", "uncertainty",
|
|
37
|
+
),
|
|
38
|
+
2: (
|
|
39
|
+
"falling", "decline", "drop", "shortage", "strain", "weakening",
|
|
40
|
+
"job losses", "cutbacks", "downgrade", "rising prices", "protest",
|
|
41
|
+
"strike",
|
|
42
|
+
),
|
|
43
|
+
3: (
|
|
44
|
+
"crisis", "collapse", "plunge", "default", "devaluation", "emergency",
|
|
45
|
+
"unrest", "shutdown", "severe shortage", "mass layoffs", "coup",
|
|
46
|
+
"credit downgrade",
|
|
47
|
+
),
|
|
48
|
+
4: (
|
|
49
|
+
"catastrophe", "famine", "hyperinflation", "economic collapse",
|
|
50
|
+
"state of emergency", "war", "coup d'etat", "humanitarian crisis",
|
|
51
|
+
"bankruptcy", "sovereign default", "civil war", "mass starvation",
|
|
52
|
+
"currency collapse",
|
|
53
|
+
),
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
DEFAULT_PILLAR_KEYWORDS: dict[str, tuple[str, ...]] = {
|
|
57
|
+
"InflationRisk": (
|
|
58
|
+
"inflation", "deflation", "consumer price", "cost of living",
|
|
59
|
+
"price hike", "price increase", "food price", "fuel price", "cpi",
|
|
60
|
+
"disinflation", "hyperinflation", "rising prices",
|
|
61
|
+
),
|
|
62
|
+
"FXExternalRisk": (
|
|
63
|
+
"exchange rate", "devaluation", "depreciation", "currency", "forex",
|
|
64
|
+
"foreign reserves", "balance of payments", "current account",
|
|
65
|
+
"trade deficit", "capital flight", "remittance", "export", "import",
|
|
66
|
+
),
|
|
67
|
+
"DebtFiscalRisk": (
|
|
68
|
+
"public debt", "sovereign debt", "fiscal deficit", "budget deficit",
|
|
69
|
+
"treasury", "government spending", "austerity", "imf", "world bank",
|
|
70
|
+
"bailout", "credit rating", "default", "bond", "eurobond",
|
|
71
|
+
"tax revenue",
|
|
72
|
+
),
|
|
73
|
+
"FinancialSectorRisk": (
|
|
74
|
+
"bank", "banking sector", "interest rate", "central bank",
|
|
75
|
+
"monetary policy", "non-performing loan", "liquidity",
|
|
76
|
+
"credit crunch", "stock exchange", "bond market", "financial sector",
|
|
77
|
+
"lending rate",
|
|
78
|
+
),
|
|
79
|
+
"GrowthRealEconomyRisk": (
|
|
80
|
+
"gdp", "economic growth", "recession", "unemployment", "job losses",
|
|
81
|
+
"manufacturing", "industrial output", "retail sales",
|
|
82
|
+
"consumer spending", "business confidence", "economic slowdown",
|
|
83
|
+
),
|
|
84
|
+
"CommoditySupplyRisk": (
|
|
85
|
+
"commodity price", "oil price", "gas price", "mining", "agriculture",
|
|
86
|
+
"harvest", "food supply", "supply chain", "shortage",
|
|
87
|
+
"import dependency", "wheat", "maize", "fertilizer",
|
|
88
|
+
),
|
|
89
|
+
"PoliticalEconomyRisk": (
|
|
90
|
+
"election", "coup", "protest", "strike", "sanctions", "corruption",
|
|
91
|
+
"policy reform", "political instability", "government shutdown",
|
|
92
|
+
"regulation", "nationalization", "unrest",
|
|
93
|
+
),
|
|
94
|
+
"ClimateDisasterRisk": (
|
|
95
|
+
"drought", "flood", "famine", "climate change", "natural disaster",
|
|
96
|
+
"cyclone", "locust", "food insecurity", "humanitarian crisis",
|
|
97
|
+
"displacement", "crop failure",
|
|
98
|
+
),
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@dataclass(frozen=True)
|
|
103
|
+
class AERIConfig:
|
|
104
|
+
"""Bundles the thresholds and (non-relevance) keyword sets used across
|
|
105
|
+
the pipeline. Severity and pillar keywords default to the first-pass
|
|
106
|
+
sets above; relevance is intentionally not part of this config -- it
|
|
107
|
+
comes from a country-keyed :class:`aeri.dictionary.Dictionary`.
|
|
108
|
+
"""
|
|
109
|
+
|
|
110
|
+
severity_keywords: Mapping[int, Sequence[str]] = field(
|
|
111
|
+
default_factory=lambda: DEFAULT_SEVERITY_KEYWORDS
|
|
112
|
+
)
|
|
113
|
+
pillar_keywords: Mapping[str, Sequence[str]] = field(
|
|
114
|
+
default_factory=lambda: DEFAULT_PILLAR_KEYWORDS
|
|
115
|
+
)
|
|
116
|
+
long_text_word_threshold: int = 300
|
|
117
|
+
multi_pillar_threshold: int = 2
|
|
118
|
+
confidence_bounds: tuple[float, float] = (1.0, 2.0)
|
|
119
|
+
coverage_min_articles: int = 100
|
|
120
|
+
#: Which dictionary component(s) count toward R_i. Defaults to the
|
|
121
|
+
#: Economic component alone; add "policy" to also count policy terms.
|
|
122
|
+
#: Unverified against source -- see README.
|
|
123
|
+
relevance_components: tuple[str, ...] = ("economic",)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
DEFAULT_CONFIG = AERIConfig()
|