cape-aeri 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. cape_aeri-0.1.0/LICENSE +21 -0
  2. cape_aeri-0.1.0/PKG-INFO +154 -0
  3. cape_aeri-0.1.0/README.md +131 -0
  4. cape_aeri-0.1.0/pyproject.toml +40 -0
  5. cape_aeri-0.1.0/setup.cfg +4 -0
  6. cape_aeri-0.1.0/src/aeri/__init__.py +39 -0
  7. cape_aeri-0.1.0/src/aeri/articles.py +63 -0
  8. cape_aeri-0.1.0/src/aeri/confidence.py +45 -0
  9. cape_aeri-0.1.0/src/aeri/config.py +126 -0
  10. cape_aeri-0.1.0/src/aeri/countries.py +117 -0
  11. cape_aeri-0.1.0/src/aeri/dictionary.py +114 -0
  12. cape_aeri-0.1.0/src/aeri/index.py +98 -0
  13. cape_aeri-0.1.0/src/aeri/match.py +106 -0
  14. cape_aeri-0.1.0/src/aeri/pillars.py +30 -0
  15. cape_aeri-0.1.0/src/aeri/relevance.py +35 -0
  16. cape_aeri-0.1.0/src/aeri/scoring.py +81 -0
  17. cape_aeri-0.1.0/src/aeri/severity.py +23 -0
  18. cape_aeri-0.1.0/src/aeri/text.py +34 -0
  19. cape_aeri-0.1.0/src/cape_aeri.egg-info/PKG-INFO +154 -0
  20. cape_aeri-0.1.0/src/cape_aeri.egg-info/SOURCES.txt +33 -0
  21. cape_aeri-0.1.0/src/cape_aeri.egg-info/dependency_links.txt +1 -0
  22. cape_aeri-0.1.0/src/cape_aeri.egg-info/requires.txt +4 -0
  23. cape_aeri-0.1.0/src/cape_aeri.egg-info/top_level.txt +1 -0
  24. cape_aeri-0.1.0/tests/test_articles.py +68 -0
  25. cape_aeri-0.1.0/tests/test_confidence.py +40 -0
  26. cape_aeri-0.1.0/tests/test_countries.py +21 -0
  27. cape_aeri-0.1.0/tests/test_dictionary.py +81 -0
  28. cape_aeri-0.1.0/tests/test_index.py +86 -0
  29. cape_aeri-0.1.0/tests/test_integration.py +55 -0
  30. cape_aeri-0.1.0/tests/test_match.py +42 -0
  31. cape_aeri-0.1.0/tests/test_pillars.py +41 -0
  32. cape_aeri-0.1.0/tests/test_relevance.py +38 -0
  33. cape_aeri-0.1.0/tests/test_scoring.py +84 -0
  34. cape_aeri-0.1.0/tests/test_severity.py +27 -0
  35. cape_aeri-0.1.0/tests/test_text.py +24 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 CAPE Economic Research and Consulting
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,154 @@
1
+ Metadata-Version: 2.4
2
+ Name: cape-aeri
3
+ Version: 0.1.0
4
+ Summary: AERI article scoring and country-month economic-risk index library.
5
+ Author: Shakir Adeyemi Tewogbade
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/CAPE-ERC/aeri-library
8
+ Project-URL: Issues, https://github.com/CAPE-ERC/aeri-library/issues
9
+ Keywords: economics,news,risk-index,africa,nlp
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Intended Audience :: Science/Research
12
+ Classifier: License :: OSI Approved :: MIT License
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Programming Language :: Python :: 3.10
15
+ Classifier: Topic :: Scientific/Engineering
16
+ Requires-Python: >=3.10
17
+ Description-Content-Type: text/markdown
18
+ License-File: LICENSE
19
+ Requires-Dist: pandas>=1.5
20
+ Provides-Extra: test
21
+ Requires-Dist: pytest>=7; extra == "test"
22
+ Dynamic: license-file
23
+
24
+ # aeri
25
+
26
+ AERI article scoring and country-month economic-risk index library, built
27
+ from the AERI/ARS work at CAPE Economic Research and Consulting.
28
+
29
+ This first release covers **Library 1** from the proposal: scoring each
30
+ scraped news article and rolling the scores up into the country-month
31
+ AERI (Africa Economic Risk Index). Pillar composite indices, the ARS
32
+ composite score and early-warning alerts (Libraries 2-4) are planned as
33
+ follow-up PRs on top of this foundation. Matching conventions, data
34
+ layout and output schema are kept consistent with the sibling
35
+ [`newsindex`](https://github.com/CAPE-ERC/newsindex-library) (EPU) library
36
+ and the production AERI Pillar API, so the CAPE libraries interoperate.
37
+
38
+ ## Install
39
+
40
+ ```bash
41
+ pip install cape-aeri # the import name is still `aeri`: import aeri
42
+ ```
43
+
44
+ For development:
45
+
46
+ ```bash
47
+ pip install -e ".[test]"
48
+ ```
49
+
50
+ ## Quickstart
51
+
52
+ ```python
53
+ import aeri
54
+
55
+ dictionary = aeri.dictionary.load("data/aeri_country_keywords.json")
56
+ articles = aeri.read_articles("path/to/Shak_Scrapped_Top_Articles")
57
+
58
+ scored = aeri.score_articles(articles, dictionary)
59
+
60
+ aeri_df = aeri.compute_aeri(scored)
61
+ diag = aeri.diagnostics(scored)
62
+ aeri_df = aeri.coverage_filter(aeri_df, diag) # drop thin country-months
63
+ ```
64
+
65
+ ## Data format
66
+
67
+ `read_articles()` expects the on-disk layout produced by CAPE's scraper:
68
+
69
+ ```
70
+ <root>/<Country>/<Publisher>/<YYYY-MM>/articles_<Publisher>_<YYYY-MM>.csv(.gz)
71
+ ```
72
+
73
+ Each CSV has columns `site`, `date`, `title`, `url`, `text`. `country` is
74
+ not a column -- it's the top-level folder name, added by `read_articles()`
75
+ along with `year_month`.
76
+
77
+ `read_articles()` does **not** auto-canonicalize country folder names
78
+ (matching `newsindex`'s loader). A folder like `Morocco_new` is kept
79
+ distinct from `Morocco` rather than silently merged -- it may be a
80
+ deliberately separate re-scrape. Apply `aeri.countries.canonicalize()`
81
+ yourself once you've decided whether/how to merge it.
82
+
83
+ ## Scoring pipeline
84
+
85
+ | Function | What it does |
86
+ | --- | --- |
87
+ | `normalize_text()` | Strip HTML; standardise case, whitespace and Unicode |
88
+ | `is_relevant()` | Economic relevance flag `R_i`, per the article's country dictionary |
89
+ | `severity_tier()` | Ordinal severity `S_i` from 0 to 4; the highest tier matched wins |
90
+ | `confidence_weight()` | `W_i = clip(1 + 0.5*TitleHint + 0.2*LongText + 0.3*MultiPillar, 1, 2)` |
91
+ | `assign_pillars()` | Flags for the 8 pillars; one article can activate several |
92
+ | `article_score()` | `A_i = R_i * S_i * W_i` |
93
+ | `compute_aeri()` | `AERI = 100 * sum(A_i) / N_t` for each country and month |
94
+ | `diagnostics()` | Total/relevant articles, relevant share, average confidence, average severity among relevant articles |
95
+ | `coverage_filter()` | Keeps country-months with at least 100 articles |
96
+
97
+ `score_articles()` runs the per-article functions over an articles
98
+ DataFrame in one pass. Relevance, severity and pillar matching run on the
99
+ article's **title and body concatenated** (`x_i = title + " " + text`),
100
+ and matching is **word-boundary aware** (`aeri.match`), not substring
101
+ search -- e.g. the severity term `"war"` does not match inside `"warning"`
102
+ or `"forward"`. `compute_aeri()` / `diagnostics()` output `Country`,
103
+ `Period` (`YYYY-MM`), `AERI`, `TotalArticles`, `RelevantArticles`,
104
+ `RelevantSharePct`, `AvgConfidence`, `AvgSeverityAmongRelevant` -- the
105
+ same column names as the production `aeri_country_period.csv` /
106
+ `aeri_diagnostics.csv` already consumed by the AERI Pillar API.
107
+
108
+ The 8 pillars (`aeri.config.PILLARS`) are named to match the Pillar API's
109
+ catalog: `InflationRisk`, `FXExternalRisk`, `DebtFiscalRisk`,
110
+ `FinancialSectorRisk`, `GrowthRealEconomyRisk`, `CommoditySupplyRisk`,
111
+ `PoliticalEconomyRisk`, `ClimateDisasterRisk`.
112
+
113
+ ## Dictionaries
114
+
115
+ `data/aeri_country_keywords.json` is the real, production EPTU-schema
116
+ keyword dictionary (54 African countries, Economic/Policy/Uncertainty
117
+ term lists) shared with `newsindex`. `is_relevant()` defaults to the
118
+ Economic component alone for `R_i` -- **this default is an assumption**;
119
+ no source document specified whether Policy terms should also count
120
+ toward relevance. Add `"policy"` via `AERIConfig.relevance_components` to
121
+ include them:
122
+
123
+ ```python
124
+ from aeri import AERIConfig
125
+ config = AERIConfig(relevance_components=("economic", "policy"))
126
+ ```
127
+
128
+ The **severity-tier and pillar keyword lists have no equivalent
129
+ production source** -- nothing in the underlying research materials
130
+ defines them as keyword lists, only conceptually ("mild concern" through
131
+ "crisis-level language"). `aeri.config.DEFAULT_SEVERITY_KEYWORDS` and
132
+ `DEFAULT_PILLAR_KEYWORDS` are a first-pass, hand-curated starting point
133
+ that needs calibration before being treated as final. Every scoring
134
+ function takes an `AERIConfig` to override them without forking the
135
+ library.
136
+
137
+ The `TitleHint`/`LongText`/`MultiPillar` thresholds inside
138
+ `confidence_weight()` (title relevance; >=300 words; >=2 active pillars)
139
+ are likewise placeholders -- the exact numeric cutoffs used in the
140
+ original research were not found in any available source and need
141
+ confirmation.
142
+
143
+ ## Roadmap
144
+
145
+ - Pillar composite indices and PCA-combined pillar measure (Library 2)
146
+ - `ars` composite score (Library 3): `ARS = 100 * (0.40*Intensity + 0.25*Breadth + 0.20*Severity + 0.15*Structure)`, risk bands at 20/40/60/80
147
+ - `riskalert` early-warning alerts (Library 4)
148
+
149
+ ## Development
150
+
151
+ ```bash
152
+ pip install -e ".[test]"
153
+ pytest
154
+ ```
@@ -0,0 +1,131 @@
1
+ # aeri
2
+
3
+ AERI article scoring and country-month economic-risk index library, built
4
+ from the AERI/ARS work at CAPE Economic Research and Consulting.
5
+
6
+ This first release covers **Library 1** from the proposal: scoring each
7
+ scraped news article and rolling the scores up into the country-month
8
+ AERI (Africa Economic Risk Index). Pillar composite indices, the ARS
9
+ composite score and early-warning alerts (Libraries 2-4) are planned as
10
+ follow-up PRs on top of this foundation. Matching conventions, data
11
+ layout and output schema are kept consistent with the sibling
12
+ [`newsindex`](https://github.com/CAPE-ERC/newsindex-library) (EPU) library
13
+ and the production AERI Pillar API, so the CAPE libraries interoperate.
14
+
15
+ ## Install
16
+
17
+ ```bash
18
+ pip install cape-aeri # the import name is still `aeri`: import aeri
19
+ ```
20
+
21
+ For development:
22
+
23
+ ```bash
24
+ pip install -e ".[test]"
25
+ ```
26
+
27
+ ## Quickstart
28
+
29
+ ```python
30
+ import aeri
31
+
32
+ dictionary = aeri.dictionary.load("data/aeri_country_keywords.json")
33
+ articles = aeri.read_articles("path/to/Shak_Scrapped_Top_Articles")
34
+
35
+ scored = aeri.score_articles(articles, dictionary)
36
+
37
+ aeri_df = aeri.compute_aeri(scored)
38
+ diag = aeri.diagnostics(scored)
39
+ aeri_df = aeri.coverage_filter(aeri_df, diag) # drop thin country-months
40
+ ```
41
+
42
+ ## Data format
43
+
44
+ `read_articles()` expects the on-disk layout produced by CAPE's scraper:
45
+
46
+ ```
47
+ <root>/<Country>/<Publisher>/<YYYY-MM>/articles_<Publisher>_<YYYY-MM>.csv(.gz)
48
+ ```
49
+
50
+ Each CSV has columns `site`, `date`, `title`, `url`, `text`. `country` is
51
+ not a column -- it's the top-level folder name, added by `read_articles()`
52
+ along with `year_month`.
53
+
54
+ `read_articles()` does **not** auto-canonicalize country folder names
55
+ (matching `newsindex`'s loader). A folder like `Morocco_new` is kept
56
+ distinct from `Morocco` rather than silently merged -- it may be a
57
+ deliberately separate re-scrape. Apply `aeri.countries.canonicalize()`
58
+ yourself once you've decided whether/how to merge it.
59
+
60
+ ## Scoring pipeline
61
+
62
+ | Function | What it does |
63
+ | --- | --- |
64
+ | `normalize_text()` | Strip HTML; standardise case, whitespace and Unicode |
65
+ | `is_relevant()` | Economic relevance flag `R_i`, per the article's country dictionary |
66
+ | `severity_tier()` | Ordinal severity `S_i` from 0 to 4; the highest tier matched wins |
67
+ | `confidence_weight()` | `W_i = clip(1 + 0.5*TitleHint + 0.2*LongText + 0.3*MultiPillar, 1, 2)` |
68
+ | `assign_pillars()` | Flags for the 8 pillars; one article can activate several |
69
+ | `article_score()` | `A_i = R_i * S_i * W_i` |
70
+ | `compute_aeri()` | `AERI = 100 * sum(A_i) / N_t` for each country and month |
71
+ | `diagnostics()` | Total/relevant articles, relevant share, average confidence, average severity among relevant articles |
72
+ | `coverage_filter()` | Keeps country-months with at least 100 articles |
73
+
74
+ `score_articles()` runs the per-article functions over an articles
75
+ DataFrame in one pass. Relevance, severity and pillar matching run on the
76
+ article's **title and body concatenated** (`x_i = title + " " + text`),
77
+ and matching is **word-boundary aware** (`aeri.match`), not substring
78
+ search -- e.g. the severity term `"war"` does not match inside `"warning"`
79
+ or `"forward"`. `compute_aeri()` / `diagnostics()` output `Country`,
80
+ `Period` (`YYYY-MM`), `AERI`, `TotalArticles`, `RelevantArticles`,
81
+ `RelevantSharePct`, `AvgConfidence`, `AvgSeverityAmongRelevant` -- the
82
+ same column names as the production `aeri_country_period.csv` /
83
+ `aeri_diagnostics.csv` already consumed by the AERI Pillar API.
84
+
85
+ The 8 pillars (`aeri.config.PILLARS`) are named to match the Pillar API's
86
+ catalog: `InflationRisk`, `FXExternalRisk`, `DebtFiscalRisk`,
87
+ `FinancialSectorRisk`, `GrowthRealEconomyRisk`, `CommoditySupplyRisk`,
88
+ `PoliticalEconomyRisk`, `ClimateDisasterRisk`.
89
+
90
+ ## Dictionaries
91
+
92
+ `data/aeri_country_keywords.json` is the real, production EPTU-schema
93
+ keyword dictionary (54 African countries, Economic/Policy/Uncertainty
94
+ term lists) shared with `newsindex`. `is_relevant()` defaults to the
95
+ Economic component alone for `R_i` -- **this default is an assumption**;
96
+ no source document specified whether Policy terms should also count
97
+ toward relevance. Add `"policy"` via `AERIConfig.relevance_components` to
98
+ include them:
99
+
100
+ ```python
101
+ from aeri import AERIConfig
102
+ config = AERIConfig(relevance_components=("economic", "policy"))
103
+ ```
104
+
105
+ The **severity-tier and pillar keyword lists have no equivalent
106
+ production source** -- nothing in the underlying research materials
107
+ defines them as keyword lists, only conceptually ("mild concern" through
108
+ "crisis-level language"). `aeri.config.DEFAULT_SEVERITY_KEYWORDS` and
109
+ `DEFAULT_PILLAR_KEYWORDS` are a first-pass, hand-curated starting point
110
+ that needs calibration before being treated as final. Every scoring
111
+ function takes an `AERIConfig` to override them without forking the
112
+ library.
113
+
114
+ The `TitleHint`/`LongText`/`MultiPillar` thresholds inside
115
+ `confidence_weight()` (title relevance; >=300 words; >=2 active pillars)
116
+ are likewise placeholders -- the exact numeric cutoffs used in the
117
+ original research were not found in any available source and need
118
+ confirmation.
119
+
120
+ ## Roadmap
121
+
122
+ - Pillar composite indices and PCA-combined pillar measure (Library 2)
123
+ - `ars` composite score (Library 3): `ARS = 100 * (0.40*Intensity + 0.25*Breadth + 0.20*Severity + 0.15*Structure)`, risk bands at 20/40/60/80
124
+ - `riskalert` early-warning alerts (Library 4)
125
+
126
+ ## Development
127
+
128
+ ```bash
129
+ pip install -e ".[test]"
130
+ pytest
131
+ ```
@@ -0,0 +1,40 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "cape-aeri"
7
+ version = "0.1.0"
8
+ description = "AERI article scoring and country-month economic-risk index library."
9
+ readme = "README.md"
10
+ requires-python = ">=3.10"
11
+ license = { text = "MIT" }
12
+ authors = [
13
+ { name = "Shakir Adeyemi Tewogbade" },
14
+ ]
15
+ keywords = ["economics", "news", "risk-index", "africa", "nlp"]
16
+ classifiers = [
17
+ "Development Status :: 3 - Alpha",
18
+ "Intended Audience :: Science/Research",
19
+ "License :: OSI Approved :: MIT License",
20
+ "Programming Language :: Python :: 3",
21
+ "Programming Language :: Python :: 3.10",
22
+ "Topic :: Scientific/Engineering",
23
+ ]
24
+ dependencies = [
25
+ "pandas>=1.5",
26
+ ]
27
+
28
+ [project.optional-dependencies]
29
+ test = ["pytest>=7"]
30
+
31
+ [project.urls]
32
+ Homepage = "https://github.com/CAPE-ERC/aeri-library"
33
+ Issues = "https://github.com/CAPE-ERC/aeri-library/issues"
34
+
35
+ [tool.setuptools.packages.find]
36
+ where = ["src"]
37
+
38
+ [tool.pytest.ini_options]
39
+ testpaths = ["tests"]
40
+ pythonpath = ["src"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,39 @@
1
+ """aeri: AERI article scoring and country-month economic-risk index library."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from . import config, countries, dictionary, match
6
+ from .articles import read_articles
7
+ from .config import DEFAULT_CONFIG, PILLARS, AERIConfig
8
+ from .confidence import confidence_weight
9
+ from .index import compute_aeri, coverage_filter, diagnostics
10
+ from .pillars import active_pillar_count, assign_pillars
11
+ from .relevance import is_relevant
12
+ from .scoring import article_score, score_articles
13
+ from .severity import severity_tier
14
+ from .text import normalize_text
15
+
16
+ __version__ = "0.1.0"
17
+
18
+ __all__ = [
19
+ "config",
20
+ "countries",
21
+ "dictionary",
22
+ "match",
23
+ "AERIConfig",
24
+ "DEFAULT_CONFIG",
25
+ "PILLARS",
26
+ "normalize_text",
27
+ "is_relevant",
28
+ "severity_tier",
29
+ "confidence_weight",
30
+ "assign_pillars",
31
+ "active_pillar_count",
32
+ "article_score",
33
+ "score_articles",
34
+ "compute_aeri",
35
+ "diagnostics",
36
+ "coverage_filter",
37
+ "read_articles",
38
+ "__version__",
39
+ ]
@@ -0,0 +1,63 @@
1
+ """Read scraped article archives into a single table.
2
+
3
+ Expects the on-disk layout produced by CAPE's scraper: ``<root>/<country>/
4
+ <newspaper>/<year-month>/articles_<newspaper>_<year-month>.csv(.gz)``, each
5
+ file holding the columns ``site``, ``date``, ``title``, ``url``, ``text``.
6
+ Matches the loader in the sibling ``newsindex`` package so both libraries
7
+ read the same archive the same way.
8
+
9
+ The inferred ``country`` is the raw top-level folder name -- it is **not**
10
+ auto-canonicalized against :mod:`aeri.countries`, because a folder like
11
+ ``"Morocco_new"`` may be a deliberately separate corpus (a re-scrape) and
12
+ silently merging it into ``"Morocco"`` could conflate two different data
13
+ sources. Apply :func:`aeri.countries.canonicalize` yourself once you've
14
+ decided that's the right thing to do for your archive.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import html
20
+ from pathlib import Path
21
+
22
+ import pandas as pd
23
+
24
+ _REQUIRED_COLUMNS = ("site", "date", "title", "url", "text")
25
+
26
+
27
+ def _unescape(value: object) -> object:
28
+ if not isinstance(value, str):
29
+ return value
30
+ return html.unescape(value)
31
+
32
+
33
+ def read_articles(root: str | Path, *, encoding: str = "utf-8") -> pd.DataFrame:
34
+ """Read every ``articles_*.csv`` / ``articles_*.csv.gz`` file under
35
+ ``root`` into one table, adding ``country`` (the top-level folder name)
36
+ and ``year_month`` (the first seven characters of ``date``) columns.
37
+
38
+ Article titles and text are HTML-unescaped (the scraper stores raw
39
+ HTML entities such as ``&#8217;``) before they reach `aeri.match`.
40
+ """
41
+ root = Path(root)
42
+ files = sorted({*root.glob("**/*.csv.gz"), *root.glob("**/*.csv")})
43
+ if not files:
44
+ raise FileNotFoundError(f"no article files found under {root}")
45
+
46
+ frames: list[pd.DataFrame] = []
47
+ for file in files:
48
+ country = file.relative_to(root).parts[0]
49
+ frame = pd.read_csv(file, encoding=encoding, compression="infer")
50
+ missing = [col for col in _REQUIRED_COLUMNS if col not in frame.columns]
51
+ if missing:
52
+ raise ValueError(f"{file}: missing column(s) {missing}")
53
+ frame["country"] = country
54
+ frame["source_file"] = str(file)
55
+ frames.append(frame)
56
+
57
+ combined = pd.concat(frames, ignore_index=True)
58
+ combined["title"] = combined["title"].map(_unescape)
59
+ combined["text"] = combined["text"].map(_unescape)
60
+ combined["year_month"] = combined["date"].astype(str).str.slice(0, 7)
61
+ return combined[
62
+ ["country", "site", "year_month", "date", "title", "url", "text", "source_file"]
63
+ ]
@@ -0,0 +1,45 @@
1
+ """Confidence weighting for article scores."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from .config import DEFAULT_CONFIG, AERIConfig
6
+ from .dictionary import CountryDictionary
7
+ from .pillars import active_pillar_count, assign_pillars
8
+ from .relevance import is_relevant
9
+ from .text import normalize_text, word_count
10
+
11
+
12
+ def confidence_weight(
13
+ title: str,
14
+ text: str,
15
+ entry: CountryDictionary,
16
+ pillar_flags: dict[str, bool] | None = None,
17
+ config: AERIConfig = DEFAULT_CONFIG,
18
+ ) -> float:
19
+ """W_i = clip(1 + 0.5*TitleHint + 0.2*LongText + 0.3*MultiPillar, 1, 2).
20
+
21
+ - TitleHint: the title itself carries an economic-relevance signal
22
+ (per the country's dictionary).
23
+ - LongText: the article body is at least `long_text_word_threshold`
24
+ words (normalized word count). This threshold is a placeholder --
25
+ no source document gave the real cutoff; see README.
26
+ - MultiPillar: the article activates `multi_pillar_threshold` or more
27
+ of the 8 pillars.
28
+
29
+ `pillar_flags` can be passed in to reuse an `assign_pillars()` result
30
+ already computed for this article; otherwise it is derived from `text`.
31
+ """
32
+ title_hint = 1.0 if is_relevant(title, entry, config) else 0.0
33
+
34
+ normalized_text = normalize_text(text)
35
+ long_text = 1.0 if word_count(normalized_text) >= config.long_text_word_threshold else 0.0
36
+
37
+ if pillar_flags is None:
38
+ pillar_flags = assign_pillars(text, config)
39
+ multi_pillar = (
40
+ 1.0 if active_pillar_count(pillar_flags) >= config.multi_pillar_threshold else 0.0
41
+ )
42
+
43
+ weight = 1.0 + 0.5 * title_hint + 0.2 * long_text + 0.3 * multi_pillar
44
+ low, high = config.confidence_bounds
45
+ return min(max(weight, low), high)
@@ -0,0 +1,126 @@
1
+ """Configuration and default keyword sets for AERI scoring.
2
+
3
+ The economic-relevance dictionary is real production data loaded
4
+ separately via :mod:`aeri.dictionary` (country-specific, see
5
+ ``data/aeri_country_keywords.json``). The severity-tier and pillar
6
+ keyword sets below have **no equivalent production source** -- no
7
+ dictionary for them was found anywhere in the underlying research
8
+ materials -- so these are a first-pass, hand-curated starting point that
9
+ needs calibration against real coverage before being treated as final.
10
+ Every scoring function accepts an `AERIConfig` so a project can supply
11
+ its own keyword sets without forking the library.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ from dataclasses import dataclass, field
17
+ from typing import Mapping, Sequence
18
+
19
+ #: The 8 AERI risk pillars, named to match the production AERI Pillar API
20
+ #: catalog (`InflationRisk`, `FXExternalRisk`, ...) and the
21
+ #: `aeri_pillars_country_period.csv` column headers it publishes.
22
+ PILLARS: tuple[str, ...] = (
23
+ "InflationRisk",
24
+ "FXExternalRisk",
25
+ "DebtFiscalRisk",
26
+ "FinancialSectorRisk",
27
+ "GrowthRealEconomyRisk",
28
+ "CommoditySupplyRisk",
29
+ "PoliticalEconomyRisk",
30
+ "ClimateDisasterRisk",
31
+ )
32
+
33
+ DEFAULT_SEVERITY_KEYWORDS: dict[int, tuple[str, ...]] = {
34
+ 1: (
35
+ "concern", "warning", "pressure", "risk", "slowdown", "softening",
36
+ "worry", "caution", "uncertainty",
37
+ ),
38
+ 2: (
39
+ "falling", "decline", "drop", "shortage", "strain", "weakening",
40
+ "job losses", "cutbacks", "downgrade", "rising prices", "protest",
41
+ "strike",
42
+ ),
43
+ 3: (
44
+ "crisis", "collapse", "plunge", "default", "devaluation", "emergency",
45
+ "unrest", "shutdown", "severe shortage", "mass layoffs", "coup",
46
+ "credit downgrade",
47
+ ),
48
+ 4: (
49
+ "catastrophe", "famine", "hyperinflation", "economic collapse",
50
+ "state of emergency", "war", "coup d'etat", "humanitarian crisis",
51
+ "bankruptcy", "sovereign default", "civil war", "mass starvation",
52
+ "currency collapse",
53
+ ),
54
+ }
55
+
56
+ DEFAULT_PILLAR_KEYWORDS: dict[str, tuple[str, ...]] = {
57
+ "InflationRisk": (
58
+ "inflation", "deflation", "consumer price", "cost of living",
59
+ "price hike", "price increase", "food price", "fuel price", "cpi",
60
+ "disinflation", "hyperinflation", "rising prices",
61
+ ),
62
+ "FXExternalRisk": (
63
+ "exchange rate", "devaluation", "depreciation", "currency", "forex",
64
+ "foreign reserves", "balance of payments", "current account",
65
+ "trade deficit", "capital flight", "remittance", "export", "import",
66
+ ),
67
+ "DebtFiscalRisk": (
68
+ "public debt", "sovereign debt", "fiscal deficit", "budget deficit",
69
+ "treasury", "government spending", "austerity", "imf", "world bank",
70
+ "bailout", "credit rating", "default", "bond", "eurobond",
71
+ "tax revenue",
72
+ ),
73
+ "FinancialSectorRisk": (
74
+ "bank", "banking sector", "interest rate", "central bank",
75
+ "monetary policy", "non-performing loan", "liquidity",
76
+ "credit crunch", "stock exchange", "bond market", "financial sector",
77
+ "lending rate",
78
+ ),
79
+ "GrowthRealEconomyRisk": (
80
+ "gdp", "economic growth", "recession", "unemployment", "job losses",
81
+ "manufacturing", "industrial output", "retail sales",
82
+ "consumer spending", "business confidence", "economic slowdown",
83
+ ),
84
+ "CommoditySupplyRisk": (
85
+ "commodity price", "oil price", "gas price", "mining", "agriculture",
86
+ "harvest", "food supply", "supply chain", "shortage",
87
+ "import dependency", "wheat", "maize", "fertilizer",
88
+ ),
89
+ "PoliticalEconomyRisk": (
90
+ "election", "coup", "protest", "strike", "sanctions", "corruption",
91
+ "policy reform", "political instability", "government shutdown",
92
+ "regulation", "nationalization", "unrest",
93
+ ),
94
+ "ClimateDisasterRisk": (
95
+ "drought", "flood", "famine", "climate change", "natural disaster",
96
+ "cyclone", "locust", "food insecurity", "humanitarian crisis",
97
+ "displacement", "crop failure",
98
+ ),
99
+ }
100
+
101
+
102
+ @dataclass(frozen=True)
103
+ class AERIConfig:
104
+ """Bundles the thresholds and (non-relevance) keyword sets used across
105
+ the pipeline. Severity and pillar keywords default to the first-pass
106
+ sets above; relevance is intentionally not part of this config -- it
107
+ comes from a country-keyed :class:`aeri.dictionary.Dictionary`.
108
+ """
109
+
110
+ severity_keywords: Mapping[int, Sequence[str]] = field(
111
+ default_factory=lambda: DEFAULT_SEVERITY_KEYWORDS
112
+ )
113
+ pillar_keywords: Mapping[str, Sequence[str]] = field(
114
+ default_factory=lambda: DEFAULT_PILLAR_KEYWORDS
115
+ )
116
+ long_text_word_threshold: int = 300
117
+ multi_pillar_threshold: int = 2
118
+ confidence_bounds: tuple[float, float] = (1.0, 2.0)
119
+ coverage_min_articles: int = 100
120
+ #: Which dictionary component(s) count toward R_i. Defaults to the
121
+ #: Economic component alone; add "policy" to also count policy terms.
122
+ #: Unverified against source -- see README.
123
+ relevance_components: tuple[str, ...] = ("economic",)
124
+
125
+
126
+ DEFAULT_CONFIG = AERIConfig()