histwardb 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- histwardb-0.1.0/LICENSE +21 -0
- histwardb-0.1.0/PKG-INFO +164 -0
- histwardb-0.1.0/README.md +117 -0
- histwardb-0.1.0/pyproject.toml +53 -0
- histwardb-0.1.0/setup.cfg +4 -0
- histwardb-0.1.0/src/histwardb/__init__.py +11 -0
- histwardb-0.1.0/src/histwardb/core.py +372 -0
- histwardb-0.1.0/src/histwardb/query.py +72 -0
- histwardb-0.1.0/src/histwardb.egg-info/PKG-INFO +164 -0
- histwardb-0.1.0/src/histwardb.egg-info/SOURCES.txt +15 -0
- histwardb-0.1.0/src/histwardb.egg-info/dependency_links.txt +1 -0
- histwardb-0.1.0/src/histwardb.egg-info/requires.txt +8 -0
- histwardb-0.1.0/src/histwardb.egg-info/top_level.txt +1 -0
- histwardb-0.1.0/tests/test_dataset.py +71 -0
- histwardb-0.1.0/tests/test_fetch.py +52 -0
- histwardb-0.1.0/tests/test_parsing.py +116 -0
- histwardb-0.1.0/tests/test_query.py +20 -0
histwardb-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Vincenzo Manto
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
histwardb-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: histwardb
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A dynamic, programmatic historical and sovereign war dataset built on Wikidata.
|
|
5
|
+
Author: Vincenzo Manto
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2026 Vincenzo Manto
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
|
|
28
|
+
Project-URL: Homepage, https://github.com/VincenzoManto/histwardb
|
|
29
|
+
Project-URL: Repository, https://github.com/VincenzoManto/histwardb
|
|
30
|
+
Project-URL: Issues, https://github.com/VincenzoManto/histwardb/issues
|
|
31
|
+
Classifier: Development Status :: 3 - Alpha
|
|
32
|
+
Classifier: Intended Audience :: Science/Research
|
|
33
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
34
|
+
Classifier: Programming Language :: Python :: 3
|
|
35
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
36
|
+
Requires-Python: >=3.9
|
|
37
|
+
Description-Content-Type: text/markdown
|
|
38
|
+
License-File: LICENSE
|
|
39
|
+
Requires-Dist: pandas>=1.5
|
|
40
|
+
Requires-Dist: requests>=2.28
|
|
41
|
+
Requires-Dist: plotly>=5.0
|
|
42
|
+
Provides-Extra: dev
|
|
43
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
44
|
+
Requires-Dist: pytest-cov>=4.0; extra == "dev"
|
|
45
|
+
Requires-Dist: ruff>=0.5; extra == "dev"
|
|
46
|
+
Dynamic: license-file
|
|
47
|
+
|
|
48
|
+
# HistWarDB 🌍⚔️
|
|
49
|
+
> *Advanced Historical & Sovereign War Dataset powered by Wikidata*
|
|
50
|
+
|
|
51
|
+
[](https://github.com/VincenzoManto/histwardb/actions/workflows/tests.yml)
|
|
52
|
+
[](https://www.python.org/)
|
|
53
|
+
[](https://opensource.org/licenses/MIT)
|
|
54
|
+
[](https://doi.org/10.5281/zenodo.23243673)
|
|
55
|
+
|
|
56
|
+
**HistoricalWarDB** (HistWarDB) is a dynamic, programmatic Python toolkit designed to extract, clean, and analyze global warfare data from Wikidata spanning from 1700 to the present. Built for researchers, historians, and data scientists, it bridges the gap between crowdsourced knowledge and rigorous historical data pipelines, supporting both modern sovereign states and historical empires.
|
|
57
|
+
|
|
58
|
+
---
|
|
59
|
+
|
|
60
|
+
## Key Features
|
|
61
|
+
|
|
62
|
+
* **Dynamic SPARQL Pipeline:** Fetches conflicts using 25-year temporal chunks to bypass query limits and ensure high data completeness.
|
|
63
|
+
* **Rigorous Anti-Noise Filtering:** Automatically excludes peacekeeping missions (`wd:Q35349`), military exercises (`wd:Q1151430`), and non-state actors.
|
|
64
|
+
* **Historical State Support:** Seamlessly tracks historical entities (e.g., *German Empire*, *Qing dynasty*, *Kingdom of Montenegro*) with built-in lifespan sanity checks.
|
|
65
|
+
* **Flexible Analytical Views:** Instantly transforms data into long formats, binary matrices (`0/1`), conflict name mappings, or custom `LIKE` keyword search results.
|
|
66
|
+
* **Interactive Visualizations:** Out-of-the-box Plotly heatmap generation to visualize global conflict intensity over centuries.
|
|
67
|
+
* **Tested and reproducible:** Network I/O, SPARQL query construction, and data-cleaning logic are separated into independent, unit-tested modules — no live Wikidata access required to run the test suite.
|
|
68
|
+
|
|
69
|
+
---
|
|
70
|
+
|
|
71
|
+
## Project layout
|
|
72
|
+
|
|
73
|
+
```
|
|
74
|
+
src/histwardb/
|
|
75
|
+
query.py # SPARQL query construction (pure, no I/O)
|
|
76
|
+
core.py # HistWarDB class: fetch, parse, clean, analytical views
|
|
77
|
+
tests/ # pytest suite (parsing, filtering, views, mocked network)
|
|
78
|
+
HistWarDB.ipynb # usage demo notebook (Colab-compatible)
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
## Installation
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
pip install git+https://github.com/VincenzoManto/histwardb.git
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
For local development (editable install + test/lint tooling):
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
git clone https://github.com/VincenzoManto/histwardb.git
|
|
91
|
+
cd histwardb
|
|
92
|
+
pip install -e ".[dev]"
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
---
|
|
96
|
+
|
|
97
|
+
## Quick Start
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
from histwardb import HistWarDB
|
|
101
|
+
|
|
102
|
+
# Initialize the dataset from 1700 to 2026
|
|
103
|
+
gwd = HistWarDB(start_year=1700, end_year=2026)
|
|
104
|
+
|
|
105
|
+
# Query specific countries in long format
|
|
106
|
+
france_uk_wars = gwd(countries=["France", "United Kingdom"], view="long", only_in_war=True)
|
|
107
|
+
print(france_uk_wars.head())
|
|
108
|
+
|
|
109
|
+
# Get a binary matrix for specific historical periods
|
|
110
|
+
binary_view = gwd(countries="German Empire", view="binary", from_date="1900-01-01", to_date="1918-12-31")
|
|
111
|
+
|
|
112
|
+
# Plot an interactive heatmap for the top 15 conflict-prone states
|
|
113
|
+
gwd.plot_heatmap(top_n=15, from_year=1800, to_year=2026).show()
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
---
|
|
117
|
+
|
|
118
|
+
## Supported Views (`view` parameter)
|
|
119
|
+
|
|
120
|
+
The dataset interface supports multiple analytical views via the `gwd(...)` call:
|
|
121
|
+
|
|
122
|
+
* **`"long"`**: Standard tidy DataFrame containing columns: `Country`, `Year`, `War_State`, and `Active_Conflicts`.
|
|
123
|
+
* **`"binary"`**: Wide pivot table with countries as rows, years as columns, and binary values (`1` = at war, `0` = peace).
|
|
124
|
+
* **`"names"`**: Wide pivot table containing the literal names of active conflicts per year.
|
|
125
|
+
* **`"list"`**: Alphabetical list of all unique conflict names extracted under current filters.
|
|
126
|
+
* **`"countries"` / `"conflicts"`**: Sorted lists of active entities or unique conflicts matching the query criteria.
|
|
127
|
+
|
|
128
|
+
---
|
|
129
|
+
|
|
130
|
+
## Testing
|
|
131
|
+
|
|
132
|
+
The test suite never queries the live Wikidata endpoint. The `HistWarDB`
|
|
133
|
+
class accepts an injectable `session` object for its HTTP calls, and all
|
|
134
|
+
parsing/filtering logic (`histwardb.core.parse_bindings`,
|
|
135
|
+
`clean_dataframe`, `build_long_grid`) is pure and tested against static
|
|
136
|
+
fixtures:
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
pip install -e ".[dev]"
|
|
140
|
+
pytest --cov=histwardb
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for details.
|
|
144
|
+
|
|
145
|
+
---
|
|
146
|
+
|
|
147
|
+
## Citation
|
|
148
|
+
|
|
149
|
+
If you use **HistWarDB** in your academic research, working paper, or data science project, please cite it using the following format (also available in machine-readable form in [`CITATION.cff`](CITATION.cff)):
|
|
150
|
+
|
|
151
|
+
```bibtex
|
|
152
|
+
@software{histwardb2026,
|
|
153
|
+
author = {Vincenzo Manto},
|
|
154
|
+
title = {HistWarDB: A Dynamic Wikidata-Based Historical War Dataset},
|
|
155
|
+
year = {2026},
|
|
156
|
+
url = {https://github.com/VincenzoManto/histwardb}
|
|
157
|
+
}
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
---
|
|
161
|
+
|
|
162
|
+
## License
|
|
163
|
+
|
|
164
|
+
Distributed under the **MIT License**. See [`LICENSE`](LICENSE) for more information.
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
# HistWarDB 🌍⚔️
|
|
2
|
+
> *Advanced Historical & Sovereign War Dataset powered by Wikidata*
|
|
3
|
+
|
|
4
|
+
[](https://github.com/VincenzoManto/histwardb/actions/workflows/tests.yml)
|
|
5
|
+
[](https://www.python.org/)
|
|
6
|
+
[](https://opensource.org/licenses/MIT)
|
|
7
|
+
[](https://doi.org/10.5281/zenodo.23243673)
|
|
8
|
+
|
|
9
|
+
**HistoricalWarDB** (HistWarDB) is a dynamic, programmatic Python toolkit designed to extract, clean, and analyze global warfare data from Wikidata spanning from 1700 to the present. Built for researchers, historians, and data scientists, it bridges the gap between crowdsourced knowledge and rigorous historical data pipelines, supporting both modern sovereign states and historical empires.
|
|
10
|
+
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
## Key Features
|
|
14
|
+
|
|
15
|
+
* **Dynamic SPARQL Pipeline:** Fetches conflicts using 25-year temporal chunks to bypass query limits and ensure high data completeness.
|
|
16
|
+
* **Rigorous Anti-Noise Filtering:** Automatically excludes peacekeeping missions (`wd:Q35349`), military exercises (`wd:Q1151430`), and non-state actors.
|
|
17
|
+
* **Historical State Support:** Seamlessly tracks historical entities (e.g., *German Empire*, *Qing dynasty*, *Kingdom of Montenegro*) with built-in lifespan sanity checks.
|
|
18
|
+
* **Flexible Analytical Views:** Instantly transforms data into long formats, binary matrices (`0/1`), conflict name mappings, or custom `LIKE` keyword search results.
|
|
19
|
+
* **Interactive Visualizations:** Out-of-the-box Plotly heatmap generation to visualize global conflict intensity over centuries.
|
|
20
|
+
* **Tested and reproducible:** Network I/O, SPARQL query construction, and data-cleaning logic are separated into independent, unit-tested modules — no live Wikidata access required to run the test suite.
|
|
21
|
+
|
|
22
|
+
---
|
|
23
|
+
|
|
24
|
+
## Project layout
|
|
25
|
+
|
|
26
|
+
```
|
|
27
|
+
src/histwardb/
|
|
28
|
+
query.py # SPARQL query construction (pure, no I/O)
|
|
29
|
+
core.py # HistWarDB class: fetch, parse, clean, analytical views
|
|
30
|
+
tests/ # pytest suite (parsing, filtering, views, mocked network)
|
|
31
|
+
HistWarDB.ipynb # usage demo notebook (Colab-compatible)
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Installation
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
pip install git+https://github.com/VincenzoManto/histwardb.git
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
For local development (editable install + test/lint tooling):
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
git clone https://github.com/VincenzoManto/histwardb.git
|
|
44
|
+
cd histwardb
|
|
45
|
+
pip install -e ".[dev]"
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
---
|
|
49
|
+
|
|
50
|
+
## Quick Start
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
from histwardb import HistWarDB
|
|
54
|
+
|
|
55
|
+
# Initialize the dataset from 1700 to 2026
|
|
56
|
+
gwd = HistWarDB(start_year=1700, end_year=2026)
|
|
57
|
+
|
|
58
|
+
# Query specific countries in long format
|
|
59
|
+
france_uk_wars = gwd(countries=["France", "United Kingdom"], view="long", only_in_war=True)
|
|
60
|
+
print(france_uk_wars.head())
|
|
61
|
+
|
|
62
|
+
# Get a binary matrix for specific historical periods
|
|
63
|
+
binary_view = gwd(countries="German Empire", view="binary", from_date="1900-01-01", to_date="1918-12-31")
|
|
64
|
+
|
|
65
|
+
# Plot an interactive heatmap for the top 15 conflict-prone states
|
|
66
|
+
gwd.plot_heatmap(top_n=15, from_year=1800, to_year=2026).show()
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
---
|
|
70
|
+
|
|
71
|
+
## Supported Views (`view` parameter)
|
|
72
|
+
|
|
73
|
+
The dataset interface supports multiple analytical views via the `gwd(...)` call:
|
|
74
|
+
|
|
75
|
+
* **`"long"`**: Standard tidy DataFrame containing columns: `Country`, `Year`, `War_State`, and `Active_Conflicts`.
|
|
76
|
+
* **`"binary"`**: Wide pivot table with countries as rows, years as columns, and binary values (`1` = at war, `0` = peace).
|
|
77
|
+
* **`"names"`**: Wide pivot table containing the literal names of active conflicts per year.
|
|
78
|
+
* **`"list"`**: Alphabetical list of all unique conflict names extracted under current filters.
|
|
79
|
+
* **`"countries"` / `"conflicts"`**: Sorted lists of active entities or unique conflicts matching the query criteria.
|
|
80
|
+
|
|
81
|
+
---
|
|
82
|
+
|
|
83
|
+
## Testing
|
|
84
|
+
|
|
85
|
+
The test suite never queries the live Wikidata endpoint. The `HistWarDB`
|
|
86
|
+
class accepts an injectable `session` object for its HTTP calls, and all
|
|
87
|
+
parsing/filtering logic (`histwardb.core.parse_bindings`,
|
|
88
|
+
`clean_dataframe`, `build_long_grid`) is pure and tested against static
|
|
89
|
+
fixtures:
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
pip install -e ".[dev]"
|
|
93
|
+
pytest --cov=histwardb
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for details.
|
|
97
|
+
|
|
98
|
+
---
|
|
99
|
+
|
|
100
|
+
## Citation
|
|
101
|
+
|
|
102
|
+
If you use **HistWarDB** in your academic research, working paper, or data science project, please cite it using the following format (also available in machine-readable form in [`CITATION.cff`](CITATION.cff)):
|
|
103
|
+
|
|
104
|
+
```bibtex
|
|
105
|
+
@software{histwardb2026,
|
|
106
|
+
author = {Vincenzo Manto},
|
|
107
|
+
title = {HistWarDB: A Dynamic Wikidata-Based Historical War Dataset},
|
|
108
|
+
year = {2026},
|
|
109
|
+
url = {https://github.com/VincenzoManto/histwardb}
|
|
110
|
+
}
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
---
|
|
114
|
+
|
|
115
|
+
## License
|
|
116
|
+
|
|
117
|
+
Distributed under the **MIT License**. See [`LICENSE`](LICENSE) for more information.
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "histwardb"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "A dynamic, programmatic historical and sovereign war dataset built on Wikidata."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { file = "LICENSE" }
|
|
11
|
+
authors = [{ name = "Vincenzo Manto" }]
|
|
12
|
+
requires-python = ">=3.9"
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Development Status :: 3 - Alpha",
|
|
15
|
+
"Intended Audience :: Science/Research",
|
|
16
|
+
"License :: OSI Approved :: MIT License",
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
"Topic :: Scientific/Engineering :: Information Analysis",
|
|
19
|
+
]
|
|
20
|
+
dependencies = [
|
|
21
|
+
"pandas>=1.5",
|
|
22
|
+
"requests>=2.28",
|
|
23
|
+
"plotly>=5.0",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
[project.optional-dependencies]
|
|
27
|
+
dev = [
|
|
28
|
+
"pytest>=7.0",
|
|
29
|
+
"pytest-cov>=4.0",
|
|
30
|
+
"ruff>=0.5",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[project.urls]
|
|
34
|
+
Homepage = "https://github.com/VincenzoManto/histwardb"
|
|
35
|
+
Repository = "https://github.com/VincenzoManto/histwardb"
|
|
36
|
+
Issues = "https://github.com/VincenzoManto/histwardb/issues"
|
|
37
|
+
|
|
38
|
+
[tool.setuptools.packages.find]
|
|
39
|
+
where = ["src"]
|
|
40
|
+
|
|
41
|
+
[tool.pytest.ini_options]
|
|
42
|
+
testpaths = ["tests"]
|
|
43
|
+
addopts = "-ra"
|
|
44
|
+
|
|
45
|
+
[tool.ruff]
|
|
46
|
+
line-length = 110
|
|
47
|
+
target-version = "py39"
|
|
48
|
+
|
|
49
|
+
[tool.ruff.lint]
|
|
50
|
+
# UP006/UP007/UP045: keep typing.Optional/Union/List for readability and
|
|
51
|
+
# consistency with the py3.9 floor, even though `from __future__ import
|
|
52
|
+
# annotations` would allow the newer `X | None` syntax.
|
|
53
|
+
ignore = ["UP006", "UP007", "UP045"]
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""HistWarDB: a dynamic, Wikidata-based historical war dataset toolkit."""
|
|
2
|
+
|
|
3
|
+
from .core import HistWarDB
|
|
4
|
+
from .query import build_sparql_query
|
|
5
|
+
|
|
6
|
+
# Backward-compatible alias used in earlier notebooks/scripts.
|
|
7
|
+
FastWikidataWarDataset = HistWarDB
|
|
8
|
+
|
|
9
|
+
__all__ = ["FastWikidataWarDataset", "HistWarDB", "build_sparql_query"]
|
|
10
|
+
|
|
11
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,372 @@
|
|
|
1
|
+
"""Core dataset class for HistWarDB.
|
|
2
|
+
|
|
3
|
+
The design deliberately separates three concerns so each can be tested in
|
|
4
|
+
isolation:
|
|
5
|
+
|
|
6
|
+
1. Network I/O (:meth:`HistWarDB._fetch_chunk`) -- talks to Wikidata.
|
|
7
|
+
2. Pure parsing/filtering (:func:`parse_bindings`, :func:`clean_dataframe`)
|
|
8
|
+
-- no I/O, fully unit-testable with static fixtures.
|
|
9
|
+
3. Grid construction and the analytical ``__call__`` interface.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import datetime
|
|
15
|
+
import re
|
|
16
|
+
from collections.abc import Iterable, Sequence
|
|
17
|
+
from typing import Optional, Union
|
|
18
|
+
|
|
19
|
+
import pandas as pd
|
|
20
|
+
import requests
|
|
21
|
+
|
|
22
|
+
from .query import DEFAULT_USER_AGENT, WIKIDATA_SPARQL_ENDPOINT, build_sparql_query
|
|
23
|
+
|
|
24
|
+
# Historical states whose real-world dissolution year is not reliably
|
|
25
|
+
# captured by Wikidata's P576/P582 properties. Used to discard anachronistic
|
|
26
|
+
# records (e.g. a "German Empire" war recorded after 1918).
|
|
27
|
+
FORCED_MAX_YEARS = {
|
|
28
|
+
"German Empire": 1918,
|
|
29
|
+
"Qing dynasty": 1912,
|
|
30
|
+
"Kingdom of Montenegro": 1918,
|
|
31
|
+
"Principality of Montenegro": 1910,
|
|
32
|
+
"Russian Soviet Federative Socialist Republic": 1991,
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
VALID_VIEWS = ("long", "binary", "names", "list", "countries", "conflicts")
|
|
36
|
+
|
|
37
|
+
_YEAR_RE = re.compile(r"([+-]?\d+)")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def parse_year(date_str: Optional[str]) -> Optional[int]:
|
|
41
|
+
"""Extract the leading year integer from a Wikidata date literal.
|
|
42
|
+
|
|
43
|
+
Returns ``None`` when the input is falsy/not a string or has no digits.
|
|
44
|
+
"""
|
|
45
|
+
if not date_str or not isinstance(date_str, str):
|
|
46
|
+
return None
|
|
47
|
+
match = _YEAR_RE.search(date_str)
|
|
48
|
+
return int(match.group(1)) if match else None
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def parse_bindings(bindings: Iterable[dict], end_year: int) -> list[dict]:
|
|
52
|
+
"""Turn raw SPARQL JSON ``bindings`` into flat conflict/country records."""
|
|
53
|
+
records: list[dict] = []
|
|
54
|
+
for item in bindings:
|
|
55
|
+
country = item["countryLabel"]["value"]
|
|
56
|
+
conflict = item["conflictLabel"]["value"]
|
|
57
|
+
|
|
58
|
+
start_year = parse_year(item["startDate"]["value"])
|
|
59
|
+
raw_end = item["endDate"]["value"] if "endDate" in item else None
|
|
60
|
+
finish_year = parse_year(raw_end) if raw_end else None
|
|
61
|
+
|
|
62
|
+
if start_year is None:
|
|
63
|
+
continue
|
|
64
|
+
|
|
65
|
+
if finish_year is None:
|
|
66
|
+
finish_year = end_year if start_year >= 1800 else (start_year + 5)
|
|
67
|
+
else:
|
|
68
|
+
finish_year = max(finish_year, start_year)
|
|
69
|
+
|
|
70
|
+
country_start = parse_year(item["countryStart"]["value"]) if "countryStart" in item else None
|
|
71
|
+
if "countryEnd" in item:
|
|
72
|
+
country_end = parse_year(item["countryEnd"]["value"])
|
|
73
|
+
elif "countryEndFallback" in item:
|
|
74
|
+
country_end = parse_year(item["countryEndFallback"]["value"])
|
|
75
|
+
else:
|
|
76
|
+
country_end = None
|
|
77
|
+
|
|
78
|
+
records.append(
|
|
79
|
+
{
|
|
80
|
+
"country": country,
|
|
81
|
+
"conflict_name": conflict,
|
|
82
|
+
"start_year": start_year,
|
|
83
|
+
"end_year": finish_year,
|
|
84
|
+
"country_start": country_start,
|
|
85
|
+
"country_end": country_end,
|
|
86
|
+
}
|
|
87
|
+
)
|
|
88
|
+
return records
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _is_forced_dead(row: pd.Series) -> bool:
|
|
92
|
+
dead_year = FORCED_MAX_YEARS.get(row["country"])
|
|
93
|
+
if dead_year is None:
|
|
94
|
+
return False
|
|
95
|
+
return row["start_year"] > dead_year or row["end_year"] > dead_year
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def clean_dataframe(df_raw: pd.DataFrame, start_year: int, end_year: int) -> pd.DataFrame:
|
|
99
|
+
"""Apply anachronism and chronological sanity filters to raw records.
|
|
100
|
+
|
|
101
|
+
Pure function: no network access, safe to unit test with synthetic
|
|
102
|
+
DataFrames.
|
|
103
|
+
"""
|
|
104
|
+
if df_raw.empty:
|
|
105
|
+
return df_raw
|
|
106
|
+
|
|
107
|
+
df = df_raw.drop_duplicates()
|
|
108
|
+
df = df[~df["country"].str.contains(r"^Q\d+$", na=False)]
|
|
109
|
+
if df.empty:
|
|
110
|
+
return df
|
|
111
|
+
|
|
112
|
+
# Drop records for historical states active after their real dissolution.
|
|
113
|
+
df = df[~df.apply(_is_forced_dead, axis=1)]
|
|
114
|
+
if df.empty:
|
|
115
|
+
return df
|
|
116
|
+
|
|
117
|
+
# Discard wars entirely outside the requested [start_year, end_year] window.
|
|
118
|
+
df = df[(df["end_year"] >= start_year) & (df["start_year"] <= end_year)]
|
|
119
|
+
|
|
120
|
+
# State lifespan consistency: a war can't predate/postdate the state,
|
|
121
|
+
# when Wikidata provides those dates. Compare via nullable Float64 so
|
|
122
|
+
# NaN comparisons resolve to False instead of raising (pandas evaluates
|
|
123
|
+
# both sides of `|` eagerly, so a plain None/int comparison would blow
|
|
124
|
+
# up even behind an `isna()` guard).
|
|
125
|
+
country_start = df["country_start"].astype("Float64")
|
|
126
|
+
country_end = df["country_end"].astype("Float64")
|
|
127
|
+
end_year_f = df["end_year"].astype("Float64")
|
|
128
|
+
start_year_f = df["start_year"].astype("Float64")
|
|
129
|
+
|
|
130
|
+
consistent_start = (country_start.isna() | (end_year_f >= country_start)).fillna(False)
|
|
131
|
+
consistent_end = (country_end.isna() | (start_year_f <= country_end)).fillna(False)
|
|
132
|
+
df = df[consistent_start & consistent_end]
|
|
133
|
+
|
|
134
|
+
return df
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def build_long_grid(df_clean: pd.DataFrame, start_year: int, end_year: int) -> pd.DataFrame:
|
|
138
|
+
"""Build the dense Country x Year long-format grid from cleaned records."""
|
|
139
|
+
countries = sorted(df_clean["country"].unique()) if not df_clean.empty else []
|
|
140
|
+
years = range(start_year, end_year + 1)
|
|
141
|
+
|
|
142
|
+
grid = pd.DataFrame(
|
|
143
|
+
[
|
|
144
|
+
{"Country": c, "Year": y, "War_State": 0, "Active_Conflicts": ""}
|
|
145
|
+
for c in countries
|
|
146
|
+
for y in years
|
|
147
|
+
]
|
|
148
|
+
)
|
|
149
|
+
if grid.empty:
|
|
150
|
+
return grid
|
|
151
|
+
|
|
152
|
+
grid = grid.set_index(["Country", "Year"])
|
|
153
|
+
|
|
154
|
+
for _, row in df_clean.iterrows():
|
|
155
|
+
c = row["country"]
|
|
156
|
+
in_y = max(row["start_year"], start_year)
|
|
157
|
+
fi_y = min(row["end_year"], end_year)
|
|
158
|
+
name = row["conflict_name"]
|
|
159
|
+
|
|
160
|
+
for y in range(in_y, fi_y + 1):
|
|
161
|
+
key = (c, y)
|
|
162
|
+
if key not in grid.index:
|
|
163
|
+
continue
|
|
164
|
+
grid.at[key, "War_State"] = 1
|
|
165
|
+
existing = grid.at[key, "Active_Conflicts"]
|
|
166
|
+
grid.at[key, "Active_Conflicts"] = f"{existing}; {name}".strip("; ") if existing else name
|
|
167
|
+
|
|
168
|
+
return grid.reset_index()
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
class HistWarDB:
|
|
172
|
+
"""Dynamic, programmatic historical war dataset built on top of Wikidata.
|
|
173
|
+
|
|
174
|
+
Supports sovereign and historical states, anti-noise filtering of
|
|
175
|
+
non-conflict events, keyword search, and multiple analytical views.
|
|
176
|
+
|
|
177
|
+
Parameters
|
|
178
|
+
----------
|
|
179
|
+
start_year, end_year:
|
|
180
|
+
Inclusive temporal window of the dataset. ``end_year`` defaults to
|
|
181
|
+
the current year.
|
|
182
|
+
chunk_size:
|
|
183
|
+
Size (in years) of the temporal windows used to page around
|
|
184
|
+
Wikidata's query result limits.
|
|
185
|
+
session:
|
|
186
|
+
Optional :class:`requests.Session`-like object used for HTTP calls;
|
|
187
|
+
override in tests to avoid real network access.
|
|
188
|
+
auto_fetch:
|
|
189
|
+
When ``True`` (default), data is fetched immediately on
|
|
190
|
+
construction. Set to ``False`` to construct an empty instance and
|
|
191
|
+
call :meth:`fetch_data` (or :meth:`load_records`) manually -- handy
|
|
192
|
+
for tests and offline usage.
|
|
193
|
+
"""
|
|
194
|
+
|
|
195
|
+
def __init__(
|
|
196
|
+
self,
|
|
197
|
+
start_year: int = 1700,
|
|
198
|
+
end_year: Optional[int] = None,
|
|
199
|
+
chunk_size: int = 25,
|
|
200
|
+
session: Optional[requests.Session] = None,
|
|
201
|
+
auto_fetch: bool = True,
|
|
202
|
+
) -> None:
|
|
203
|
+
self.start_year = start_year
|
|
204
|
+
self.end_year = end_year if end_year else datetime.datetime.now(tz=datetime.timezone.utc).year
|
|
205
|
+
self.chunk_size = chunk_size
|
|
206
|
+
self.session = session or requests
|
|
207
|
+
self.df_long: Optional[pd.DataFrame] = None
|
|
208
|
+
|
|
209
|
+
if auto_fetch:
|
|
210
|
+
self.fetch_data()
|
|
211
|
+
|
|
212
|
+
# -- Network layer -----------------------------------------------------
|
|
213
|
+
|
|
214
|
+
def _fetch_chunk(self, chunk_start: int, chunk_end: int) -> list[dict]:
|
|
215
|
+
"""Fetch and parse a single temporal chunk. Returns raw bindings."""
|
|
216
|
+
query = build_sparql_query(chunk_start, chunk_end, self.end_year)
|
|
217
|
+
headers = {"User-Agent": DEFAULT_USER_AGENT}
|
|
218
|
+
response = self.session.get(
|
|
219
|
+
WIKIDATA_SPARQL_ENDPOINT,
|
|
220
|
+
params={"query": query, "format": "json"},
|
|
221
|
+
headers=headers,
|
|
222
|
+
timeout=30,
|
|
223
|
+
)
|
|
224
|
+
response.raise_for_status()
|
|
225
|
+
return response.json()["results"]["bindings"]
|
|
226
|
+
|
|
227
|
+
def fetch_data(self) -> None:
|
|
228
|
+
"""Fetch all chunks from Wikidata and build the long-format grid."""
|
|
229
|
+
all_records: list[dict] = []
|
|
230
|
+
for chunk_start in range(self.start_year, self.end_year, self.chunk_size):
|
|
231
|
+
chunk_end = min(chunk_start + self.chunk_size, self.end_year)
|
|
232
|
+
try:
|
|
233
|
+
bindings = self._fetch_chunk(chunk_start, chunk_end)
|
|
234
|
+
all_records.extend(parse_bindings(bindings, self.end_year))
|
|
235
|
+
except requests.RequestException as exc:
|
|
236
|
+
print(f"Block {chunk_start}-{chunk_end} temporarily skipped: {exc}")
|
|
237
|
+
continue
|
|
238
|
+
|
|
239
|
+
self.load_records(all_records)
|
|
240
|
+
|
|
241
|
+
def load_records(self, records: list[dict]) -> None:
|
|
242
|
+
"""Build the dataset from a list of flat conflict/country records.
|
|
243
|
+
|
|
244
|
+
Bypasses the network layer entirely -- used by :meth:`fetch_data`
|
|
245
|
+
and directly in tests/offline pipelines.
|
|
246
|
+
"""
|
|
247
|
+
if not records:
|
|
248
|
+
self.df_long = pd.DataFrame(columns=["Country", "Year", "War_State", "Active_Conflicts"])
|
|
249
|
+
return
|
|
250
|
+
|
|
251
|
+
df_raw = pd.DataFrame(records)
|
|
252
|
+
df_clean = clean_dataframe(df_raw, self.start_year, self.end_year)
|
|
253
|
+
self.df_long = build_long_grid(df_clean, self.start_year, self.end_year)
|
|
254
|
+
|
|
255
|
+
# -- Query interface -----------------------------------------------------
|
|
256
|
+
|
|
257
|
+
def __call__(
|
|
258
|
+
self,
|
|
259
|
+
countries: Optional[Union[str, Sequence[str]]] = None,
|
|
260
|
+
view: str = "long",
|
|
261
|
+
from_date: Optional[Union[str, int]] = None,
|
|
262
|
+
to_date: Optional[Union[str, int]] = None,
|
|
263
|
+
only_in_war: bool = False,
|
|
264
|
+
keyword: Optional[str] = None,
|
|
265
|
+
):
|
|
266
|
+
"""Query the dataset. See module docstring for supported ``view`` values."""
|
|
267
|
+
if self.df_long is None:
|
|
268
|
+
raise ValueError("Dataset not initialized. Call fetch_data() or load_records() first.")
|
|
269
|
+
if view not in VALID_VIEWS:
|
|
270
|
+
raise ValueError(f"Invalid view {view!r}. Choose from: {', '.join(VALID_VIEWS)}")
|
|
271
|
+
|
|
272
|
+
df = self.df_long.copy()
|
|
273
|
+
|
|
274
|
+
if countries:
|
|
275
|
+
c_list = [countries] if isinstance(countries, str) else list(countries)
|
|
276
|
+
regex_pattern = "|".join(re.escape(c) for c in c_list)
|
|
277
|
+
df = df[df["Country"].str.contains(regex_pattern, case=False, na=False)]
|
|
278
|
+
|
|
279
|
+
if from_date is not None:
|
|
280
|
+
df = df[df["Year"] >= int(str(from_date).split("-")[0])]
|
|
281
|
+
if to_date is not None:
|
|
282
|
+
df = df[df["Year"] <= int(str(to_date).split("-")[0])]
|
|
283
|
+
if only_in_war:
|
|
284
|
+
df = df[df["War_State"] == 1]
|
|
285
|
+
if keyword:
|
|
286
|
+
df = df[df["Active_Conflicts"].str.contains(keyword, case=False, na=False)]
|
|
287
|
+
|
|
288
|
+
return self._render_view(df, view)
|
|
289
|
+
|
|
290
|
+
@staticmethod
|
|
291
|
+
def _render_view(df: pd.DataFrame, view: str):
|
|
292
|
+
if view == "long":
|
|
293
|
+
return df
|
|
294
|
+
if view == "binary":
|
|
295
|
+
return df.pivot(index="Country", columns="Year", values="War_State").fillna(0).astype(int)
|
|
296
|
+
if view == "names":
|
|
297
|
+
return df.pivot(index="Country", columns="Year", values="Active_Conflicts").fillna("")
|
|
298
|
+
if view == "list":
|
|
299
|
+
return sorted({item for s in df["Active_Conflicts"].dropna() for item in s.split("; ") if item})
|
|
300
|
+
if view == "countries":
|
|
301
|
+
active_df = df[df["War_State"] == 1]
|
|
302
|
+
return sorted(active_df["Country"].unique())
|
|
303
|
+
if view == "conflicts":
|
|
304
|
+
active_df = df[df["War_State"] == 1]
|
|
305
|
+
unique_conflicts = set()
|
|
306
|
+
for s in active_df["Active_Conflicts"].dropna():
|
|
307
|
+
for item in s.split("; "):
|
|
308
|
+
if item:
|
|
309
|
+
unique_conflicts.add(item)
|
|
310
|
+
return sorted(unique_conflicts)
|
|
311
|
+
raise AssertionError(f"Unhandled view: {view}") # pragma: no cover
|
|
312
|
+
|
|
313
|
+
@property
|
|
314
|
+
def available_countries(self) -> list[str]:
|
|
315
|
+
if self.df_long is None or self.df_long.empty:
|
|
316
|
+
return []
|
|
317
|
+
return sorted(self.df_long["Country"].unique())
|
|
318
|
+
|
|
319
|
+
# -- Visualization / export ---------------------------------------------
|
|
320
|
+
|
|
321
|
+
def plot_heatmap(self, top_n: int = 20, from_year: Optional[int] = None, to_year: Optional[int] = None):
|
|
322
|
+
"""Build (and return) a Plotly heatmap figure of war intensity over time.
|
|
323
|
+
|
|
324
|
+
Does not call ``.show()`` so it can be used headlessly / tested.
|
|
325
|
+
"""
|
|
326
|
+
import plotly.graph_objects as go
|
|
327
|
+
|
|
328
|
+
from_year = self.start_year if from_year is None else from_year
|
|
329
|
+
to_year = self.end_year if to_year is None else to_year
|
|
330
|
+
|
|
331
|
+
df_range = self.df_long[(self.df_long["Year"] >= from_year) & (self.df_long["Year"] <= to_year)]
|
|
332
|
+
top_countries = df_range.groupby("Country")["War_State"].sum().nlargest(top_n).index.tolist()
|
|
333
|
+
df_filtered = df_range[df_range["Country"].isin(top_countries)]
|
|
334
|
+
|
|
335
|
+
matrix_val = df_filtered.pivot(index="Country", columns="Year", values="War_State").fillna(0)
|
|
336
|
+
matrix_txt = df_filtered.pivot(index="Country", columns="Year", values="Active_Conflicts").fillna(
|
|
337
|
+
"No recorded conflicts"
|
|
338
|
+
)
|
|
339
|
+
|
|
340
|
+
fig = go.Figure(
|
|
341
|
+
data=go.Heatmap(
|
|
342
|
+
z=matrix_val.values,
|
|
343
|
+
x=matrix_val.columns,
|
|
344
|
+
y=matrix_val.index,
|
|
345
|
+
text=matrix_txt.values,
|
|
346
|
+
hovertemplate="<b>Country:</b> %{y}<br><b>Year:</b> %{x}<br><b>Conflict(s):</b> %{text}<extra></extra>",
|
|
347
|
+
colorscale=[[0, "#ffffff"], [1, "#d9534f"]],
|
|
348
|
+
showscale=False,
|
|
349
|
+
)
|
|
350
|
+
)
|
|
351
|
+
fig.update_layout(
|
|
352
|
+
title=f"Global War State from Wikidata (Sovereign + Historical States) ({from_year}-{to_year})",
|
|
353
|
+
xaxis_title="Year",
|
|
354
|
+
yaxis_title="Country / Historical State",
|
|
355
|
+
xaxis={'dtick': 10},
|
|
356
|
+
height=700,
|
|
357
|
+
)
|
|
358
|
+
return fig
|
|
359
|
+
|
|
360
|
+
def export_csv(self, query_result: pd.DataFrame, filename: str = "histwardb_export.csv") -> str:
|
|
361
|
+
"""Export a query result to CSV and return the written path.
|
|
362
|
+
|
|
363
|
+
In a Colab environment, also triggers a browser download.
|
|
364
|
+
"""
|
|
365
|
+
query_result.to_csv(filename)
|
|
366
|
+
try:
|
|
367
|
+
from google.colab import files # type: ignore
|
|
368
|
+
|
|
369
|
+
files.download(filename)
|
|
370
|
+
except ImportError:
|
|
371
|
+
pass
|
|
372
|
+
return filename
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""SPARQL query construction for HistWarDB.
|
|
2
|
+
|
|
3
|
+
Kept separate from the network/IO layer so the query text itself can be
|
|
4
|
+
unit-tested without hitting the live Wikidata endpoint.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
WIKIDATA_SPARQL_ENDPOINT = "https://query.wikidata.org/sparql"
|
|
8
|
+
|
|
9
|
+
DEFAULT_USER_AGENT = "HistWarDB/0.1 (academic-research; +https://github.com/VincenzoManto/histwardb)"
|
|
10
|
+
|
|
11
|
+
# Conflict class: military conflict (and subclasses).
|
|
12
|
+
CONFLICT_CLASS = "wd:Q198"
|
|
13
|
+
|
|
14
|
+
# Classes excluded from being treated as a "war": peacekeeping missions and
|
|
15
|
+
# military exercises are noise for a sovereign-war dataset.
|
|
16
|
+
EXCLUDED_CONFLICT_CLASSES = ("wd:Q35349", "wd:Q1151430")
|
|
17
|
+
|
|
18
|
+
# Entity types eligible as belligerents: sovereign state, historical country,
|
|
19
|
+
# country.
|
|
20
|
+
COUNTRY_TYPES = ("wd:Q3624078", "wd:Q3024240", "wd:Q6256")
|
|
21
|
+
|
|
22
|
+
# Non-state entity classes excluded from the belligerent side: human,
|
|
23
|
+
# organization, national sports team.
|
|
24
|
+
EXCLUDED_COUNTRY_CLASSES = ("wd:Q5", "wd:Q83370", "wd:Q7278")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def build_sparql_query(chunk_start: int, chunk_end: int, end_year: int) -> str:
|
|
28
|
+
"""Build the SPARQL query fetching conflicts overlapping [chunk_start, chunk_end).
|
|
29
|
+
|
|
30
|
+
Parameters
|
|
31
|
+
----------
|
|
32
|
+
chunk_start, chunk_end:
|
|
33
|
+
Temporal window (inclusive/exclusive) used to page around Wikidata's
|
|
34
|
+
query result limits.
|
|
35
|
+
end_year:
|
|
36
|
+
Overall dataset end year, used as a fallback for ongoing conflicts
|
|
37
|
+
with no recorded end date.
|
|
38
|
+
"""
|
|
39
|
+
excluded_conflicts = "\n".join(
|
|
40
|
+
f"MINUS {{ ?conflict wdt:P31/wdt:279* {cls} . }}" for cls in EXCLUDED_CONFLICT_CLASSES
|
|
41
|
+
)
|
|
42
|
+
excluded_countries = "\n".join(
|
|
43
|
+
f"MINUS {{ ?country wdt:P31/wdt:279* {cls} . }}" for cls in EXCLUDED_COUNTRY_CLASSES
|
|
44
|
+
)
|
|
45
|
+
country_values = " ".join(COUNTRY_TYPES)
|
|
46
|
+
|
|
47
|
+
return f"""
|
|
48
|
+
SELECT DISTINCT ?conflictLabel ?countryLabel ?startDate ?endDate ?countryStart ?countryEnd ?countryEndFallback WHERE {{
|
|
49
|
+
?conflict wdt:P31/wdt:279* {CONFLICT_CLASS} ;
|
|
50
|
+
wdt:P710 ?country ;
|
|
51
|
+
wdt:P580 ?startDate .
|
|
52
|
+
|
|
53
|
+
OPTIONAL {{ ?conflict wdt:P582 ?endDate . }}
|
|
54
|
+
|
|
55
|
+
{excluded_conflicts}
|
|
56
|
+
|
|
57
|
+
?country wdt:P31/wdt:279* ?countryType .
|
|
58
|
+
VALUES ?countryType {{ {country_values} }}
|
|
59
|
+
|
|
60
|
+
OPTIONAL {{ ?country wdt:P571 ?countryStart . }}
|
|
61
|
+
OPTIONAL {{ ?country wdt:P576 ?countryEnd . }}
|
|
62
|
+
OPTIONAL {{ ?country wdt:P582 ?countryEndFallback . }}
|
|
63
|
+
|
|
64
|
+
{excluded_countries}
|
|
65
|
+
|
|
66
|
+
BIND(COALESCE(?endDate, "{end_year}-12-31T23:59:59Z"^^xsd:dateTime) as ?effEndDate)
|
|
67
|
+
FILTER(?startDate < "{chunk_end}-01-01T00:00:00Z"^^xsd:dateTime && ?effEndDate >= "{chunk_start}-01-01T00:00:00Z"^^xsd:dateTime)
|
|
68
|
+
|
|
69
|
+
SERVICE wikibase:label {{ bd:serviceParam wikibase:language "en". }}
|
|
70
|
+
}}
|
|
71
|
+
LIMIT 5000
|
|
72
|
+
"""
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: histwardb
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A dynamic, programmatic historical and sovereign war dataset built on Wikidata.
|
|
5
|
+
Author: Vincenzo Manto
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2026 Vincenzo Manto
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
|
|
28
|
+
Project-URL: Homepage, https://github.com/VincenzoManto/histwardb
|
|
29
|
+
Project-URL: Repository, https://github.com/VincenzoManto/histwardb
|
|
30
|
+
Project-URL: Issues, https://github.com/VincenzoManto/histwardb/issues
|
|
31
|
+
Classifier: Development Status :: 3 - Alpha
|
|
32
|
+
Classifier: Intended Audience :: Science/Research
|
|
33
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
34
|
+
Classifier: Programming Language :: Python :: 3
|
|
35
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
36
|
+
Requires-Python: >=3.9
|
|
37
|
+
Description-Content-Type: text/markdown
|
|
38
|
+
License-File: LICENSE
|
|
39
|
+
Requires-Dist: pandas>=1.5
|
|
40
|
+
Requires-Dist: requests>=2.28
|
|
41
|
+
Requires-Dist: plotly>=5.0
|
|
42
|
+
Provides-Extra: dev
|
|
43
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
44
|
+
Requires-Dist: pytest-cov>=4.0; extra == "dev"
|
|
45
|
+
Requires-Dist: ruff>=0.5; extra == "dev"
|
|
46
|
+
Dynamic: license-file
|
|
47
|
+
|
|
48
|
+
# HistWarDB 🌍⚔️
|
|
49
|
+
> *Advanced Historical & Sovereign War Dataset powered by Wikidata*
|
|
50
|
+
|
|
51
|
+
[](https://github.com/VincenzoManto/histwardb/actions/workflows/tests.yml)
|
|
52
|
+
[](https://www.python.org/)
|
|
53
|
+
[](https://opensource.org/licenses/MIT)
|
|
54
|
+
[](https://doi.org/10.5281/zenodo.23243673)
|
|
55
|
+
|
|
56
|
+
**HistoricalWarDB** (HistWarDB) is a dynamic, programmatic Python toolkit designed to extract, clean, and analyze global warfare data from Wikidata spanning from 1700 to the present. Built for researchers, historians, and data scientists, it bridges the gap between crowdsourced knowledge and rigorous historical data pipelines, supporting both modern sovereign states and historical empires.
|
|
57
|
+
|
|
58
|
+
---
|
|
59
|
+
|
|
60
|
+
## Key Features
|
|
61
|
+
|
|
62
|
+
* **Dynamic SPARQL Pipeline:** Fetches conflicts using 25-year temporal chunks to bypass query limits and ensure high data completeness.
|
|
63
|
+
* **Rigorous Anti-Noise Filtering:** Automatically excludes peacekeeping missions (`wd:Q35349`), military exercises (`wd:Q1151430`), and non-state actors.
|
|
64
|
+
* **Historical State Support:** Seamlessly tracks historical entities (e.g., *German Empire*, *Qing dynasty*, *Kingdom of Montenegro*) with built-in lifespan sanity checks.
|
|
65
|
+
* **Flexible Analytical Views:** Instantly transforms data into long formats, binary matrices (`0/1`), conflict name mappings, or custom `LIKE` keyword search results.
|
|
66
|
+
* **Interactive Visualizations:** Out-of-the-box Plotly heatmap generation to visualize global conflict intensity over centuries.
|
|
67
|
+
* **Tested and reproducible:** Network I/O, SPARQL query construction, and data-cleaning logic are separated into independent, unit-tested modules — no live Wikidata access required to run the test suite.
|
|
68
|
+
|
|
69
|
+
---
|
|
70
|
+
|
|
71
|
+
## Project layout
|
|
72
|
+
|
|
73
|
+
```
|
|
74
|
+
src/histwardb/
|
|
75
|
+
query.py # SPARQL query construction (pure, no I/O)
|
|
76
|
+
core.py # HistWarDB class: fetch, parse, clean, analytical views
|
|
77
|
+
tests/ # pytest suite (parsing, filtering, views, mocked network)
|
|
78
|
+
HistWarDB.ipynb # usage demo notebook (Colab-compatible)
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
## Installation
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
pip install git+https://github.com/VincenzoManto/histwardb.git
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
For local development (editable install + test/lint tooling):
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
git clone https://github.com/VincenzoManto/histwardb.git
|
|
91
|
+
cd histwardb
|
|
92
|
+
pip install -e ".[dev]"
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
---
|
|
96
|
+
|
|
97
|
+
## Quick Start
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
from histwardb import HistWarDB
|
|
101
|
+
|
|
102
|
+
# Initialize the dataset from 1700 to 2026
|
|
103
|
+
gwd = HistWarDB(start_year=1700, end_year=2026)
|
|
104
|
+
|
|
105
|
+
# Query specific countries in long format
|
|
106
|
+
france_uk_wars = gwd(countries=["France", "United Kingdom"], view="long", only_in_war=True)
|
|
107
|
+
print(france_uk_wars.head())
|
|
108
|
+
|
|
109
|
+
# Get a binary matrix for specific historical periods
|
|
110
|
+
binary_view = gwd(countries="German Empire", view="binary", from_date="1900-01-01", to_date="1918-12-31")
|
|
111
|
+
|
|
112
|
+
# Plot an interactive heatmap for the top 15 conflict-prone states
|
|
113
|
+
gwd.plot_heatmap(top_n=15, from_year=1800, to_year=2026).show()
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
---
|
|
117
|
+
|
|
118
|
+
## Supported Views (`view` parameter)
|
|
119
|
+
|
|
120
|
+
The dataset interface supports multiple analytical views via the `gwd(...)` call:
|
|
121
|
+
|
|
122
|
+
* **`"long"`**: Standard tidy DataFrame containing columns: `Country`, `Year`, `War_State`, and `Active_Conflicts`.
|
|
123
|
+
* **`"binary"`**: Wide pivot table with countries as rows, years as columns, and binary values (`1` = at war, `0` = peace).
|
|
124
|
+
* **`"names"`**: Wide pivot table containing the literal names of active conflicts per year.
|
|
125
|
+
* **`"list"`**: Alphabetical list of all unique conflict names extracted under current filters.
|
|
126
|
+
* **`"countries"` / `"conflicts"`**: Sorted lists of active entities or unique conflicts matching the query criteria.
|
|
127
|
+
|
|
128
|
+
---
|
|
129
|
+
|
|
130
|
+
## Testing
|
|
131
|
+
|
|
132
|
+
The test suite never queries the live Wikidata endpoint. The `HistWarDB`
|
|
133
|
+
class accepts an injectable `session` object for its HTTP calls, and all
|
|
134
|
+
parsing/filtering logic (`histwardb.core.parse_bindings`,
|
|
135
|
+
`clean_dataframe`, `build_long_grid`) is pure and tested against static
|
|
136
|
+
fixtures:
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
pip install -e ".[dev]"
|
|
140
|
+
pytest --cov=histwardb
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for details.
|
|
144
|
+
|
|
145
|
+
---
|
|
146
|
+
|
|
147
|
+
## Citation
|
|
148
|
+
|
|
149
|
+
If you use **HistWarDB** in your academic research, working paper, or data science project, please cite it using the following format (also available in machine-readable form in [`CITATION.cff`](CITATION.cff)):
|
|
150
|
+
|
|
151
|
+
```bibtex
|
|
152
|
+
@software{histwardb2026,
|
|
153
|
+
author = {Vincenzo Manto},
|
|
154
|
+
title = {HistWarDB: A Dynamic Wikidata-Based Historical War Dataset},
|
|
155
|
+
year = {2026},
|
|
156
|
+
url = {https://github.com/VincenzoManto/histwardb}
|
|
157
|
+
}
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
---
|
|
161
|
+
|
|
162
|
+
## License
|
|
163
|
+
|
|
164
|
+
Distributed under the **MIT License**. See [`LICENSE`](LICENSE) for more information.
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/histwardb/__init__.py
|
|
5
|
+
src/histwardb/core.py
|
|
6
|
+
src/histwardb/query.py
|
|
7
|
+
src/histwardb.egg-info/PKG-INFO
|
|
8
|
+
src/histwardb.egg-info/SOURCES.txt
|
|
9
|
+
src/histwardb.egg-info/dependency_links.txt
|
|
10
|
+
src/histwardb.egg-info/requires.txt
|
|
11
|
+
src/histwardb.egg-info/top_level.txt
|
|
12
|
+
tests/test_dataset.py
|
|
13
|
+
tests/test_fetch.py
|
|
14
|
+
tests/test_parsing.py
|
|
15
|
+
tests/test_query.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
histwardb
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
from histwardb import HistWarDB
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def test_long_view_has_full_grid(dataset_from_bindings):
|
|
7
|
+
df = dataset_from_bindings(view="long")
|
|
8
|
+
# 1900-2026 inclusive = 127 years, for each of the 3 surviving countries
|
|
9
|
+
years = dataset_from_bindings.end_year - dataset_from_bindings.start_year + 1
|
|
10
|
+
assert len(df) == years * len(dataset_from_bindings.available_countries)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def test_war_state_set_for_active_years(dataset_from_bindings):
|
|
14
|
+
df = dataset_from_bindings(countries="France", from_date=1939, to_date=1945, view="long")
|
|
15
|
+
assert (df["War_State"] == 1).all()
|
|
16
|
+
before = dataset_from_bindings(countries="France", from_date=1930, to_date=1938, view="long")
|
|
17
|
+
assert (before["War_State"] == 0).all()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def test_german_empire_anachronism_excluded(dataset_from_bindings):
|
|
21
|
+
assert "German Empire" not in dataset_from_bindings.available_countries
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def test_binary_view_shape(dataset_from_bindings):
|
|
25
|
+
binary = dataset_from_bindings(countries=["France", "United Kingdom"], view="binary")
|
|
26
|
+
assert set(binary.index) == {"France", "United Kingdom"}
|
|
27
|
+
assert binary.loc["France", 1940] == 1
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_names_view_contains_conflict_name(dataset_from_bindings):
|
|
31
|
+
names = dataset_from_bindings(countries="France", from_date=1940, to_date=1940, view="names")
|
|
32
|
+
assert "Test War One" in names.loc["France", 1940]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def test_list_view_returns_sorted_unique_names(dataset_from_bindings):
|
|
36
|
+
result = dataset_from_bindings(view="list")
|
|
37
|
+
assert result == sorted(set(result))
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def test_countries_view_only_includes_active(dataset_from_bindings):
|
|
41
|
+
result = dataset_from_bindings(from_date=1939, to_date=1945, view="countries")
|
|
42
|
+
assert "France" in result
|
|
43
|
+
assert "United Kingdom" in result
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_conflicts_view(dataset_from_bindings):
|
|
47
|
+
result = dataset_from_bindings(from_date=1939, to_date=1945, view="conflicts")
|
|
48
|
+
assert "Test War One" in result
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def test_keyword_filter(dataset_from_bindings):
|
|
52
|
+
result = dataset_from_bindings(keyword="Ongoing", view="countries", only_in_war=True)
|
|
53
|
+
assert result == ["Narnia"]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def test_invalid_view_raises(dataset_from_bindings):
|
|
57
|
+
with pytest.raises(ValueError):
|
|
58
|
+
dataset_from_bindings(view="not-a-real-view")
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def test_uninitialized_dataset_raises():
|
|
62
|
+
db = HistWarDB(auto_fetch=False)
|
|
63
|
+
with pytest.raises(ValueError):
|
|
64
|
+
db(view="long")
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def test_load_records_with_empty_list_produces_empty_frame():
|
|
68
|
+
db = HistWarDB(auto_fetch=False)
|
|
69
|
+
db.load_records([])
|
|
70
|
+
assert db.df_long.empty
|
|
71
|
+
assert db.available_countries == []
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Tests for the network layer, using a fake requests session (no live calls)."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
from histwardb import HistWarDB
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class FakeResponse:
|
|
8
|
+
def __init__(self, bindings, status_code=200):
|
|
9
|
+
self._bindings = bindings
|
|
10
|
+
self.status_code = status_code
|
|
11
|
+
|
|
12
|
+
def raise_for_status(self):
|
|
13
|
+
if self.status_code >= 400:
|
|
14
|
+
raise RuntimeError(f"HTTP {self.status_code}")
|
|
15
|
+
|
|
16
|
+
def json(self):
|
|
17
|
+
return {"results": {"bindings": self._bindings}}
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class FakeSession:
|
|
21
|
+
"""Returns the same canned bindings for every chunk request."""
|
|
22
|
+
|
|
23
|
+
def __init__(self, bindings):
|
|
24
|
+
self.bindings = bindings
|
|
25
|
+
self.calls = []
|
|
26
|
+
|
|
27
|
+
def get(self, url, params=None, headers=None, timeout=None):
|
|
28
|
+
self.calls.append((url, params))
|
|
29
|
+
return FakeResponse(self.bindings)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class FailingSession:
|
|
33
|
+
def get(self, url, params=None, headers=None, timeout=None):
|
|
34
|
+
import requests
|
|
35
|
+
|
|
36
|
+
raise requests.ConnectionError("simulated network failure")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def test_fetch_data_uses_injected_session(sample_bindings):
|
|
40
|
+
session = FakeSession(sample_bindings)
|
|
41
|
+
db = HistWarDB(start_year=1925, end_year=1975, chunk_size=25, session=session)
|
|
42
|
+
assert db.df_long is not None
|
|
43
|
+
assert not db.df_long.empty
|
|
44
|
+
# one request per 25-year chunk in [1925, 1975)
|
|
45
|
+
assert len(session.calls) == 2
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def test_fetch_data_survives_network_failures():
|
|
49
|
+
db = HistWarDB(start_year=1900, end_year=1950, session=FailingSession())
|
|
50
|
+
# all chunks fail -> empty but initialized dataframe, no exception raised
|
|
51
|
+
assert db.df_long is not None
|
|
52
|
+
assert db.df_long.empty
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import pandas as pd
|
|
2
|
+
import pytest
|
|
3
|
+
|
|
4
|
+
from histwardb.core import clean_dataframe, parse_bindings, parse_year
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@pytest.mark.parametrize(
|
|
8
|
+
"value,expected",
|
|
9
|
+
[
|
|
10
|
+
(None, None),
|
|
11
|
+
("", None),
|
|
12
|
+
(123, None), # not a string
|
|
13
|
+
("1939-01-01T00:00:00Z", 1939),
|
|
14
|
+
("-0500-01-01T00:00:00Z", -500),
|
|
15
|
+
("no digits here", None),
|
|
16
|
+
],
|
|
17
|
+
)
|
|
18
|
+
def test_parse_year(value, expected):
|
|
19
|
+
assert parse_year(value) == expected
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def test_parse_bindings_fills_missing_end_date_recent(sample_bindings):
|
|
23
|
+
records = parse_bindings(sample_bindings, end_year=2026)
|
|
24
|
+
narnia = next(r for r in records if r["country"] == "Narnia")
|
|
25
|
+
assert narnia["start_year"] == 2020
|
|
26
|
+
assert narnia["end_year"] == 2026 # no end date, recent conflict -> runs to dataset end
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def test_parse_bindings_skips_entries_without_start_date():
|
|
30
|
+
bindings = [
|
|
31
|
+
{
|
|
32
|
+
"countryLabel": {"value": "Atlantis"},
|
|
33
|
+
"conflictLabel": {"value": "Mystery War"},
|
|
34
|
+
"startDate": {"value": "not-a-date"},
|
|
35
|
+
}
|
|
36
|
+
]
|
|
37
|
+
# parse_year will return None for "not-a-date" has no digits -> skipped
|
|
38
|
+
records = parse_bindings(bindings, end_year=2026)
|
|
39
|
+
assert records == []
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def test_clean_dataframe_drops_forced_dead_states():
|
|
43
|
+
df_raw = pd.DataFrame(
|
|
44
|
+
[
|
|
45
|
+
{
|
|
46
|
+
"country": "German Empire",
|
|
47
|
+
"conflict_name": "Anachronistic War",
|
|
48
|
+
"start_year": 1925,
|
|
49
|
+
"end_year": 1925,
|
|
50
|
+
"country_start": None,
|
|
51
|
+
"country_end": None,
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
"country": "France",
|
|
55
|
+
"conflict_name": "Real War",
|
|
56
|
+
"start_year": 1940,
|
|
57
|
+
"end_year": 1944,
|
|
58
|
+
"country_start": None,
|
|
59
|
+
"country_end": None,
|
|
60
|
+
},
|
|
61
|
+
]
|
|
62
|
+
)
|
|
63
|
+
cleaned = clean_dataframe(df_raw, start_year=1900, end_year=2026)
|
|
64
|
+
assert "German Empire" not in cleaned["country"].values
|
|
65
|
+
assert "France" in cleaned["country"].values
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def test_clean_dataframe_enforces_country_lifespan():
|
|
69
|
+
df_raw = pd.DataFrame(
|
|
70
|
+
[
|
|
71
|
+
{
|
|
72
|
+
"country": "Short-Lived Republic",
|
|
73
|
+
"conflict_name": "Post-dissolution War",
|
|
74
|
+
"start_year": 2000,
|
|
75
|
+
"end_year": 2001,
|
|
76
|
+
"country_start": 1990,
|
|
77
|
+
"country_end": 1995, # dissolved before the war started
|
|
78
|
+
}
|
|
79
|
+
]
|
|
80
|
+
)
|
|
81
|
+
cleaned = clean_dataframe(df_raw, start_year=1900, end_year=2026)
|
|
82
|
+
assert cleaned.empty
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def test_clean_dataframe_drops_wikidata_qid_placeholders():
|
|
86
|
+
df_raw = pd.DataFrame(
|
|
87
|
+
[
|
|
88
|
+
{
|
|
89
|
+
"country": "Q12345",
|
|
90
|
+
"conflict_name": "Unlabeled War",
|
|
91
|
+
"start_year": 1950,
|
|
92
|
+
"end_year": 1951,
|
|
93
|
+
"country_start": None,
|
|
94
|
+
"country_end": None,
|
|
95
|
+
}
|
|
96
|
+
]
|
|
97
|
+
)
|
|
98
|
+
cleaned = clean_dataframe(df_raw, start_year=1900, end_year=2026)
|
|
99
|
+
assert cleaned.empty
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def test_clean_dataframe_out_of_range_is_discarded():
|
|
103
|
+
df_raw = pd.DataFrame(
|
|
104
|
+
[
|
|
105
|
+
{
|
|
106
|
+
"country": "Old Kingdom",
|
|
107
|
+
"conflict_name": "Ancient War",
|
|
108
|
+
"start_year": 1600,
|
|
109
|
+
"end_year": 1605,
|
|
110
|
+
"country_start": None,
|
|
111
|
+
"country_end": None,
|
|
112
|
+
}
|
|
113
|
+
]
|
|
114
|
+
)
|
|
115
|
+
cleaned = clean_dataframe(df_raw, start_year=1900, end_year=2026)
|
|
116
|
+
assert cleaned.empty
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from histwardb.query import build_sparql_query
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def test_build_sparql_query_contains_temporal_bounds():
|
|
5
|
+
q = build_sparql_query(chunk_start=1900, chunk_end=1925, end_year=2026)
|
|
6
|
+
assert "1925-01-01T00:00:00Z" in q
|
|
7
|
+
assert "1900-01-01T00:00:00Z" in q
|
|
8
|
+
assert "2026-12-31T23:59:59Z" in q
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def test_build_sparql_query_excludes_noise_classes():
|
|
12
|
+
q = build_sparql_query(1900, 1925, 2026)
|
|
13
|
+
assert "wd:Q35349" in q # peacekeeping missions
|
|
14
|
+
assert "wd:Q1151430" in q # military exercises
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def test_build_sparql_query_is_valid_looking_sparql():
|
|
18
|
+
q = build_sparql_query(1900, 1925, 2026)
|
|
19
|
+
assert q.strip().startswith("SELECT DISTINCT")
|
|
20
|
+
assert q.count("{") == q.count("}")
|