coord-ingest 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- coord_ingest-0.1.0/LICENSE +21 -0
- coord_ingest-0.1.0/PKG-INFO +57 -0
- coord_ingest-0.1.0/README.md +42 -0
- coord_ingest-0.1.0/pyproject.toml +30 -0
- coord_ingest-0.1.0/setup.cfg +4 -0
- coord_ingest-0.1.0/src/coord_ingest/__init__.py +50 -0
- coord_ingest-0.1.0/src/coord_ingest/adapters.py +196 -0
- coord_ingest-0.1.0/src/coord_ingest/data/sample_feed.json +8 -0
- coord_ingest-0.1.0/src/coord_ingest/pipeline.py +70 -0
- coord_ingest-0.1.0/src/coord_ingest.egg-info/PKG-INFO +57 -0
- coord_ingest-0.1.0/src/coord_ingest.egg-info/SOURCES.txt +13 -0
- coord_ingest-0.1.0/src/coord_ingest.egg-info/dependency_links.txt +1 -0
- coord_ingest-0.1.0/src/coord_ingest.egg-info/requires.txt +1 -0
- coord_ingest-0.1.0/src/coord_ingest.egg-info/top_level.txt +1 -0
- coord_ingest-0.1.0/tests/test_ingest.py +60 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Gabriel Mahia / AfriKaziOS
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: coord-ingest
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Turn open world-signal feeds (quakes, disasters, outbreaks) into africa-coord-bus coordination events — East Africa scoped, MIT, no API keys.
|
|
5
|
+
Author: AfriKaziOS maintainers
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/gabrielmahia/coord-ingest
|
|
8
|
+
Project-URL: Repository, https://github.com/gabrielmahia/coord-ingest
|
|
9
|
+
Keywords: osint,east-africa,coordination,mcp,early-warning,feeds
|
|
10
|
+
Requires-Python: >=3.10
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Requires-Dist: africa-coord-bus>=0.2.0
|
|
14
|
+
Dynamic: license-file
|
|
15
|
+
|
|
16
|
+
# coord-ingest
|
|
17
|
+
|
|
18
|
+
Global situational-awareness dashboards ingest hundreds of live feeds and turn them into an actionable picture — but they score the world's Tier-1 countries, and East Africa isn't on that list. The signals that matter here (drought onset, basin flooding, cholera clusters, rift-zone quakes) are in the same public feeds; nothing was turning them into events a regional coordination system could act on.
|
|
19
|
+
|
|
20
|
+
`coord-ingest` is that missing rail: an MIT-licensed adapter that converts open, key-free world-signal feeds into [`africa-coord-bus`](https://github.com/gabrielmahia/africa-coord-bus) coordination events, **filtered to East Africa**, so the bus can cascade them to the right domains (insurance, crop advisory, water testing, cross-border alerts).
|
|
21
|
+
|
|
22
|
+
It ingests and normalizes; the bus routes. Clean separation, composable rails.
|
|
23
|
+
|
|
24
|
+
## What it does
|
|
25
|
+
|
|
26
|
+
- **Adapters** for public sources — USGS earthquakes, ReliefWeb disasters, GDACS alerts — each mapping raw records to typed `CoordinationEvent`s. Add a feed by adding an adapter.
|
|
27
|
+
- **East-Africa filter** — a signal becomes an event only if it lands in the regional bounding box or names a regional country. Global noise is dropped at ingest.
|
|
28
|
+
- **Offline-capable** — ships with a `SampleAdapter` over bundled fixtures, so the full pipeline (and its tests) run with no network and no API keys.
|
|
29
|
+
- **Feeds the bus, doesn't replace it** — routing, cascading, and cross-border logic stay in `africa-coord-bus`.
|
|
30
|
+
|
|
31
|
+
## Use it
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
pip install coord-ingest
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
from coord_ingest import IngestPipeline, SampleAdapter
|
|
39
|
+
|
|
40
|
+
# Offline demo — no keys, no network
|
|
41
|
+
summary = IngestPipeline([SampleAdapter()]).run()
|
|
42
|
+
print(summary["collected"], "East Africa events") # global noise already dropped
|
|
43
|
+
print(summary["by_domain"]) # {'water': 3, 'health': 1, 'transport': 1}
|
|
44
|
+
|
|
45
|
+
# With live feeds + the bus:
|
|
46
|
+
# from coord_ingest import USGSQuakeAdapter, GDACSAdapter
|
|
47
|
+
# from africa_coord_bus import EventBus
|
|
48
|
+
# IngestPipeline([USGSQuakeAdapter(), GDACSAdapter()]).run(bus=EventBus(...))
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
## Why not just fork a dashboard?
|
|
52
|
+
|
|
53
|
+
The global open-source dashboards that inspired this are AGPL applications. This is deliberately the opposite: a small **MIT library** that produces standard bus events any tool can consume — including a dashboard, if someone wants to build one. Rails, not another train.
|
|
54
|
+
|
|
55
|
+
## IP & Collaboration
|
|
56
|
+
|
|
57
|
+
MIT. New feed adapters welcome via GitHub Issues (public, key-free sources, East-Africa relevant). Full policy: [docs/architecture/IP_POLICY.md](docs/architecture/IP_POLICY.md). Security: [SECURITY.md](SECURITY.md).
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# coord-ingest
|
|
2
|
+
|
|
3
|
+
Global situational-awareness dashboards ingest hundreds of live feeds and turn them into an actionable picture — but they score the world's Tier-1 countries, and East Africa isn't on that list. The signals that matter here (drought onset, basin flooding, cholera clusters, rift-zone quakes) are in the same public feeds; nothing was turning them into events a regional coordination system could act on.
|
|
4
|
+
|
|
5
|
+
`coord-ingest` is that missing rail: an MIT-licensed adapter that converts open, key-free world-signal feeds into [`africa-coord-bus`](https://github.com/gabrielmahia/africa-coord-bus) coordination events, **filtered to East Africa**, so the bus can cascade them to the right domains (insurance, crop advisory, water testing, cross-border alerts).
|
|
6
|
+
|
|
7
|
+
It ingests and normalizes; the bus routes. Clean separation, composable rails.
|
|
8
|
+
|
|
9
|
+
## What it does
|
|
10
|
+
|
|
11
|
+
- **Adapters** for public sources — USGS earthquakes, ReliefWeb disasters, GDACS alerts — each mapping raw records to typed `CoordinationEvent`s. Add a feed by adding an adapter.
|
|
12
|
+
- **East-Africa filter** — a signal becomes an event only if it lands in the regional bounding box or names a regional country. Global noise is dropped at ingest.
|
|
13
|
+
- **Offline-capable** — ships with a `SampleAdapter` over bundled fixtures, so the full pipeline (and its tests) run with no network and no API keys.
|
|
14
|
+
- **Feeds the bus, doesn't replace it** — routing, cascading, and cross-border logic stay in `africa-coord-bus`.
|
|
15
|
+
|
|
16
|
+
## Use it
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
pip install coord-ingest
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
```python
|
|
23
|
+
from coord_ingest import IngestPipeline, SampleAdapter
|
|
24
|
+
|
|
25
|
+
# Offline demo — no keys, no network
|
|
26
|
+
summary = IngestPipeline([SampleAdapter()]).run()
|
|
27
|
+
print(summary["collected"], "East Africa events") # global noise already dropped
|
|
28
|
+
print(summary["by_domain"]) # {'water': 3, 'health': 1, 'transport': 1}
|
|
29
|
+
|
|
30
|
+
# With live feeds + the bus:
|
|
31
|
+
# from coord_ingest import USGSQuakeAdapter, GDACSAdapter
|
|
32
|
+
# from africa_coord_bus import EventBus
|
|
33
|
+
# IngestPipeline([USGSQuakeAdapter(), GDACSAdapter()]).run(bus=EventBus(...))
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Why not just fork a dashboard?
|
|
37
|
+
|
|
38
|
+
The global open-source dashboards that inspired this are AGPL applications. This is deliberately the opposite: a small **MIT library** that produces standard bus events any tool can consume — including a dashboard, if someone wants to build one. Rails, not another train.
|
|
39
|
+
|
|
40
|
+
## IP & Collaboration
|
|
41
|
+
|
|
42
|
+
MIT. New feed adapters welcome via GitHub Issues (public, key-free sources, East-Africa relevant). Full policy: [docs/architecture/IP_POLICY.md](docs/architecture/IP_POLICY.md). Security: [SECURITY.md](SECURITY.md).
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "coord-ingest"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Turn open world-signal feeds (quakes, disasters, outbreaks) into africa-coord-bus coordination events — East Africa scoped, MIT, no API keys."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{name = "AfriKaziOS maintainers"}]
|
|
14
|
+
keywords = ["osint","east-africa","coordination","mcp","early-warning","feeds"]
|
|
15
|
+
dependencies = ["africa-coord-bus>=0.2.0"]
|
|
16
|
+
|
|
17
|
+
[project.urls]
|
|
18
|
+
Homepage = "https://github.com/gabrielmahia/coord-ingest"
|
|
19
|
+
Repository = "https://github.com/gabrielmahia/coord-ingest"
|
|
20
|
+
|
|
21
|
+
[tool.setuptools]
|
|
22
|
+
package-dir = {"" = "src"}
|
|
23
|
+
packages = ["coord_ingest"]
|
|
24
|
+
|
|
25
|
+
[tool.setuptools.package-data]
|
|
26
|
+
coord_ingest = ["data/*.json"]
|
|
27
|
+
|
|
28
|
+
[tool.ruff]
|
|
29
|
+
line-length = 100
|
|
30
|
+
target-version = "py310"
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""coord-ingest — turn open world-signal feeds into coordination events.
|
|
2
|
+
|
|
3
|
+
WHY THIS EXISTS
|
|
4
|
+
---------------
|
|
5
|
+
The coordination stack can *route* events (africa-coord-bus) and *serve* domain
|
|
6
|
+
data (35 MCP servers), but nothing turned raw, real-world signals INTO
|
|
7
|
+
coordination events. Dashboards like worldmonitor prove the ingest+correlate
|
|
8
|
+
pattern at global scale — but they are AGPL applications scored on Tier-1
|
|
9
|
+
countries, and East Africa is not Tier-1. This is the missing *rail*: an
|
|
10
|
+
MIT-licensed adapter that maps public feeds (USGS quakes, ReliefWeb disasters,
|
|
11
|
+
GDACS alerts, WHO outbreaks) into typed CoordinationEvents the bus already knows
|
|
12
|
+
how to cascade — scoped to the region the global dashboards skip.
|
|
13
|
+
|
|
14
|
+
DESIGN
|
|
15
|
+
------
|
|
16
|
+
- **Source-agnostic.** Each adapter implements `FeedAdapter.fetch() -> list[dict]`
|
|
17
|
+
and `.to_events() -> list[CoordinationEvent]`. Add a feed by adding an adapter.
|
|
18
|
+
- **East-Africa filtered.** A signal only becomes an event if it lands in the
|
|
19
|
+
region bounding box or names a regional country. Global noise is dropped at
|
|
20
|
+
ingest, not routing.
|
|
21
|
+
- **No keys, offline-capable.** Ships with a `SampleAdapter` (bundled fixtures)
|
|
22
|
+
so the whole pipeline runs and tests without network — the same local-first
|
|
23
|
+
ethos, applied to our region.
|
|
24
|
+
- **Feeds the bus, doesn't replace it.** Output is CoordinationEvents; routing,
|
|
25
|
+
cascading, and cross-border logic stay in africa-coord-bus where they belong.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
from .adapters import (
|
|
31
|
+
FeedAdapter,
|
|
32
|
+
SampleAdapter,
|
|
33
|
+
USGSQuakeAdapter,
|
|
34
|
+
ReliefWebAdapter,
|
|
35
|
+
GDACSAdapter,
|
|
36
|
+
)
|
|
37
|
+
from .pipeline import IngestPipeline, EAST_AFRICA_BBOX, in_east_africa
|
|
38
|
+
|
|
39
|
+
__version__ = "0.1.0"
|
|
40
|
+
|
|
41
|
+
__all__ = [
|
|
42
|
+
"FeedAdapter",
|
|
43
|
+
"SampleAdapter",
|
|
44
|
+
"USGSQuakeAdapter",
|
|
45
|
+
"ReliefWebAdapter",
|
|
46
|
+
"GDACSAdapter",
|
|
47
|
+
"IngestPipeline",
|
|
48
|
+
"EAST_AFRICA_BBOX",
|
|
49
|
+
"in_east_africa",
|
|
50
|
+
]
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
"""Feed adapters: public world-signal source -> CoordinationEvent.
|
|
2
|
+
|
|
3
|
+
Each adapter is small and single-purpose. Real network adapters fetch public,
|
|
4
|
+
key-free feeds; SampleAdapter uses bundled fixtures so the pipeline runs offline.
|
|
5
|
+
All adapters emit africa-coord-bus CoordinationEvents — the bus owns routing.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
from abc import ABC, abstractmethod
|
|
12
|
+
from importlib import resources
|
|
13
|
+
|
|
14
|
+
from africa_coord_bus.event import CoordinationEvent, EventDomain, EventSeverity
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _severity_from_magnitude(mag: float) -> EventSeverity:
|
|
18
|
+
if mag >= 7.0:
|
|
19
|
+
return EventSeverity.CRITICAL
|
|
20
|
+
if mag >= 6.0:
|
|
21
|
+
return EventSeverity.ALERT
|
|
22
|
+
if mag >= 5.0:
|
|
23
|
+
return EventSeverity.WARNING
|
|
24
|
+
return EventSeverity.INFO
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class FeedAdapter(ABC):
|
|
28
|
+
"""Base contract: fetch raw records, convert to CoordinationEvents."""
|
|
29
|
+
|
|
30
|
+
#: bus domain this feed primarily maps to
|
|
31
|
+
domain: EventDomain = EventDomain.CIVIC
|
|
32
|
+
#: event_type emitted (must match routing-table triggers to cascade)
|
|
33
|
+
event_type: str = "external_signal"
|
|
34
|
+
#: stable source label
|
|
35
|
+
source: str = "coord-ingest"
|
|
36
|
+
|
|
37
|
+
@abstractmethod
|
|
38
|
+
def fetch(self) -> list[dict]:
|
|
39
|
+
"""Return raw feed records as dicts. Network or fixtures."""
|
|
40
|
+
|
|
41
|
+
def to_events(self) -> list[CoordinationEvent]:
|
|
42
|
+
events = []
|
|
43
|
+
for rec in self.fetch():
|
|
44
|
+
ev = self._record_to_event(rec)
|
|
45
|
+
if ev is not None:
|
|
46
|
+
events.append(ev)
|
|
47
|
+
return events
|
|
48
|
+
|
|
49
|
+
@abstractmethod
|
|
50
|
+
def _record_to_event(self, rec: dict) -> CoordinationEvent | None:
|
|
51
|
+
"""Map one raw record to a CoordinationEvent (or None to drop)."""
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class SampleAdapter(FeedAdapter):
|
|
55
|
+
"""Offline adapter over bundled fixtures — runs the whole pipeline with no
|
|
56
|
+
network and no keys. Fixtures cover quake, flood, and outbreak signals in
|
|
57
|
+
East Africa so cascades can be demonstrated end to end."""
|
|
58
|
+
|
|
59
|
+
domain = EventDomain.WATER
|
|
60
|
+
event_type = "external_signal"
|
|
61
|
+
source = "coord-ingest.sample"
|
|
62
|
+
|
|
63
|
+
def fetch(self) -> list[dict]:
|
|
64
|
+
text = resources.files("coord_ingest").joinpath("data/sample_feed.json").read_text("utf-8")
|
|
65
|
+
return json.loads(text)
|
|
66
|
+
|
|
67
|
+
def _record_to_event(self, rec: dict) -> CoordinationEvent | None:
|
|
68
|
+
domain = EventDomain(rec["domain"])
|
|
69
|
+
return CoordinationEvent(
|
|
70
|
+
domain=domain,
|
|
71
|
+
event_type=rec["event_type"],
|
|
72
|
+
source=self.source,
|
|
73
|
+
severity=EventSeverity(rec.get("severity", "warning")),
|
|
74
|
+
data={
|
|
75
|
+
"title": rec.get("title", ""),
|
|
76
|
+
"lat": rec.get("lat"),
|
|
77
|
+
"lon": rec.get("lon"),
|
|
78
|
+
"country": rec.get("country"),
|
|
79
|
+
"border_adjacent": rec.get("border_adjacent", False),
|
|
80
|
+
"origin_feed": "sample",
|
|
81
|
+
},
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class USGSQuakeAdapter(FeedAdapter):
|
|
86
|
+
"""USGS earthquakes (public GeoJSON, no key). Quakes near dams, lakes, or
|
|
87
|
+
settlements are routed as water/transport signals for downstream assessment."""
|
|
88
|
+
|
|
89
|
+
domain = EventDomain.TRANSPORT
|
|
90
|
+
event_type = "infrastructure_shock"
|
|
91
|
+
source = "coord-ingest.usgs"
|
|
92
|
+
FEED_URL = "https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/4.5_day.geojson"
|
|
93
|
+
|
|
94
|
+
def __init__(self, records: list[dict] | None = None):
|
|
95
|
+
# allow injected records for testing without network
|
|
96
|
+
self._injected = records
|
|
97
|
+
|
|
98
|
+
def fetch(self) -> list[dict]:
|
|
99
|
+
if self._injected is not None:
|
|
100
|
+
return self._injected
|
|
101
|
+
import urllib.request
|
|
102
|
+
|
|
103
|
+
with urllib.request.urlopen(self.FEED_URL, timeout=20) as r:
|
|
104
|
+
payload = json.load(r)
|
|
105
|
+
return payload.get("features", [])
|
|
106
|
+
|
|
107
|
+
def _record_to_event(self, rec: dict) -> CoordinationEvent | None:
|
|
108
|
+
props = rec.get("properties", {})
|
|
109
|
+
geom = rec.get("geometry", {})
|
|
110
|
+
coords = geom.get("coordinates", [None, None])
|
|
111
|
+
lon, lat = coords[0], coords[1]
|
|
112
|
+
mag = props.get("mag") or 0.0
|
|
113
|
+
return CoordinationEvent(
|
|
114
|
+
domain=self.domain,
|
|
115
|
+
event_type=self.event_type,
|
|
116
|
+
source=self.source,
|
|
117
|
+
severity=_severity_from_magnitude(mag),
|
|
118
|
+
data={
|
|
119
|
+
"title": props.get("place", ""),
|
|
120
|
+
"magnitude": mag,
|
|
121
|
+
"lat": lat,
|
|
122
|
+
"lon": lon,
|
|
123
|
+
"origin_feed": "usgs",
|
|
124
|
+
},
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
class ReliefWebAdapter(FeedAdapter):
|
|
129
|
+
"""ReliefWeb disaster reports (public API). Maps humanitarian disaster
|
|
130
|
+
signals to the water/health domains for coordination follow-up."""
|
|
131
|
+
|
|
132
|
+
domain = EventDomain.HEALTH
|
|
133
|
+
event_type = "disaster_report"
|
|
134
|
+
source = "coord-ingest.reliefweb"
|
|
135
|
+
|
|
136
|
+
def __init__(self, records: list[dict] | None = None):
|
|
137
|
+
self._injected = records
|
|
138
|
+
|
|
139
|
+
def fetch(self) -> list[dict]:
|
|
140
|
+
if self._injected is not None:
|
|
141
|
+
return self._injected
|
|
142
|
+
import urllib.request
|
|
143
|
+
|
|
144
|
+
url = ("https://api.reliefweb.int/v1/disasters?appname=coord-ingest"
|
|
145
|
+
"&profile=list&preset=latest&limit=20")
|
|
146
|
+
with urllib.request.urlopen(url, timeout=20) as r:
|
|
147
|
+
return json.load(r).get("data", [])
|
|
148
|
+
|
|
149
|
+
def _record_to_event(self, rec: dict) -> CoordinationEvent | None:
|
|
150
|
+
fields = rec.get("fields", {})
|
|
151
|
+
name = fields.get("name", "")
|
|
152
|
+
return CoordinationEvent(
|
|
153
|
+
domain=self.domain,
|
|
154
|
+
event_type=self.event_type,
|
|
155
|
+
source=self.source,
|
|
156
|
+
severity=EventSeverity.WARNING,
|
|
157
|
+
data={"title": name, "origin_feed": "reliefweb"},
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class GDACSAdapter(FeedAdapter):
|
|
162
|
+
"""GDACS global disaster alerts. Alert colour maps to severity; the pipeline
|
|
163
|
+
filters to East Africa downstream."""
|
|
164
|
+
|
|
165
|
+
domain = EventDomain.WATER
|
|
166
|
+
event_type = "disaster_alert"
|
|
167
|
+
source = "coord-ingest.gdacs"
|
|
168
|
+
_COLOUR = {"Green": EventSeverity.INFO, "Orange": EventSeverity.ALERT, "Red": EventSeverity.CRITICAL}
|
|
169
|
+
|
|
170
|
+
def __init__(self, records: list[dict] | None = None):
|
|
171
|
+
self._injected = records
|
|
172
|
+
|
|
173
|
+
def fetch(self) -> list[dict]:
|
|
174
|
+
if self._injected is not None:
|
|
175
|
+
return self._injected
|
|
176
|
+
import urllib.request
|
|
177
|
+
|
|
178
|
+
with urllib.request.urlopen("https://www.gdacs.org/gdacsapi/api/events/geteventlist/SEARCH", timeout=20) as r:
|
|
179
|
+
return json.load(r).get("features", [])
|
|
180
|
+
|
|
181
|
+
def _record_to_event(self, rec: dict) -> CoordinationEvent | None:
|
|
182
|
+
props = rec.get("properties", rec)
|
|
183
|
+
colour = props.get("alertlevel", "Green")
|
|
184
|
+
return CoordinationEvent(
|
|
185
|
+
domain=self.domain,
|
|
186
|
+
event_type=self.event_type,
|
|
187
|
+
source=self.source,
|
|
188
|
+
severity=self._COLOUR.get(colour, EventSeverity.INFO),
|
|
189
|
+
data={
|
|
190
|
+
"title": props.get("eventname") or props.get("description", ""),
|
|
191
|
+
"lat": props.get("latitude"),
|
|
192
|
+
"lon": props.get("longitude"),
|
|
193
|
+
"country": props.get("country"),
|
|
194
|
+
"origin_feed": "gdacs",
|
|
195
|
+
},
|
|
196
|
+
)
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
[
|
|
2
|
+
{"domain":"water","event_type":"drought_alert","severity":"alert","title":"Prolonged dry spell, northern rangeland","lat":2.3,"lon":37.9,"country":"Kenya","border_adjacent":true},
|
|
3
|
+
{"domain":"water","event_type":"flood_alert","severity":"warning","title":"River basin rising after heavy rains","lat":-7.8,"lon":38.3,"country":"Tanzania","border_adjacent":false},
|
|
4
|
+
{"domain":"health","event_type":"disease_outbreak","severity":"alert","title":"Cholera cases reported near lake shore","lat":0.1,"lon":34.7,"country":"Kenya","border_adjacent":false},
|
|
5
|
+
{"domain":"transport","event_type":"infrastructure_shock","severity":"warning","title":"Magnitude 5.2 quake near rift settlements","lat":-1.3,"lon":36.1,"country":"Kenya","border_adjacent":false},
|
|
6
|
+
{"domain":"water","event_type":"drought_alert","severity":"info","title":"Seasonal forecast: below-average rainfall","lat":9.0,"lon":40.5,"country":"Ethiopia","border_adjacent":false},
|
|
7
|
+
{"domain":"water","event_type":"flood_alert","severity":"critical","title":"Flash flooding — GLOBAL NOISE (should be dropped)","lat":51.5,"lon":-0.1,"country":"United Kingdom","border_adjacent":false}
|
|
8
|
+
]
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""IngestPipeline — run adapters, filter to East Africa, hand events to the bus.
|
|
2
|
+
|
|
3
|
+
The pipeline is deliberately thin: adapters produce events, the pipeline drops
|
|
4
|
+
anything outside the region, and routing/cascading stays in africa-coord-bus.
|
|
5
|
+
This keeps the ingest rail composable and the bus authoritative.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from africa_coord_bus.event import CoordinationEvent
|
|
11
|
+
|
|
12
|
+
# Rough East Africa bounding box (lat_min, lat_max, lon_min, lon_max).
|
|
13
|
+
# Covers Kenya, Tanzania, Uganda, Rwanda, Burundi, Ethiopia, Somalia, S. Sudan.
|
|
14
|
+
EAST_AFRICA_BBOX = (-12.0, 15.0, 28.0, 52.0)
|
|
15
|
+
|
|
16
|
+
EAST_AFRICA_COUNTRIES = {
|
|
17
|
+
"kenya", "tanzania", "uganda", "rwanda", "burundi", "ethiopia",
|
|
18
|
+
"somalia", "south sudan", "sudan", "djibouti", "eritrea",
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def in_east_africa(event: CoordinationEvent) -> bool:
|
|
23
|
+
"""True if the event's location falls in the region (by bbox or country)."""
|
|
24
|
+
data = event.data or {}
|
|
25
|
+
country = str(data.get("country") or "").strip().lower()
|
|
26
|
+
if country and country in EAST_AFRICA_COUNTRIES:
|
|
27
|
+
return True
|
|
28
|
+
lat, lon = data.get("lat"), data.get("lon")
|
|
29
|
+
if lat is None or lon is None:
|
|
30
|
+
# No geo and no known country -> cannot confirm region; drop conservatively.
|
|
31
|
+
return False
|
|
32
|
+
lat_min, lat_max, lon_min, lon_max = EAST_AFRICA_BBOX
|
|
33
|
+
return lat_min <= lat <= lat_max and lon_min <= lon <= lon_max
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class IngestPipeline:
|
|
37
|
+
"""Collects events from adapters, filters to East Africa, optionally routes."""
|
|
38
|
+
|
|
39
|
+
def __init__(self, adapters, region_filter=in_east_africa):
|
|
40
|
+
self.adapters = list(adapters)
|
|
41
|
+
self.region_filter = region_filter
|
|
42
|
+
|
|
43
|
+
def collect(self) -> list[CoordinationEvent]:
|
|
44
|
+
"""All region-relevant events across every adapter."""
|
|
45
|
+
out: list[CoordinationEvent] = []
|
|
46
|
+
for adapter in self.adapters:
|
|
47
|
+
try:
|
|
48
|
+
events = adapter.to_events()
|
|
49
|
+
except Exception: # noqa: BLE001 — one bad feed must not sink the run
|
|
50
|
+
continue
|
|
51
|
+
out.extend(e for e in events if self.region_filter(e))
|
|
52
|
+
return out
|
|
53
|
+
|
|
54
|
+
def run(self, bus=None) -> dict:
|
|
55
|
+
"""Collect region events; if a bus is given, publish each and return a
|
|
56
|
+
summary. Without a bus, returns the events for inspection."""
|
|
57
|
+
events = self.collect()
|
|
58
|
+
summary = {"collected": len(events), "by_domain": {}, "published": 0}
|
|
59
|
+
for e in events:
|
|
60
|
+
d = e.domain.value
|
|
61
|
+
summary["by_domain"][d] = summary["by_domain"].get(d, 0) + 1
|
|
62
|
+
if bus is not None:
|
|
63
|
+
for e in events:
|
|
64
|
+
try:
|
|
65
|
+
bus.publish(e)
|
|
66
|
+
summary["published"] += 1
|
|
67
|
+
except Exception: # noqa: BLE001
|
|
68
|
+
pass
|
|
69
|
+
summary["events"] = events
|
|
70
|
+
return summary
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: coord-ingest
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Turn open world-signal feeds (quakes, disasters, outbreaks) into africa-coord-bus coordination events — East Africa scoped, MIT, no API keys.
|
|
5
|
+
Author: AfriKaziOS maintainers
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/gabrielmahia/coord-ingest
|
|
8
|
+
Project-URL: Repository, https://github.com/gabrielmahia/coord-ingest
|
|
9
|
+
Keywords: osint,east-africa,coordination,mcp,early-warning,feeds
|
|
10
|
+
Requires-Python: >=3.10
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Requires-Dist: africa-coord-bus>=0.2.0
|
|
14
|
+
Dynamic: license-file
|
|
15
|
+
|
|
16
|
+
# coord-ingest
|
|
17
|
+
|
|
18
|
+
Global situational-awareness dashboards ingest hundreds of live feeds and turn them into an actionable picture — but they score the world's Tier-1 countries, and East Africa isn't on that list. The signals that matter here (drought onset, basin flooding, cholera clusters, rift-zone quakes) are in the same public feeds; nothing was turning them into events a regional coordination system could act on.
|
|
19
|
+
|
|
20
|
+
`coord-ingest` is that missing rail: an MIT-licensed adapter that converts open, key-free world-signal feeds into [`africa-coord-bus`](https://github.com/gabrielmahia/africa-coord-bus) coordination events, **filtered to East Africa**, so the bus can cascade them to the right domains (insurance, crop advisory, water testing, cross-border alerts).
|
|
21
|
+
|
|
22
|
+
It ingests and normalizes; the bus routes. Clean separation, composable rails.
|
|
23
|
+
|
|
24
|
+
## What it does
|
|
25
|
+
|
|
26
|
+
- **Adapters** for public sources — USGS earthquakes, ReliefWeb disasters, GDACS alerts — each mapping raw records to typed `CoordinationEvent`s. Add a feed by adding an adapter.
|
|
27
|
+
- **East-Africa filter** — a signal becomes an event only if it lands in the regional bounding box or names a regional country. Global noise is dropped at ingest.
|
|
28
|
+
- **Offline-capable** — ships with a `SampleAdapter` over bundled fixtures, so the full pipeline (and its tests) run with no network and no API keys.
|
|
29
|
+
- **Feeds the bus, doesn't replace it** — routing, cascading, and cross-border logic stay in `africa-coord-bus`.
|
|
30
|
+
|
|
31
|
+
## Use it
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
pip install coord-ingest
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
from coord_ingest import IngestPipeline, SampleAdapter
|
|
39
|
+
|
|
40
|
+
# Offline demo — no keys, no network
|
|
41
|
+
summary = IngestPipeline([SampleAdapter()]).run()
|
|
42
|
+
print(summary["collected"], "East Africa events") # global noise already dropped
|
|
43
|
+
print(summary["by_domain"]) # {'water': 3, 'health': 1, 'transport': 1}
|
|
44
|
+
|
|
45
|
+
# With live feeds + the bus:
|
|
46
|
+
# from coord_ingest import USGSQuakeAdapter, GDACSAdapter
|
|
47
|
+
# from africa_coord_bus import EventBus
|
|
48
|
+
# IngestPipeline([USGSQuakeAdapter(), GDACSAdapter()]).run(bus=EventBus(...))
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
## Why not just fork a dashboard?
|
|
52
|
+
|
|
53
|
+
The global open-source dashboards that inspired this are AGPL applications. This is deliberately the opposite: a small **MIT library** that produces standard bus events any tool can consume — including a dashboard, if someone wants to build one. Rails, not another train.
|
|
54
|
+
|
|
55
|
+
## IP & Collaboration
|
|
56
|
+
|
|
57
|
+
MIT. New feed adapters welcome via GitHub Issues (public, key-free sources, East-Africa relevant). Full policy: [docs/architecture/IP_POLICY.md](docs/architecture/IP_POLICY.md). Security: [SECURITY.md](SECURITY.md).
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/coord_ingest/__init__.py
|
|
5
|
+
src/coord_ingest/adapters.py
|
|
6
|
+
src/coord_ingest/pipeline.py
|
|
7
|
+
src/coord_ingest.egg-info/PKG-INFO
|
|
8
|
+
src/coord_ingest.egg-info/SOURCES.txt
|
|
9
|
+
src/coord_ingest.egg-info/dependency_links.txt
|
|
10
|
+
src/coord_ingest.egg-info/requires.txt
|
|
11
|
+
src/coord_ingest.egg-info/top_level.txt
|
|
12
|
+
src/coord_ingest/data/sample_feed.json
|
|
13
|
+
tests/test_ingest.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
africa-coord-bus>=0.2.0
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
coord_ingest
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Tests: adapters emit valid events, region filter works, pipeline runs offline."""
|
|
2
|
+
from africa_coord_bus.event import CoordinationEvent, EventDomain, EventSeverity
|
|
3
|
+
from coord_ingest import (SampleAdapter, USGSQuakeAdapter, GDACSAdapter,
|
|
4
|
+
IngestPipeline, in_east_africa)
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def test_sample_adapter_emits_events():
|
|
8
|
+
evs = SampleAdapter().to_events()
|
|
9
|
+
assert len(evs) == 6
|
|
10
|
+
assert all(isinstance(e, CoordinationEvent) for e in evs)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def test_region_filter_drops_global_noise():
|
|
14
|
+
pipe = IngestPipeline([SampleAdapter()])
|
|
15
|
+
events = pipe.collect()
|
|
16
|
+
# 6 fixtures, 1 is London -> must be dropped
|
|
17
|
+
assert len(events) == 5
|
|
18
|
+
titles = [e.data["title"] for e in events]
|
|
19
|
+
assert not any("GLOBAL NOISE" in t for t in titles)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def test_region_filter_by_country_and_bbox():
|
|
23
|
+
e_ke = CoordinationEvent(domain=EventDomain.WATER, event_type="x", source="t",
|
|
24
|
+
data={"country": "Kenya"})
|
|
25
|
+
e_uk = CoordinationEvent(domain=EventDomain.WATER, event_type="x", source="t",
|
|
26
|
+
data={"country": "United Kingdom", "lat": 51.5, "lon": -0.1})
|
|
27
|
+
e_geo = CoordinationEvent(domain=EventDomain.WATER, event_type="x", source="t",
|
|
28
|
+
data={"lat": -1.3, "lon": 36.8}) # Nairobi, no country
|
|
29
|
+
assert in_east_africa(e_ke)
|
|
30
|
+
assert not in_east_africa(e_uk)
|
|
31
|
+
assert in_east_africa(e_geo)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_usgs_adapter_maps_magnitude_to_severity():
|
|
35
|
+
feature = {"properties": {"place": "near Nairobi", "mag": 6.3},
|
|
36
|
+
"geometry": {"coordinates": [36.8, -1.3, 10]}}
|
|
37
|
+
evs = USGSQuakeAdapter(records=[feature]).to_events()
|
|
38
|
+
assert len(evs) == 1
|
|
39
|
+
assert evs[0].severity == EventSeverity.ALERT # 6.0-6.9
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def test_gdacs_colour_maps_to_severity():
|
|
43
|
+
rec = {"properties": {"alertlevel": "Red", "eventname": "Cyclone",
|
|
44
|
+
"latitude": -6.8, "longitude": 39.2, "country": "Tanzania"}}
|
|
45
|
+
evs = GDACSAdapter(records=[rec]).to_events()
|
|
46
|
+
assert evs[0].severity == EventSeverity.CRITICAL
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def test_pipeline_summary_offline():
|
|
50
|
+
summary = IngestPipeline([SampleAdapter()]).run()
|
|
51
|
+
assert summary["collected"] == 5
|
|
52
|
+
assert "water" in summary["by_domain"]
|
|
53
|
+
assert summary["published"] == 0 # no bus passed
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def test_pipeline_survives_a_broken_adapter():
|
|
57
|
+
class Broken:
|
|
58
|
+
def to_events(self): raise RuntimeError("feed down")
|
|
59
|
+
summary = IngestPipeline([Broken(), SampleAdapter()]).run()
|
|
60
|
+
assert summary["collected"] == 5 # good adapter still ran
|