geosint 1.0.0__py3-none-any.whl

geosint/__init__.py

@@ -0,0 +1,8 @@
+"""geosint - geospatial OSINT collection and monitoring.
+
+Collects points of interest and organizations from public map data,
+records provenance for every observation, and reports what changed
+between collections.
+"""
+
+__version__ = "1.0.0"

geosint/__main__.py

@@ -0,0 +1,6 @@
+import sys
+
+from .cli import run
+
+if __name__ == "__main__":
+    sys.exit(run())

geosint/analysis.py

@@ -0,0 +1,314 @@
+"""Correlation across collected entities.
+
+Everything else in this tool describes entities one at a time. This module
+looks for the relationships BETWEEN them, which is where most of the
+investigative value actually sits.
+
+The core question: which entities that present as unrelated are in fact
+connected? Four kinds of link are cheap to compute from data already
+collected, and each has a plain-language meaning:
+
+  shared phone   - two "different" organisations answering one number.
+                   Common for genuine reasons (a shared reception, a
+                   franchise call centre, a serviced office) and for
+                   interesting ones (one operator behind several fronts).
+  shared domain  - two entities publishing the same website host. Same
+                   reading: often a chain, sometimes not.
+  operator/brand - an explicit self-declared affiliation in the map data.
+                   The strongest link available because the source states it.
+  co-location    - entities at, or within metres of, the same coordinates.
+
+A link is a LEAD FOR AN ANALYST, never a conclusion. Every function here
+returns the evidence and leaves the judgement to a person - the labels say
+"shared", not "same owner", because the data supports the first and not
+the second.
+"""
+
+from __future__ import annotations
+
+import math
+import sqlite3
+from collections import defaultdict
+from contextlib import closing
+from typing import Dict, List, Optional
+
+from .geo import (
+    KM_PER_DEGREE_LAT,
+    KM_PER_DEGREE_LON_EQUATOR,
+    haversine_km,
+    parse_coord,
+    valid_point,
+)
+
+# Short codes and emergency numbers legitimately serve many unrelated
+# organisations, so a cluster built on one is noise. Matched EXACTLY, not
+# as substrings: "999" appears inside a huge share of ordinary numbers
+# (+1 555 019 9999 contains it), and substring matching silently dropped
+# them from every correlation.
+EMERGENCY_NUMBERS = frozenset({
+    "111", "112", "911", "999", "000", "119", "110", "115", "118", "122",
+})
+# Anything this short is a service or short code rather than a line
+# belonging to one organisation.
+MIN_REAL_PHONE_DIGITS = 6
+
+# Above this many members, a "shared" value is almost certainly a shared
+# service (a booking platform, a council switchboard) rather than a link
+# between the entities themselves.
+DEFAULT_MAX_CLUSTER = 25
+
+
+def is_generic_number(phone_norm: Optional[str]) -> bool:
+    """True for short codes and emergency numbers, which link nothing."""
+    digits = (phone_norm or "").lstrip("+")
+    if not digits:
+        return False
+    return digits in EMERGENCY_NUMBERS or len(digits) < MIN_REAL_PHONE_DIGITS
+
+
+def _key(row: dict) -> tuple:
+    """An entity's identity: its source id within its case."""
+    return (row["entity_id"], row.get("case_name") or "")
+
+
+def _fetch(conn: sqlite3.Connection, case_name: Optional[str]) -> List[dict]:
+    sql = "SELECT * FROM entities"
+    params: list = []
+    if case_name:
+        sql += ' WHERE case_name = ?'
+        params.append(case_name)
+    with closing(conn.cursor()) as cur:
+        cur.execute(sql, params)
+        return [dict(r) for r in cur.fetchall()]
+
+
+def _cluster(
+    rows: List[dict], key_field: str, max_cluster: int
+) -> List[dict]:
+    buckets: Dict[str, List[dict]] = defaultdict(list)
+    for row in rows:
+        key = (row.get(key_field) or "").strip()
+        if not key:
+            continue
+        buckets[key].append(row)
+
+    out = []
+    for key, members in buckets.items():
+        if len(members) < 2 or len(members) > max_cluster:
+            continue
+        out.append({
+            "key": key,
+            "count": len(members),
+            "members": sorted(members, key=lambda m: (m.get("name") or "")),
+            "names": sorted({m.get("name") or "(unnamed)" for m in members}),
+        })
+    return sorted(out, key=lambda c: -c["count"])
+
+
+def shared_phones(
+    conn: sqlite3.Connection,
+    case_name: Optional[str] = None,
+    max_cluster: int = DEFAULT_MAX_CLUSTER,
+) -> List[dict]:
+    rows = [r for r in _fetch(conn, case_name) if not is_generic_number(r.get("phone_norm"))]
+    clusters = _cluster(rows, "phone_norm", max_cluster)
+    # A single organisation mapped twice is a dedupe miss, not a link.
+    return [c for c in clusters if len(set(c["names"])) > 1]
+
+
+def shared_domains(
+    conn: sqlite3.Connection,
+    case_name: Optional[str] = None,
+    max_cluster: int = DEFAULT_MAX_CLUSTER,
+) -> List[dict]:
+    clusters = _cluster(_fetch(conn, case_name), "domain", max_cluster)
+    return [c for c in clusters if len(set(c["names"])) > 1]
+
+
+def affiliations(
+    conn: sqlite3.Connection, case_name: Optional[str] = None
+) -> List[dict]:
+    """Operator/brand groupings - affiliation the source states outright.
+
+    Not capped like the coincidental links: a chain with 400 branches
+    genuinely is one affiliation, and saying so is the finding. Still
+    requires two distinct names, for the same reason shared_phones does -
+    one organisation mapped twice is a dedupe miss, not a link.
+    """
+    rows = _fetch(conn, case_name)
+    out = []
+    for field in ("operator", "brand"):
+        for cluster in _cluster(rows, field, max_cluster=10_000):
+            if len(set(cluster["names"])) < 2:
+                continue
+            cluster["link"] = field
+            out.append(cluster)
+    return sorted(out, key=lambda c: -c["count"])
+
+
+def co_located(
+    conn: sqlite3.Connection,
+    case_name: Optional[str] = None,
+    radius_m: float = 25.0,
+    max_cluster: int = DEFAULT_MAX_CLUSTER,
+) -> List[dict]:
+    """Entities within `radius_m` of each other.
+
+    Proximity is transitive for clustering purposes: if A is near B and B
+    is near C, all three belong to one cluster even when A and C are
+    further apart than the threshold. That makes this a connected-
+    components problem, so it uses union-find.
+
+    An earlier version tracked group membership in a dict and handled only
+    three of the four cases - both endpoints unassigned, and either one
+    unassigned - silently doing nothing when a linking pair joined two
+    groups that already existed. A continuous line of entities was then
+    reported as two or three separate clusters depending on the order
+    SQLite happened to return the rows in, which is the worst property a
+    correlation feature can have: the same data gives a different answer.
+
+    Cost: O(n) grid bucketing plus true distance only for candidate pairs
+    in adjacent cells. Fine for a case-sized set, and deliberately not
+    meant for a country-scale extract - filter by case for those.
+    """
+    rows = [
+        r for r in _fetch(conn, case_name)
+        if valid_point(parse_coord(r.get("lat")), parse_coord(r.get("lon")))
+    ]
+    if radius_m <= 0 or len(rows) < 2:
+        return []
+    radius_km = radius_m / 1000.0
+
+    # The grid only prunes candidate pairs, so it is correct exactly when a
+    # point within `radius_km` is guaranteed to land in the same cell or an
+    # adjacent one. That means the cell must be at least the radius wide in
+    # BOTH axes, in degrees:
+    #
+    #   * latitude degrees are a constant ~110.574 km, so that side is easy.
+    #   * longitude degrees shrink towards the poles, so a cell sized for
+    #     the equator is far too narrow at high latitude. A fixed 0.01
+    #     degrees - the previous value - is ~1.1 km at the equator but only
+    #     ~2 m at 89.9 degrees, where a 25 m threshold would then need to
+    #     reach thirteen cells and silently did not.
+    #
+    # Sizing the longitude cell for the highest latitude actually present
+    # keeps one uniform grid. Near the poles that makes cells very wide,
+    # which degrades to latitude-band bucketing - slower, still correct.
+    lats = [parse_coord(r["lat"]) for r in rows]
+    worst_lat = min(max(abs(v) for v in lats), 89.9)
+    lat_cell = max(radius_km / KM_PER_DEGREE_LAT, 1e-9)
+    lon_cell = min(
+        max(radius_km / (KM_PER_DEGREE_LON_EQUATOR *
+                         math.cos(math.radians(worst_lat))), 1e-9),
+        360.0,
+    )
+
+    grid: Dict[tuple, List[dict]] = defaultdict(list)
+    for r in rows:
+        lat, lon = parse_coord(r["lat"]), parse_coord(r["lon"])
+        # floor(), not int(): int() truncates towards zero, which makes the
+        # cells straddling the equator and the prime meridian twice as wide
+        # as every other cell.
+        grid[(math.floor(lat / lat_cell), math.floor(lon / lon_cell))].append(r)
+
+    # Keyed by (entity_id, case_name): with no case filter, the same place
+    # held by two cases is two rows, and must stay two nodes here.
+    parent: Dict[tuple, tuple] = {_key(r): _key(r) for r in rows}
+
+    def find(x: tuple) -> tuple:
+        while parent[x] != x:
+            parent[x] = parent[parent[x]]  # path halving
+            x = parent[x]
+        return x
+
+    def union(x: tuple, y: tuple) -> None:
+        rx, ry = find(x), find(y)
+        if rx != ry:
+            parent[ry] = rx
+
+    # list() because grid is a defaultdict: .get() below must not be
+    # changed to grid[...], which would insert empty cells mid-iteration.
+    for (gy, gx), bucket in list(grid.items()):
+        neighbourhood: List[dict] = []
+        for dy in (-1, 0, 1):
+            for dx in (-1, 0, 1):
+                neighbourhood.extend(grid.get((gy + dy, gx + dx), []))
+        for a in bucket:
+            for b in neighbourhood:
+                if _key(a) >= _key(b):
+                    continue  # each unordered pair is considered once
+                if find(_key(a)) == find(_key(b)):
+                    continue  # already known to be connected
+                d = haversine_km(
+                    parse_coord(a["lat"]), parse_coord(a["lon"]),
+                    parse_coord(b["lat"]), parse_coord(b["lon"]),
+                )
+                if d <= radius_km:
+                    union(_key(a), _key(b))
+
+    by_root: Dict[tuple, List[dict]] = defaultdict(list)
+    for r in rows:
+        by_root[find(_key(r))].append(r)
+
+    out = []
+    for members in by_root.values():
+        if len(members) < 2:
+            continue
+        names = sorted({m.get("name") or "(unnamed)" for m in members})
+        if len(names) < 2 or len(members) > max_cluster:
+            continue
+        out.append({
+            "key": f"{len(members)} entities within {radius_m:.0f}m",
+            "count": len(members),
+            "members": sorted(members, key=lambda m: (m.get("name") or "")),
+            "names": names,
+        })
+    # Sort is stable and fully specified so the same data always prints in
+    # the same order regardless of how SQLite returned the rows.
+    return sorted(out, key=lambda c: (-c["count"], c["names"][0]))
+
+
+def link_summary(
+    conn: sqlite3.Connection, case_name: Optional[str] = None
+) -> Dict[str, List[dict]]:
+    return {
+        "shared_phone": shared_phones(conn, case_name),
+        "shared_domain": shared_domains(conn, case_name),
+        "affiliation": affiliations(conn, case_name),
+        "co_located": co_located(conn, case_name),
+    }
+
+
+def apply_link_indicators(conn: sqlite3.Connection, case_name: Optional[str] = None) -> int:
+    """Write shared_phone / shared_domain back onto the entities they
+    apply to, so the links are queryable with the same filters as every
+    other indicator instead of living only in a report.
+
+    Recomputed from scratch for the case each time: an entity whose
+    number or domain is no longer shared loses the tag. Only adding meant
+    a link that had stopped holding was reported forever."""
+    from .entities import IND_SHARED_DOMAIN, IND_SHARED_PHONE
+
+    link_slugs = {IND_SHARED_PHONE, IND_SHARED_DOMAIN}
+    tagged: Dict[tuple, set] = defaultdict(set)
+    for cluster in shared_phones(conn, case_name):
+        for m in cluster["members"]:
+            tagged[_key(m)].add(IND_SHARED_PHONE)
+    for cluster in shared_domains(conn, case_name):
+        for m in cluster["members"]:
+            tagged[_key(m)].add(IND_SHARED_DOMAIN)
+
+    updated = 0
+    with closing(conn.cursor()) as cur:
+        for row in _fetch(conn, case_name):
+            current = {s for s in (row.get("indicators") or "").split(",") if s}
+            merged = (current - link_slugs) | tagged.get(_key(row), set())
+            if merged != current:
+                cur.execute(
+                    "UPDATE entities SET indicators = ? "
+                    "WHERE entity_id = ? AND case_name = ?",
+                    (",".join(sorted(merged)), *_key(row)),
+                )
+                updated += 1
+    conn.commit()
+    return updated

geosint/api_client.py

@@ -0,0 +1,202 @@
+"""Retrying wrapper around Google's Places API (New).
+
+Deliberately targets the CURRENT API - the older "Places API (Legacy)"
+endpoints (maps.googleapis.com/maps/api/place/...) can no longer be
+enabled on new Google Cloud projects at all (Google froze Legacy to new
+projects in March 2025). A script built against Legacy fails outright
+with REQUEST_DENIED for anyone setting this up fresh in 2026.
+
+Docs: https://developers.google.com/maps/documentation/places/web-service/text-search
+
+Cost note: Text Search (New) bills at Pro-tier pricing by default (5,000
+free calls/month). Adding rating/review fields to the field mask bumps
+the ENTIRE call to Enterprise-tier pricing (only 1,000 free calls/month,
+~$35-40/1,000 after that) - Google bills every field in a response at the
+highest tier any single requested field belongs to. This client leaves
+rating/review fields OUT of the default field mask for that reason; pass
+include_ratings=True if you specifically want them and accept the cost.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from typing import Optional, Tuple
+
+import requests
+
+from .exceptions import PlacesAPIError
+from .geo import bbox_around
+
+logger = logging.getLogger(__name__)
+
+SEARCH_TEXT_URL = "https://places.googleapis.com/v1/places:searchText"
+
+BASE_FIELDS = (
+    "places.id,places.displayName,places.formattedAddress,"
+    "places.nationalPhoneNumber,places.websiteUri,places.businessStatus,"
+    "places.location,places.types,nextPageToken"
+)
+RATING_FIELDS = "places.rating,places.userRatingCount"
+
+RETRYABLE_HTTP_STATUSES = {429, 500, 502, 503, 504}
+
+
+class PlacesClient:
+    """Text Search (New) already returns phone/website/type data in one
+    call - no separate per-place Details request needed, unlike the old
+    Legacy flow (search, then N detail calls). That means the billable
+    unit here is roughly one call per ~20 results, not one per business."""
+
+    def __init__(
+        self,
+        api_key: str,
+        max_retries: int = 4,
+        timeout: float = 15.0,
+        include_ratings: bool = False,
+        session: Optional[requests.Session] = None,
+    ) -> None:
+        self.api_key = api_key
+        self.max_retries = max_retries
+        self.timeout = timeout
+        self.include_ratings = include_ratings
+        self.session = session or requests.Session()
+        if include_ratings:
+            logger.warning(
+                "include_ratings=True: this call now bills at Enterprise-tier "
+                "pricing (only 1,000 free calls/month) instead of Pro-tier "
+                "(5,000 free calls/month)."
+            )
+
+    @property
+    def _field_mask(self) -> str:
+        return BASE_FIELDS + ("," + RATING_FIELDS if self.include_ratings else "")
+
+    def _post(self, body: dict) -> dict:
+        headers = {
+            "Content-Type": "application/json",
+            "X-Goog-Api-Key": self.api_key,
+            "X-Goog-FieldMask": self._field_mask,
+        }
+        last_error: Optional[Exception] = None
+
+        for attempt in range(1, self.max_retries + 1):
+            try:
+                resp = self.session.post(
+                    SEARCH_TEXT_URL, json=body, headers=headers, timeout=self.timeout
+                )
+            except requests.RequestException as exc:
+                last_error = exc
+                wait = min(2**attempt, 30)
+                logger.warning(
+                    "Request failed (attempt %d/%d): %s - retrying in %ds",
+                    attempt, self.max_retries, exc, wait,
+                )
+                time.sleep(wait)
+                continue
+
+            if resp.status_code == 200:
+                return resp.json()
+
+            if resp.status_code in RETRYABLE_HTTP_STATUSES and attempt < self.max_retries:
+                wait = min(2**attempt, 30)
+                logger.warning(
+                    "HTTP %d from Places API - retrying in %ds (attempt %d/%d)",
+                    resp.status_code, wait, attempt, self.max_retries,
+                )
+                time.sleep(wait)
+                continue
+
+            raise PlacesAPIError(
+                f"Places API error: HTTP {resp.status_code} - {resp.text[:300]}"
+            )
+
+        raise PlacesAPIError(
+            f"Request failed after {self.max_retries} attempts: {last_error}"
+        )
+
+    def text_search(
+        self,
+        query: str,
+        max_results: int,
+        center: Optional[Tuple[float, float]] = None,
+        radius_km: Optional[float] = None,
+    ) -> list:
+        """Paginate Text Search (New) up to max_results.
+
+        `center`/`radius_km` become a locationRestriction rectangle
+        enclosing the circle, which is Places' own way of bounding a text
+        query; the results still need narrowing to the circle. Before this, the CLI
+        accepted --near/--radius for --provider google, validated the
+        coordinates, and then dropped them: the request went out as
+        "pharmacy in " with no location at all, returned results from
+        anywhere on Earth, and stored them with a full provenance block
+        that said nothing about the constraint having been discarded.
+        """
+        results: list = []
+        base: dict = {"textQuery": query}
+        if center is not None and radius_km:
+            # Text Search (New) accepts only a rectangular viewport as a
+            # locationRestriction - a circle is valid for locationBias
+            # alone, and sending one got the request rejected. So this
+            # sends the box enclosing the circle; the caller narrows the
+            # results to the true radius, as the OSM path does.
+            box = bbox_around(center[0], center[1], float(radius_km))
+            base["locationRestriction"] = {
+                "rectangle": {
+                    "low": {"latitude": float(box["south"]),
+                            "longitude": float(box["west"])},
+                    "high": {"latitude": float(box["north"]),
+                             "longitude": float(box["east"])},
+                }
+            }
+        body = dict(base)
+        page = 1
+
+        while True:
+            data = self._post(body)
+            batch = data.get("places", [])
+            results.extend(batch)
+            logger.info(
+                "Page %d: +%d results (total %d)", page, len(batch), len(results)
+            )
+
+            token = data.get("nextPageToken")
+            if not token or len(results) >= max_results:
+                break
+
+            time.sleep(2)  # short delay before a new pageToken becomes valid
+            body = dict(base)
+            body["pageToken"] = token
+            page += 1
+
+        return [_normalize(p) for p in results[:max_results]]
+
+
+def _normalize(place: dict) -> dict:
+    """Map Places API (New) field names onto the source-neutral shape
+    entities.py expects (the same shape the OSM collector produces)."""
+    location = place.get("location") or {}
+    return {
+        "source": "google",
+        "source_ref": place.get("id", ""),
+        "name": (place.get("displayName") or {}).get("text", ""),
+        "phone": place.get("nationalPhoneNumber", ""),
+        "website": place.get("websiteUri"),
+        "email": "",
+        "lat": location.get("latitude"),
+        "lon": location.get("longitude"),
+        "address": place.get("formattedAddress", ""),
+        "postcode": "",
+        "country": "",
+        "operator": "",
+        "brand": "",
+        "ref": "",
+        "wikidata": "",
+        "opening_hours": "",
+        "survey_date": "",
+        "operational_status": place.get("businessStatus", "OPERATIONAL"),
+        "rating": place.get("rating"),
+        "reviews": place.get("userRatingCount"),
+        "types": place.get("types", []),
+    }

geosint/batch.py

@@ -0,0 +1,69 @@
+"""Batch mode: run many collections from one CSV in a single command.
+
+The columns are `what,area[,osm_tag]`. `niche,city` - the vocabulary this
+tool used before the OSINT pivot - is still accepted as an alias so CSVs
+written against the old names keep working, but the field names on
+BatchQuery are the current ones: keeping `niche`/`city` in the code meant
+the pivot was cosmetic everywhere the reader could not see.
+"""
+
+from __future__ import annotations
+
+import csv
+from dataclasses import dataclass
+from typing import List, Optional
+
+WHAT_COLUMNS = ("what", "niche")
+AREA_COLUMNS = ("area", "city")
+
+
+@dataclass
+class BatchQuery:
+    what: str
+    area: str
+    osm_tag: Optional[str] = None
+
+
+def _pick_column(fieldnames: List[str], candidates: tuple) -> Optional[str]:
+    for candidate in candidates:
+        if candidate in fieldnames:
+            return candidate
+    return None
+
+
+def load_batch_queries(path: str) -> List[BatchQuery]:
+    """Read a CSV with a header row containing at least 'what' and 'area'
+    (or the older 'niche' and 'city'); 'osm_tag' is optional per row."""
+    with open(path, newline="", encoding="utf-8") as f:
+        reader = csv.DictReader(f)
+        fieldnames = [name.strip().lower() for name in (reader.fieldnames or [])]
+        what_col = _pick_column(fieldnames, WHAT_COLUMNS)
+        area_col = _pick_column(fieldnames, AREA_COLUMNS)
+        if not what_col or not area_col:
+            raise ValueError(
+                "Batch CSV needs a header row with 'what' and 'area' columns "
+                "(or the older 'niche' and 'city'), optionally 'osm_tag'."
+            )
+
+        # DictReader keys come from the raw header, which may differ from
+        # the lowercased name matched above.
+        raw_by_lower = {n.strip().lower(): n for n in (reader.fieldnames or [])}
+        what_key = raw_by_lower[what_col]
+        area_key = raw_by_lower[area_col]
+        tag_key = raw_by_lower.get("osm_tag")
+
+        queries: List[BatchQuery] = []
+        for row in reader:
+            what = (row.get(what_key) or "").strip()
+            area = (row.get(area_key) or "").strip()
+            osm_tag = (row.get(tag_key) or "").strip() if tag_key else ""
+            if not what and not osm_tag:
+                continue
+            if not area:
+                continue
+            queries.append(BatchQuery(what=what, area=area, osm_tag=osm_tag or None))
+
+    if not queries:
+        raise ValueError(f"No valid (what, area) rows found in {path}.")
+
+    return queries