groundsource 0.1.0__py3-none-any.whl

groundsource/__init__.py

@@ -0,0 +1,6 @@
+"""groundsource — Python package for Google's Groundsource flash flood dataset."""
+
+from groundsource.db import FloodDB
+
+__version__ = "0.1.0"
+__all__ = ["FloodDB"]

groundsource/cache.py

@@ -0,0 +1,63 @@
+"""Download and cache management for the Groundsource dataset."""
+
+import os
+import sys
+import urllib.request
+from pathlib import Path
+
+ZENODO_URL = "https://zenodo.org/records/18647054/files/groundsource_2026.parquet?download=1"
+PARQUET_FILENAME = "groundsource_2026.parquet"
+ENRICHED_FILENAME = "groundsource_enriched.parquet"
+
+
+def get_cache_dir() -> Path:
+    """Return platform-appropriate cache directory."""
+    if sys.platform == "win32":
+        base = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local"))
+    elif sys.platform == "darwin":
+        base = Path.home() / "Library" / "Caches"
+    else:
+        base = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache"))
+    cache_dir = base / "groundsource"
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    return cache_dir
+
+
+def get_raw_parquet_path() -> Path:
+    return get_cache_dir() / PARQUET_FILENAME
+
+
+def get_enriched_parquet_path() -> Path:
+    return get_cache_dir() / ENRICHED_FILENAME
+
+
+def download_parquet(force: bool = False) -> Path:
+    """Download the raw Parquet from Zenodo if not already cached."""
+    path = get_raw_parquet_path()
+    if path.exists() and not force:
+        return path
+
+    print(f"Downloading Groundsource dataset ({PARQUET_FILENAME})...")
+    print(f"Source: {ZENODO_URL}")
+    print(f"Destination: {path}")
+
+    def _progress(block_num, block_size, total_size):
+        downloaded = block_num * block_size
+        if total_size > 0:
+            pct = min(100, downloaded * 100 / total_size)
+            mb = downloaded / (1024 * 1024)
+            total_mb = total_size / (1024 * 1024)
+            sys.stdout.write(f"\r  {mb:.0f}/{total_mb:.0f} MB ({pct:.1f}%)")
+            sys.stdout.flush()
+
+    urllib.request.urlretrieve(ZENODO_URL, str(path), reporthook=_progress)
+    print("\n  Download complete.")
+    return path
+
+
+def load_from_local(path: str) -> Path:
+    """Use a local Parquet file instead of downloading."""
+    p = Path(path)
+    if not p.exists():
+        raise FileNotFoundError(f"Local parquet not found: {path}")
+    return p

groundsource/charts.py

@@ -0,0 +1,206 @@
+"""Chart generators for Groundsource analysis. LinkedIn-worthy matplotlib charts."""
+
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import matplotlib.ticker as ticker
+
+
+# Consistent style for all charts
+STYLE = {
+    "figure.facecolor": "white",
+    "axes.facecolor": "#f8f9fa",
+    "axes.grid": True,
+    "grid.alpha": 0.3,
+    "font.family": "sans-serif",
+    "font.size": 11,
+}
+
+
+def _apply_style():
+    plt.rcParams.update(STYLE)
+
+
+def plot_hockey_stick(yearly_counts: pd.DataFrame, save_path: str = None) -> plt.Figure:
+    """Chart 1: Total events per year — the 807x hockey stick.
+
+    yearly_counts: DataFrame with columns [year, count]
+    """
+    _apply_style()
+    fig, ax = plt.subplots(figsize=(12, 6))
+
+    ax.bar(yearly_counts["year"], yearly_counts["count"],
+           color="#1a73e8", alpha=0.85, width=0.8)
+
+    # Annotate the growth
+    first_year = yearly_counts.iloc[0]
+    peak_year = yearly_counts.loc[yearly_counts["count"].idxmax()]
+    growth = peak_year["count"] / first_year["count"] if first_year["count"] > 0 else 0
+
+    ax.annotate(
+        f'{first_year["count"]:,.0f} events',
+        xy=(first_year["year"], first_year["count"]),
+        xytext=(first_year["year"] + 3, peak_year["count"] * 0.3),
+        fontsize=10, color="#666",
+        arrowprops=dict(arrowstyle="->", color="#999"),
+    )
+    ax.annotate(
+        f'{peak_year["count"]:,.0f} events\n({growth:,.0f}x growth)',
+        xy=(peak_year["year"], peak_year["count"]),
+        xytext=(peak_year["year"] - 6, peak_year["count"] * 1.05),
+        fontsize=10, fontweight="bold", color="#d93025",
+        arrowprops=dict(arrowstyle="->", color="#d93025"),
+    )
+
+    ax.set_title("Flash Flood Events Detected Per Year\n— or is it news coverage?",
+                 fontsize=16, fontweight="bold", pad=15)
+    ax.set_xlabel("Year", fontsize=12)
+    ax.set_ylabel("Events Detected", fontsize=12)
+    ax.yaxis.set_major_formatter(ticker.FuncFormatter(lambda x, _: f"{x:,.0f}"))
+
+    ax.text(0.02, 0.95,
+            "Source: Google Groundsource dataset (2.6M events from news articles, 2000–2026)",
+            transform=ax.transAxes, fontsize=8, color="#999", va="top")
+
+    plt.tight_layout()
+    if save_path:
+        fig.savefig(save_path, dpi=150, bbox_inches="tight")
+        print(f"  Saved: {save_path}")
+    return fig
+
+
+def plot_bias_normalized(yearly_counts: pd.DataFrame, save_path: str = None) -> plt.Figure:
+    """Chart 2: Overlay event growth vs estimated digital news growth.
+
+    Uses a simple exponential proxy for global digital news volume.
+    Internet users: ~400M (2000) → ~5.5B (2025) ≈ 13.75x
+    Online news output grew even faster due to digital-native outlets.
+    We use a conservative 15-20x estimate for indexed news articles.
+    """
+    _apply_style()
+    fig, ax = plt.subplots(figsize=(12, 6))
+
+    years = yearly_counts["year"].values
+    counts = yearly_counts["count"].values
+
+    # Normalize both to year 2007 (when dataset has enough events to be meaningful)
+    ref_idx = np.where(years == 2007)[0]
+    if len(ref_idx) == 0:
+        ref_idx = [7]  # fallback
+    ref_idx = ref_idx[0]
+
+    norm_events = counts / counts[ref_idx]
+
+    # Conservative proxy: internet users grew ~14x from 2000 to 2025
+    # Online news articles grew faster. Use a simple logistic-like growth curve.
+    internet_users_billions = {
+        2000: 0.41, 2001: 0.50, 2002: 0.63, 2003: 0.72, 2004: 0.82,
+        2005: 1.02, 2006: 1.15, 2007: 1.37, 2008: 1.57, 2009: 1.77,
+        2010: 2.02, 2011: 2.23, 2012: 2.49, 2013: 2.73, 2014: 2.96,
+        2015: 3.19, 2016: 3.42, 2017: 3.65, 2018: 3.90, 2019: 4.13,
+        2020: 4.59, 2021: 4.90, 2022: 5.16, 2023: 5.35, 2024: 5.52,
+        2025: 5.56, 2026: 5.60,
+    }
+    news_proxy = np.array([internet_users_billions.get(y, 5.6) for y in years])
+    norm_news = news_proxy / news_proxy[ref_idx]
+
+    ax.plot(years, norm_events, "o-", color="#d93025", linewidth=2.5,
+            markersize=5, label="Detected flood events", zorder=3)
+    ax.plot(years, norm_news, "s--", color="#1a73e8", linewidth=2,
+            markersize=4, label="Internet users (proxy for digital news)", zorder=2)
+
+    ax.fill_between(years, norm_events, norm_news,
+                    where=(norm_events > norm_news),
+                    alpha=0.1, color="#d93025")
+
+    ax.set_title("Are Floods Increasing — or Is News Coverage?\nBoth curves normalized to 2007 = 1.0",
+                 fontsize=15, fontweight="bold", pad=15)
+    ax.set_xlabel("Year", fontsize=12)
+    ax.set_ylabel("Growth relative to 2007", fontsize=12)
+    ax.legend(fontsize=11, loc="upper left")
+    ax.set_yscale("log")
+    ax.yaxis.set_major_formatter(ticker.FuncFormatter(lambda x, _: f"{x:.0f}x"))
+
+    ax.text(0.02, 0.02,
+            "Event detection grows faster than internet adoption — likely because\n"
+            "news digitization (articles going online) grew faster than raw user count.",
+            transform=ax.transAxes, fontsize=9, color="#666", va="bottom",
+            style="italic")
+
+    plt.tight_layout()
+    if save_path:
+        fig.savefig(save_path, dpi=150, bbox_inches="tight")
+        print(f"  Saved: {save_path}")
+    return fig
+
+
+def plot_country_growth(growth_df: pd.DataFrame, n: int = 20,
+                        save_path: str = None) -> plt.Figure:
+    """Chart 3: Countries with highest growth factor — what might be real.
+
+    growth_df: from trends.country_growth_ranking()
+    """
+    _apply_style()
+    top = growth_df.head(n).copy()
+    top = top.sort_values("growth_factor", ascending=True)  # horizontal bar, ascending
+
+    fig, ax = plt.subplots(figsize=(10, 8))
+
+    colors = ["#d93025" if g > 50 else "#ea8600" if g > 20 else "#1a73e8"
+              for g in top["growth_factor"]]
+
+    ax.barh(top["country"], top["growth_factor"], color=colors, alpha=0.85)
+
+    for i, (_, row) in enumerate(top.iterrows()):
+        ax.text(row["growth_factor"] + 0.5, i,
+                f'{row["growth_factor"]:.0f}x',
+                va="center", fontsize=9, fontweight="bold")
+
+    ax.set_title("Which Countries Show the Fastest Growth in Detected Flood Events?\n"
+                 "Growth factor: avg events/year (2018–2025) vs (2005–2012)",
+                 fontsize=14, fontweight="bold", pad=15)
+    ax.set_xlabel("Growth Factor (higher = faster acceleration)", fontsize=11)
+
+    ax.text(0.98, 0.02,
+            "⚠ High growth may reflect news digitization, not actual flood increase.\n"
+            "Countries with low baseline coverage will show inflated growth.",
+            transform=ax.transAxes, fontsize=8, color="#999",
+            ha="right", va="bottom", style="italic")
+
+    plt.tight_layout()
+    if save_path:
+        fig.savefig(save_path, dpi=150, bbox_inches="tight")
+        print(f"  Saved: {save_path}")
+    return fig
+
+
+def plot_top_countries(country_counts: pd.DataFrame, n: int = 20,
+                       save_path: str = None) -> plt.Figure:
+    """Chart 4: Top N countries by total event count — the LinkedIn chart."""
+    _apply_style()
+    top = country_counts.head(n).copy()
+    top = top.sort_values("count", ascending=True)
+
+    fig, ax = plt.subplots(figsize=(10, 8))
+
+    ax.barh(top["country"], top["count"], color="#1a73e8", alpha=0.85)
+
+    for i, (_, row) in enumerate(top.iterrows()):
+        ax.text(row["count"] + 500, i,
+                f'{row["count"]:,.0f}',
+                va="center", fontsize=9)
+
+    ax.set_title("Top 20 Countries by Total Flash Flood Events Detected (2000–2026)",
+                 fontsize=14, fontweight="bold", pad=15)
+    ax.set_xlabel("Total Events Detected", fontsize=11)
+    ax.xaxis.set_major_formatter(ticker.FuncFormatter(lambda x, _: f"{x:,.0f}"))
+
+    ax.text(0.98, 0.02,
+            "Source: Google Groundsource — 2.6M events extracted by Gemini from news articles",
+            transform=ax.transAxes, fontsize=8, color="#999", ha="right", va="bottom")
+
+    plt.tight_layout()
+    if save_path:
+        fig.savefig(save_path, dpi=150, bbox_inches="tight")
+        print(f"  Saved: {save_path}")
+    return fig

groundsource/data/__init__.py

@@ -0,0 +1 @@
+# Data directory — contains Natural Earth shapefiles bundled with the package.

groundsource/data/ne_110m_admin_0_countries.cpg

@@ -0,0 +1 @@
+UTF-8

groundsource/data/ne_110m_admin_0_countries.prj

@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.017453292519943295]]

groundsource/db.py

@@ -0,0 +1,322 @@
+"""FloodDB — Main interface to the Groundsource flash flood dataset."""
+
+import pandas as pd
+import numpy as np
+
+from groundsource.cache import (
+    download_parquet,
+    get_enriched_parquet_path,
+    load_from_local,
+)
+from groundsource.spatial import enrich_dataframe, search_by_radius, search_by_bbox
+from groundsource import trends as _trends
+from groundsource import charts as _charts
+
+# Common country name aliases -> Natural Earth canonical name
+_COUNTRY_ALIASES = {
+    "usa": "United States of America",
+    "us": "United States of America",
+    "united states": "United States of America",
+    "america": "United States of America",
+    "uk": "United Kingdom",
+    "britain": "United Kingdom",
+    "great britain": "United Kingdom",
+    "england": "United Kingdom",
+    "uae": "United Arab Emirates",
+    "south korea": "South Korea",
+    "korea": "South Korea",
+    "north korea": "North Korea",
+    "dr congo": "Dem. Rep. Congo",
+    "democratic republic of congo": "Dem. Rep. Congo",
+    "drc": "Dem. Rep. Congo",
+    "congo": "Dem. Rep. Congo",
+    "ivory coast": "Ivory Coast",
+    "cote d'ivoire": "Ivory Coast",
+    "czech republic": "Czechia",
+    "czechia": "Czechia",
+    "bosnia": "Bosnia and Herz.",
+    "dominican republic": "Dominican Rep.",
+    "central african republic": "Central African Rep.",
+}
+
+# Simple built-in city geocoding (major cities only, no external API needed)
+# Format: city_name -> (lon, lat)
+_CITY_COORDS = {
+    "houston": (-95.37, 29.76), "mumbai": (72.88, 19.08),
+    "jakarta": (106.85, -6.21), "lagos": (3.39, 6.52),
+    "dhaka": (90.41, 23.81), "bangkok": (100.50, 13.76),
+    "manila": (120.98, 14.60), "kolkata": (88.36, 22.57),
+    "chennai": (80.27, 13.08), "delhi": (77.21, 28.61),
+    "new york": (-74.01, 40.71), "los angeles": (-118.24, 34.05),
+    "chicago": (-87.63, 41.88), "miami": (-80.19, 25.76),
+    "london": (-0.13, 51.51), "paris": (2.35, 48.86),
+    "berlin": (13.40, 52.52), "rome": (12.50, 41.90),
+    "madrid": (-3.70, 40.42), "tokyo": (139.69, 35.69),
+    "beijing": (116.41, 39.90), "shanghai": (121.47, 31.23),
+    "sydney": (151.21, -33.87), "melbourne": (144.96, -37.81),
+    "sao paulo": (-46.63, -23.55), "rio de janeiro": (-43.17, -22.91),
+    "buenos aires": (-58.38, -34.60), "mexico city": (-99.13, 19.43),
+    "cairo": (31.24, 30.04), "nairobi": (36.82, -1.29),
+    "johannesburg": (28.05, -26.20), "cape town": (18.42, -33.93),
+    "istanbul": (28.98, 41.01), "moscow": (37.62, 55.76),
+    "dubai": (55.30, 25.20), "singapore": (103.85, 1.29),
+    "kuala lumpur": (101.69, 3.14), "hanoi": (105.85, 21.03),
+    "ho chi minh city": (106.63, 10.82), "seoul": (126.98, 37.57),
+    "osaka": (135.50, 34.69), "lima": (-77.04, -12.05),
+    "bogota": (-74.07, 4.71), "santiago": (-70.65, -33.45),
+    "accra": (-0.19, 5.56), "kinshasa": (15.31, -4.32),
+    "addis ababa": (38.75, 9.02), "dar es salaam": (39.27, -6.79),
+    "karachi": (67.01, 24.86), "lahore": (74.35, 31.56),
+    "islamabad": (73.05, 33.69), "kabul": (69.17, 34.53),
+    "kathmandu": (85.32, 27.72), "colombo": (79.86, 6.93),
+    "yangon": (96.15, 16.87), "phnom penh": (104.92, 11.56),
+    "taipei": (121.57, 25.03), "hong kong": (114.17, 22.32),
+    "amsterdam": (4.90, 52.37), "brussels": (4.35, 50.85),
+    "vienna": (16.37, 48.21), "zurich": (8.54, 47.38),
+    "munich": (11.58, 48.14), "milan": (9.19, 45.46),
+    "barcelona": (2.17, 41.39), "lisbon": (-9.14, 38.74),
+    "athens": (23.73, 37.98), "bucharest": (26.10, 44.43),
+    "warsaw": (21.01, 52.23), "prague": (14.42, 50.08),
+    "budapest": (19.04, 47.50), "stockholm": (18.07, 59.33),
+    "oslo": (10.75, 59.91), "copenhagen": (12.57, 55.68),
+    "helsinki": (24.94, 60.17), "dublin": (-6.26, 53.35),
+    "toronto": (-79.38, 43.65), "vancouver": (-123.12, 49.28),
+    "montreal": (-73.57, 45.50), "abuja": (7.49, 9.06),
+    "new orleans": (-90.07, 29.95), "denver": (-104.99, 39.74),
+    "atlanta": (-84.39, 33.75), "dallas": (-96.80, 32.78),
+    "seattle": (-122.33, 47.61), "san francisco": (-122.42, 37.77),
+    "phoenix": (-112.07, 33.45), "las vegas": (-115.14, 36.17),
+    "washington": (-77.04, 38.91), "boston": (-71.06, 42.36),
+    "philadelphia": (-75.17, 39.95), "detroit": (-83.05, 42.33),
+    "minneapolis": (-93.27, 44.98), "portland": (-122.68, 45.52),
+    "pittsburgh": (-79.99, 40.44), "st louis": (-90.20, 38.63),
+    "nashville": (-86.78, 36.16), "charlotte": (-80.84, 35.23),
+}
+
+
+def _resolve_country(name: str, df: pd.DataFrame) -> pd.Series:
+    """Resolve a country name to a boolean mask, handling aliases and fuzzy matching."""
+    lower = name.lower()
+
+    # Check aliases first
+    if lower in _COUNTRY_ALIASES:
+        canonical = _COUNTRY_ALIASES[lower]
+        return df["country"] == canonical
+
+    # Exact case-insensitive match
+    mask = df["country"].str.lower() == lower
+    if mask.sum() > 0:
+        return mask
+
+    # Substring match as fallback
+    return df["country"].str.contains(name, case=False, na=False)
+
+
+class FloodDB:
+    """Interface to Google's Groundsource flash flood dataset (2.6M events, 2000–2026).
+
+    Usage:
+        from groundsource import FloodDB
+        db = FloodDB()  # auto-downloads from Zenodo if not cached
+        floods = db.search(country="India")
+        trend = db.trend("India")
+    """
+
+    def __init__(self, local_path: str = None):
+        """Load the Groundsource dataset.
+
+        Args:
+            local_path: Path to a local copy of groundsource_2026.parquet.
+                        If None, downloads from Zenodo and caches automatically.
+        """
+        enriched_path = get_enriched_parquet_path()
+
+        if enriched_path.exists():
+            print(f"Loading enriched dataset from cache...")
+            self._df = pd.read_parquet(enriched_path)
+            print(f"  Loaded {len(self._df):,} events.")
+        else:
+            # Load raw parquet
+            if local_path:
+                raw_path = load_from_local(local_path)
+            else:
+                raw_path = download_parquet()
+
+            print("Loading raw dataset...")
+            raw_df = pd.read_parquet(raw_path)
+            print(f"  Loaded {len(raw_df):,} raw events.")
+
+            # Enrich with spatial data
+            print("Enriching with country/continent data (one-time, ~2 min)...")
+            self._df = enrich_dataframe(raw_df)
+
+            # Cache enriched version
+            print(f"  Caching enriched dataset to {enriched_path}...")
+            self._df.to_parquet(enriched_path, index=False)
+            print("  Cached. Future loads will be instant.")
+
+    @property
+    def df(self) -> pd.DataFrame:
+        """Access the full enriched DataFrame."""
+        return self._df
+
+    def __len__(self) -> int:
+        return len(self._df)
+
+    def __repr__(self) -> str:
+        years = f"{self._df['year'].min()}–{self._df['year'].max()}"
+        countries = self._df["country"].nunique()
+        return f"FloodDB({len(self._df):,} events, {years}, {countries} countries)"
+
+    def info(self) -> dict:
+        """Summary statistics about the dataset."""
+        df = self._df
+        return {
+            "total_events": len(df),
+            "date_range": (df["start_date"].min(), df["start_date"].max()),
+            "countries": df["country"].nunique(),
+            "continents": sorted(df["continent"].dropna().unique().tolist()),
+            "untagged_events": int(df["country"].isna().sum()),
+            "columns": list(df.columns),
+        }
+
+    def to_dataframe(self) -> pd.DataFrame:
+        """Return the full enriched dataset as a pandas DataFrame."""
+        return self._df.copy()
+
+    @staticmethod
+    def available_cities() -> list:
+        """List all city names available for city search."""
+        return sorted(_CITY_COORDS.keys())
+
+    def available_countries(self) -> list:
+        """List all country names in the dataset."""
+        return sorted(self._df["country"].dropna().unique().tolist())
+
+    # ── Search ──────────────────────────────────────────────────────────
+
+    def search(self, country: str = None, continent: str = None,
+               city: str = None, radius_km: float = 100,
+               bbox: list = None,
+               year_range: tuple = None) -> pd.DataFrame:
+        """Search for flood events by location and/or time.
+
+        Args:
+            country: Country name (e.g., "India", "United States of America")
+            continent: Continent name (e.g., "Asia", "Africa")
+            city: City name for radius search (e.g., "Houston")
+            radius_km: Radius in km for city search (default 100)
+            bbox: Bounding box [min_lat, min_lon, max_lat, max_lon]
+            year_range: Tuple of (start_year, end_year) inclusive
+
+        Returns:
+            Filtered DataFrame of matching events.
+        """
+        result = self._df
+
+        if year_range:
+            result = result[(result["year"] >= year_range[0]) & (result["year"] <= year_range[1])]
+
+        if continent:
+            result = result[result["continent"].str.lower().fillna("") == continent.lower()]
+
+        if country:
+            mask = _resolve_country(country, result)
+            result = result[mask]
+
+        if city:
+            city_lower = city.lower()
+            if city_lower in _CITY_COORDS:
+                lon, lat = _CITY_COORDS[city_lower]
+                result = search_by_radius(result, lon, lat, radius_km)
+            else:
+                raise ValueError(
+                    f"City '{city}' not in built-in gazetteer. "
+                    f"Use bbox=[min_lat, min_lon, max_lat, max_lon] or "
+                    f"search(country=...) instead. "
+                    f"Available cities: {len(_CITY_COORDS)}"
+                )
+
+        if bbox:
+            result = search_by_bbox(result, bbox)
+
+        return result.copy()
+
+    # ── Trend Analysis ──────────────────────────────────────────────────
+
+    def trend(self, country: str = None, continent: str = None) -> pd.DataFrame:
+        """Yearly event counts, optionally filtered by country/continent.
+
+        Returns DataFrame with columns: year, count
+        """
+        df = self._df
+        if country:
+            df = df[_resolve_country(country, df)]
+        if continent:
+            df = df[df["continent"].str.lower().fillna("") == continent.lower()]
+        return _trends.yearly_counts(df)
+
+    def growth(self, country: str = None, continent: str = None,
+               early: tuple = (2005, 2012), recent: tuple = (2018, 2025)) -> dict:
+        """Compare average yearly events between two periods."""
+        counts = self.trend(country=country, continent=continent)
+        return _trends.growth_rate(counts, early=early, recent=recent)
+
+    def compare(self, countries: list) -> pd.DataFrame:
+        """Side-by-side yearly counts for multiple countries.
+
+        Accepts aliases (e.g., "USA", "UK") and case-insensitive names.
+        """
+        # Resolve each country name to its canonical form
+        resolved = []
+        for c in countries:
+            lower = c.lower()
+            if lower in _COUNTRY_ALIASES:
+                resolved.append(_COUNTRY_ALIASES[lower])
+            else:
+                # Find best match in dataset
+                mask = self._df["country"].str.lower() == lower
+                if mask.sum() > 0:
+                    resolved.append(self._df.loc[mask, "country"].iloc[0])
+                else:
+                    resolved.append(c)  # pass through, will show zeros
+        return _trends.compare_countries(self._df, resolved)
+
+    def top_countries(self, n: int = 20) -> pd.DataFrame:
+        """Top N countries by total event count."""
+        return _trends.top_countries(self._df, n=n)
+
+    def country_growth_ranking(self, n: int = 20, **kwargs) -> pd.DataFrame:
+        """Rank countries by flood event growth rate."""
+        return _trends.country_growth_ranking(self._df, n=n, **kwargs)
+
+    def bias_check(self) -> pd.DataFrame:
+        """Global yearly event counts for bias analysis.
+
+        Returns the hockey-stick curve that should be compared against
+        digital news growth. If the curves track together, the dataset's
+        growth reflects news digitization, not increasing floods.
+        """
+        return _trends.bias_check(self._df)
+
+    # ── Charts ──────────────────────────────────────────────────────────
+
+    def plot_hockey_stick(self, save_path: str = None):
+        """Chart 1: The 807x hockey stick — events per year."""
+        data = self.bias_check()
+        return _charts.plot_hockey_stick(data, save_path=save_path)
+
+    def plot_bias(self, save_path: str = None):
+        """Chart 2: Event growth vs internet/news growth."""
+        data = self.bias_check()
+        return _charts.plot_bias_normalized(data, save_path=save_path)
+
+    def plot_country_growth(self, n: int = 20, save_path: str = None, **kwargs):
+        """Chart 3: Countries with fastest flood event growth."""
+        data = self.country_growth_ranking(n=n, **kwargs)
+        return _charts.plot_country_growth(data, n=n, save_path=save_path)
+
+    def plot_top_countries(self, n: int = 20, save_path: str = None):
+        """Chart 4: Top N countries by total events — the LinkedIn chart."""
+        data = self.top_countries(n=n)
+        return _charts.plot_top_countries(data, n=n, save_path=save_path)

groundsource/spatial.py

@@ -0,0 +1,131 @@
+"""Spatial operations: WKB decoding, country tagging, geocoding, search."""
+
+from pathlib import Path
+from math import radians, cos, sin, asin, sqrt
+
+import numpy as np
+import pandas as pd
+import geopandas as gpd
+from shapely import wkb
+from shapely.geometry import Point
+
+
+# Path to bundled Natural Earth shapefile
+_DATA_DIR = Path(__file__).parent / "data"
+_NE_SHP = _DATA_DIR / "ne_110m_admin_0_countries.shp"
+
+
+def decode_centroids(geometry_series: pd.Series) -> pd.DataFrame:
+    """Decode WKB geometry bytes to centroid lon/lat columns.
+
+    Returns DataFrame with columns: centroid_lon, centroid_lat
+    """
+    lons = np.empty(len(geometry_series), dtype=np.float64)
+    lats = np.empty(len(geometry_series), dtype=np.float64)
+
+    for i, geom_bytes in enumerate(geometry_series):
+        geom = wkb.loads(geom_bytes)
+        c = geom.centroid
+        lons[i] = c.x
+        lats[i] = c.y
+
+    return pd.DataFrame({"centroid_lon": lons, "centroid_lat": lats})
+
+
+def tag_countries(lons: np.ndarray, lats: np.ndarray) -> pd.DataFrame:
+    """Spatial join centroids against Natural Earth to get country/continent.
+
+    Returns DataFrame with columns: country, iso_a3, continent
+    """
+    if not _NE_SHP.exists():
+        raise FileNotFoundError(
+            f"Natural Earth shapefile not found at {_NE_SHP}. "
+            "Please reinstall the groundsource package."
+        )
+
+    world = gpd.read_file(_NE_SHP)[["NAME", "ISO_A3", "CONTINENT", "geometry"]]
+
+    points = gpd.GeoDataFrame(
+        {"idx": np.arange(len(lons))},
+        geometry=gpd.points_from_xy(lons, lats),
+        crs="EPSG:4326",
+    )
+
+    joined = gpd.sjoin(points, world, how="left", predicate="within")
+
+    # Handle duplicates from sjoin (point on border matched to multiple countries)
+    joined = joined.drop_duplicates(subset="idx", keep="first")
+    joined = joined.sort_values("idx").reset_index(drop=True)
+
+    return pd.DataFrame({
+        "country": joined["NAME"].values,
+        "iso_a3": joined["ISO_A3"].values,
+        "continent": joined["CONTINENT"].values,
+    })
+
+
+def enrich_dataframe(df: pd.DataFrame) -> pd.DataFrame:
+    """Add centroid_lon, centroid_lat, country, iso_a3, continent to DataFrame."""
+    print("  Decoding WKB geometries (2.6M polygons)...")
+    centroids = decode_centroids(df["geometry"])
+
+    print("  Tagging countries via spatial join...")
+    countries = tag_countries(centroids["centroid_lon"].values, centroids["centroid_lat"].values)
+
+    result = df.copy()
+    result["centroid_lon"] = centroids["centroid_lon"].values
+    result["centroid_lat"] = centroids["centroid_lat"].values
+    result["country"] = countries["country"].values
+    result["iso_a3"] = countries["iso_a3"].values
+    result["continent"] = countries["continent"].values
+
+    # Parse dates
+    result["start_date"] = pd.to_datetime(result["start_date"])
+    result["end_date"] = pd.to_datetime(result["end_date"])
+    result["year"] = result["start_date"].dt.year
+
+    # Drop raw geometry bytes (no longer needed after enrichment)
+    result = result.drop(columns=["geometry"])
+
+    untagged = result["country"].isna().sum()
+    pct = untagged / len(result) * 100
+    print(f"  Enrichment complete. {untagged:,} events ({pct:.1f}%) "
+          f"fell outside country boundaries (ocean/border).")
+
+    return result
+
+
+def haversine_km(lon1, lat1, lon2, lat2):
+    """Haversine distance in km between two points."""
+    lon1, lat1, lon2, lat2 = map(radians, [lon1, lat1, lon2, lat2])
+    dlon = lon2 - lon1
+    dlat = lat2 - lat1
+    a = sin(dlat / 2) ** 2 + cos(lat1) * cos(lat2) * sin(dlon / 2) ** 2
+    return 6371 * 2 * asin(sqrt(a))
+
+
+def search_by_radius(df: pd.DataFrame, lon: float, lat: float, radius_km: float) -> pd.DataFrame:
+    """Filter events within radius_km of a point using vectorized haversine."""
+    lon1 = np.radians(df["centroid_lon"].values)
+    lat1 = np.radians(df["centroid_lat"].values)
+    lon2 = radians(lon)
+    lat2 = radians(lat)
+
+    dlat = lat2 - lat1
+    dlon = lon2 - lon1
+    a = np.sin(dlat / 2) ** 2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon / 2) ** 2
+    dist_km = 6371 * 2 * np.arcsin(np.sqrt(a))
+
+    return df[dist_km <= radius_km].copy()
+
+
+def search_by_bbox(df: pd.DataFrame, bbox: list) -> pd.DataFrame:
+    """Filter events within bounding box [min_lat, min_lon, max_lat, max_lon]."""
+    min_lat, min_lon, max_lat, max_lon = bbox
+    mask = (
+        (df["centroid_lat"] >= min_lat)
+        & (df["centroid_lat"] <= max_lat)
+        & (df["centroid_lon"] >= min_lon)
+        & (df["centroid_lon"] <= max_lon)
+    )
+    return df[mask].copy()

groundsource/trends.py

@@ -0,0 +1,113 @@
+"""Trend analysis and bias detection for flood event data."""
+
+import pandas as pd
+import numpy as np
+
+
+def yearly_counts(df: pd.DataFrame, group_col: str = None, group_val: str = None) -> pd.DataFrame:
+    """Count events per year, optionally filtered by a group column.
+
+    Returns DataFrame with columns: year, count
+    """
+    subset = df
+    if group_col and group_val:
+        subset = df[df[group_col] == group_val]
+
+    counts = subset.groupby("year").size().reset_index(name="count")
+    return counts
+
+
+def growth_rate(counts: pd.DataFrame, early=(2000, 2012), recent=(2015, 2025)) -> dict:
+    """Compare average yearly counts between two periods.
+
+    Returns dict with early_avg, recent_avg, growth_factor.
+    """
+    early_avg = counts[(counts["year"] >= early[0]) & (counts["year"] <= early[1])]["count"].mean()
+    recent_avg = counts[(counts["year"] >= recent[0]) & (counts["year"] <= recent[1])]["count"].mean()
+
+    if early_avg == 0 or pd.isna(early_avg):
+        factor = float("inf") if recent_avg > 0 else 0
+    else:
+        factor = recent_avg / early_avg
+
+    return {
+        "early_period": early,
+        "recent_period": recent,
+        "early_avg": float(round(early_avg, 1)) if not pd.isna(early_avg) else 0.0,
+        "recent_avg": float(round(recent_avg, 1)) if not pd.isna(recent_avg) else 0.0,
+        "growth_factor": float(round(factor, 1)),
+    }
+
+
+def top_countries(df: pd.DataFrame, n: int = 20) -> pd.DataFrame:
+    """Top N countries by total event count."""
+    counts = df.groupby("country").size().reset_index(name="count")
+    return counts.sort_values("count", ascending=False).head(n).reset_index(drop=True)
+
+
+def country_growth_ranking(df: pd.DataFrame, n: int = 20,
+                           early=(2005, 2012), recent=(2018, 2025),
+                           min_early_events: int = 10) -> pd.DataFrame:
+    """Rank countries by flood event growth rate between two periods.
+
+    Filters to countries with at least min_early_events in the early period
+    to avoid divide-by-zero / noise from tiny samples.
+    """
+    early_counts = (
+        df[(df["year"] >= early[0]) & (df["year"] <= early[1])]
+        .groupby("country").size().reset_index(name="early_count")
+    )
+    recent_counts = (
+        df[(df["year"] >= recent[0]) & (df["year"] <= recent[1])]
+        .groupby("country").size().reset_index(name="recent_count")
+    )
+
+    merged = early_counts.merge(recent_counts, on="country", how="inner")
+    merged = merged[merged["early_count"] >= min_early_events]
+
+    early_years = early[1] - early[0] + 1
+    recent_years = recent[1] - recent[0] + 1
+    merged["early_avg"] = merged["early_count"] / early_years
+    merged["recent_avg"] = merged["recent_count"] / recent_years
+    merged["growth_factor"] = merged["recent_avg"] / merged["early_avg"]
+
+    return merged.sort_values("growth_factor", ascending=False).head(n).reset_index(drop=True)
+
+
+def bias_check(df: pd.DataFrame) -> pd.DataFrame:
+    """Return global yearly event counts for bias analysis.
+
+    The returned DataFrame has columns: year, count.
+    Users should plot this and compare against digital news growth curves.
+    A hockey-stick shape matching internet/news growth indicates
+    reporting bias, not necessarily increasing flood frequency.
+    """
+    counts = df.groupby("year").size().reset_index(name="count")
+
+    # Add some context columns
+    base_year = counts.loc[counts["year"].idxmin(), "count"]
+    counts["cumulative_growth"] = counts["count"] / base_year if base_year > 0 else 0
+
+    return counts
+
+
+def compare_countries(df: pd.DataFrame, countries: list) -> pd.DataFrame:
+    """Side-by-side yearly counts for multiple countries.
+
+    Returns DataFrame with columns: year, country1, country2, ...
+    """
+    result = None
+    for country in countries:
+        counts = yearly_counts(df, group_col="country", group_val=country)
+        counts = counts.rename(columns={"count": country})
+        if result is None:
+            result = counts
+        else:
+            result = result.merge(counts, on="year", how="outer")
+
+    if result is not None:
+        result = result.sort_values("year").fillna(0)
+        for col in countries:
+            result[col] = result[col].astype(int)
+
+    return result

groundsource-0.1.0.dist-info/METADATA

@@ -0,0 +1,150 @@
+Metadata-Version: 2.4
+Name: groundsource
+Version: 0.1.0
+Summary: Python package for Google's Groundsource flash flood dataset — 2.6M events, 150+ countries, 2000–2026
+Author: Shara
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/sharanry/groundsource
+Project-URL: Repository, https://github.com/sharanry/groundsource
+Project-URL: Issues, https://github.com/sharanry/groundsource/issues
+Keywords: flood,flash-flood,climate,groundsource,google,geospatial,dataset,gemini,natural-disaster,news-mining
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: GIS
+Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pandas>=1.5
+Requires-Dist: pyarrow>=10.0
+Requires-Dist: geopandas>=0.13
+Requires-Dist: shapely>=2.0
+Requires-Dist: matplotlib>=3.6
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Dynamic: license-file
+
+# groundsource
+
+**Python package for Google's Groundsource flash flood dataset.**
+
+Google used Gemini to extract 2.6 million flash flood events from news articles across 150+ countries (2000-2026). The raw data is a 667MB Parquet file with undocumented WKB geometries and no location labels. This package decodes the geometries, tags every event with country and continent, and provides a clean search and analysis API.
+
+```python
+from groundsource import FloodDB
+
+db = FloodDB()  # auto-downloads + enriches on first run
+floods = db.search(country="India", year_range=(2020, 2025))
+```
+
+## Installation
+
+```bash
+pip install groundsource
+```
+
+**Requirements:** Python 3.9+, pandas, pyarrow, geopandas, shapely, matplotlib
+
+On first run, the package downloads the dataset from Zenodo (~667MB), decodes 2.6M WKB polygons, and performs a spatial join against Natural Earth boundaries. This takes 2-3 minutes and is cached locally for instant subsequent loads.
+
+## Usage
+
+### Search
+
+```python
+from groundsource import FloodDB
+db = FloodDB()
+
+# By country (supports common aliases: "USA", "UK", "UAE", etc.)
+db.search(country="India")
+db.search(country="USA", year_range=(2020, 2025))
+
+# By city (98 major cities built-in, default 100km radius)
+db.search(city="Houston", radius_km=50)
+
+# By continent or bounding box
+db.search(continent="Asia")
+db.search(bbox=[0, 95, 25, 120])  # [min_lat, min_lon, max_lat, max_lon]
+```
+
+### Trend Analysis
+
+```python
+db.trend(country="India")                        # yearly event counts
+db.growth(country="India")                       # growth rate between two periods
+db.compare(["USA", "UK", "India", "Indonesia"])  # side-by-side comparison
+db.top_countries(20)                             # ranked by total events
+db.country_growth_ranking(20)                    # ranked by growth acceleration
+db.bias_check()                                  # global yearly counts for bias analysis
+```
+
+### Built-in Charts
+
+```python
+db.plot_hockey_stick(save_path="hockey_stick.png")
+db.plot_bias(save_path="bias.png")
+db.plot_top_countries(save_path="top_countries.png")
+db.plot_country_growth(save_path="growth.png")
+```
+
+### Raw DataFrame Access
+
+```python
+df = db.to_dataframe()
+# Columns: uuid, area_km2, start_date, end_date, centroid_lon, centroid_lat,
+#           country, iso_a3, continent, year
+```
+
+## What This Package Does
+
+The raw Parquet from Zenodo has 5 columns with no documentation:
+
+| Raw Column | Type | Issue |
+|-----------|------|-------|
+| `uuid` | string | ID only |
+| `area_km2` | float | Usable as-is |
+| `geometry` | WKB binary | Requires `shapely` to decode |
+| `start_date` | string | Not parsed as datetime |
+| `end_date` | string | Not parsed as datetime |
+
+This package enriches each event with:
+
+| Added Column | Source |
+|-------------|--------|
+| `centroid_lon`, `centroid_lat` | Decoded from WKB polygons |
+| `country`, `iso_a3` | Spatial join against Natural Earth |
+| `continent` | Natural Earth |
+| `year` | Extracted from `start_date` |
+
+## Reporting Bias
+
+The dataset shows 498 events in 2000 and 402,012 in 2024. This does not mean floods increased 807x. The data is extracted from news articles, and digital news coverage grew dramatically over this period. Any trend analysis should account for this reporting bias. Use `db.bias_check()` and `db.plot_bias()` to visualize this.
+
+![Bias Analysis](charts/02_bias_normalized.png)
+
+## Top Countries by Events Detected
+
+![Top Countries](charts/04_top_countries.png)
+
+## Dataset
+
+- **Source:** [Google Groundsource](https://research.google/blog/introducing-groundsource-turning-news-reports-into-data-with-gemini/)
+- **Download:** [Zenodo](https://zenodo.org/records/18647054) (CC BY 4.0)
+- **Records:** 2,646,302 events across 175 countries, 2000-2026
+- **Method:** Gemini parsed ~5M news articles
+- **Accuracy:** 60% location+timing, 82% practically useful (per Google)
+
+## License
+
+MIT. The underlying dataset is licensed CC BY 4.0 by Google.
+
+## Citation
+
+> Google Research. *Groundsource: Turning News Reports into Data with Gemini.* Zenodo, 2026. DOI: [10.5281/zenodo.18647054](https://zenodo.org/records/18647054)

groundsource-0.1.0.dist-info/RECORD

@@ -0,0 +1,17 @@
+groundsource/__init__.py,sha256=inQIRcjV65ACtxIrYFrP5o8Mi5YpuV2ce6e96i983_c,167
+groundsource/cache.py,sha256=tbvKJSHBGrERVri92WZYGyVczya-i2vqBWwQF2C05LQ,2082
+groundsource/charts.py,sha256=0g8KhGwi7YnQHSVvZvJxxcpWWT-P_DvlwZDsFJjOcPg,8084
+groundsource/db.py,sha256=8QWs4Ld-hzGJpALWhCG-BTK_XNfHHVlI3qve09GLuec,13644
+groundsource/spatial.py,sha256=QfjCUi2ANavv2hyoEhX2TQAL9tckyiVAoCuDgi5Whr4,4591
+groundsource/trends.py,sha256=8olS2F0epwZE1KWDLyb7MUdlNgeXvsso6yz2jkqjGDY,4260
+groundsource/data/__init__.py,sha256=fMw6NvHh0R15QoK3Nvsn6UxML3TEmfqc38TWzQDpED4,81
+groundsource/data/ne_110m_admin_0_countries.cpg,sha256=OtMDH1UDpEBK-CUmLugjLMBNTqZoPULF3QovKiesmCQ,5
+groundsource/data/ne_110m_admin_0_countries.dbf,sha256=H-5nfNTgOzZ4duA4YesQGX5AIqhGv5IGDgMTQyhjeFs,531808
+groundsource/data/ne_110m_admin_0_countries.prj,sha256=Mlnw5VKQqCsTUGRvYE6KfuHiE2wDIKQPrYOKtAgZ__g,147
+groundsource/data/ne_110m_admin_0_countries.shp,sha256=CONBYG6DkeRYw_CN6zEt5mS1a_rjdgZMWqCu5mgaX1U,180924
+groundsource/data/ne_110m_admin_0_countries.shx,sha256=iwvirZfdSEruXC68mGl9U3LoMriuWKNaZhru9rmFZo0,1516
+groundsource-0.1.0.dist-info/licenses/LICENSE,sha256=J_TEk_POk6Le3dSCQUYuAbyVwpqGP-4RTjT6cQMF16k,1078
+groundsource-0.1.0.dist-info/METADATA,sha256=bYwQCMtjuaE9q17dvIeqRZ3ACZCyXW7NSRkkibUP5C4,5680
+groundsource-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+groundsource-0.1.0.dist-info/top_level.txt,sha256=x8jiZoCtnyZzmlaeNj9l_KeNusbMFG6ZFYTDcPoH1Bk,13
+groundsource-0.1.0.dist-info/RECORD,,

groundsource-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

groundsource-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Sharath Sivamalaisamy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

groundsource-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+groundsource