utdquake 0.2.2__tar.gz → 0.2.3__tar.gz

{utdquake-0.2.2/utdquake.egg-info → utdquake-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: utdquake
-Version: 0.2.2
+Version: 0.2.3
 Summary: University of Texas at Dallas Earthquake Dataset
 Home-page: https://github.com/ecastillot/UTDQuake
 Author: ecastillot (Emmanuel Castillo)

{utdquake-0.2.2 → utdquake-0.2.3}/utdquake/__init__.py

@@ -9,6 +9,10 @@ includes:
 - `Network`: Class for network-specific access, including EventBank, picks, and stations.
 - `download_snapshot`: Function to download UTDQuake data from Hugging Face.
 - `load`: Function to load datasets from Hugging Face by key.
+- `publish_network`: Function to publish a network's manifests to the shared Hub repo.
+- `remove_network`: Function to remove a network's files from the shared Hub repo.
+- `generate_network_figures`: Function to generate a network's documentation figures.
+- `publish_network_figures`: Function to publish those figures to GitHub.
 
 Usage
 -----
@@ -20,11 +24,17 @@ Usage
 >>> net.events.head()
 
 """
-__version__ = "0.2.2"
+__version__ = "0.2.3"
 
 from .core.utdquake import Dataset, Network
 from .core.data import download_snapshot, load
 from .core.patch import utdquake_obspy_patch
+from .hub import publish_network, remove_network
+from .figures import generate_network_figures, publish_network_figures
 
 utdquake_obspy_patch()
-__all__ = ["Dataset", "Network", "download_snapshot", "load"]
+__all__ = [
+    "Dataset", "Network", "download_snapshot", "load",
+    "publish_network", "remove_network",
+    "generate_network_figures", "publish_network_figures",
+]

{utdquake-0.2.2 → utdquake-0.2.3}/utdquake/bank/bank.py

@@ -73,6 +73,7 @@ class UTDQBank(obsplus.EventBank):
         self.das = das
         self.contributor = os.path.basename(self.bank_path)
         self.db_paths = self.__prepare_paths()
+        self.utdq_paths = self.db_paths
 
     def __prepare_paths(self) -> None:
         """
@@ -297,7 +298,7 @@ class UTDQBank(obsplus.EventBank):
             raise FileNotFoundError(f"Folder not found: {folder_path}")
 
         # Build full glob pattern (e.g., /path/to/folder/*.xml)
-        pattern = os.path.join(folder_path, file_extension)
+        pattern = os.path.join(folder_path, "**", file_extension)
 
         # Retrieve all matching files
         files: List[str] = glob.glob(pattern, recursive=True)
@@ -373,9 +374,9 @@ class UTDQBank(obsplus.EventBank):
         ebank_index_path = os.path.join(self.bank_path, ".index.db")
 
         if starttime is None:
-            starttime = self.read_index()["time"].min()
+            starttime = self.read_index()["time"].min() - pd.Timedelta(seconds=1)
         if endtime is None:
-            endtime = self.read_index()["time"].max()
+            endtime = self.read_index()["time"].max() + pd.Timedelta(seconds=1)
 
         iteration = 0
         for catalog_dict  in ut.catalog_generator(self,starttime=starttime,
@@ -536,7 +537,7 @@ class UTDQBank(obsplus.EventBank):
                             i,
                             total_chunks,
                         )
-                        catalog.apply_utdq_qc()
+                        catalog = catalog.apply_utdq_qc()
 
                     events_df = catalog.utdq_events_to_df()
                     picks_df = catalog.utdq_picks_to_df()
@@ -628,7 +629,7 @@ class UTDQBank(obsplus.EventBank):
                     )
                 """
 
-        df = ut._read_table(self.index_path, query)
+        df = ut._read_table(self.utdq_paths[".utdquake/export/db/stations"], query)
 
         df = sanitize_dataframe(df,
                                 order_cols=PREF_STATIONS_ORDER,

{utdquake-0.2.2 → utdquake-0.2.3}/utdquake/bank/utils.py

@@ -1228,11 +1228,14 @@ def get_stations_summary(stations_folder,
         # --- Collapse into one row ---
         summary = {}
         exclude_cols = {"dist_deg", "esaz", "used", "origin_id", "creation_time"}
+        std_cols = {"calculated_latitude", "calculated_longitude"}
         for col in df.columns:
             if col in exclude_cols:
                 continue
             if pd.api.types.is_float_dtype(df[col]):
                 summary[col] = df[col].mean()
+                if col in std_cols:
+                    summary[f"{col}_std"] = df[col].std()
             elif pd.api.types.is_bool_dtype(df[col]):
                 summary[col] = df[col].any()   # True if any True
             elif pd.api.types.is_integer_dtype(df[col]):
@@ -1242,6 +1245,12 @@ def get_stations_summary(stations_folder,
             else:
                 summary[col] = df[col].iloc[0]  # fallback
 
+        # Number of arrivals that contributed to the calculated position
+        # (one row per origin at this point, since drop_duplicates above
+        # collapses to one entry per origin_id).
+        if "calculated_latitude" in df.columns:
+            summary["calculated_num_entries"] = int(df["calculated_latitude"].notna().sum())
+
         summary["db_path"] = db_path
         summary["creation_time"] = UTCDateTime.now().strftime("%Y-%m-%d %H:%M:%S.%f")
         return summary
@@ -1294,15 +1303,15 @@ def process_arrival(arrival, source_loc,
             "network_type": None,
             "station": None,
             "channel": None,
-            "dist_deg": np.NaN,
-            "esaz": np.NaN,
+            "dist_deg": np.nan,
+            "esaz": np.nan,
             "confirmed": False,
-            "confirmed_latitude": np.NaN,
-            "confirmed_longitude": np.NaN,
-            "confirmed_elevation": np.NaN,
+            "confirmed_latitude": np.nan,
+            "confirmed_longitude": np.nan,
+            "confirmed_elevation": np.nan,
             "calculated": False,
-            "calculated_latitude": np.NaN,
-            "calculated_longitude": np.NaN,
+            "calculated_latitude": np.nan,
+            "calculated_longitude": np.nan,
             "used": False,
             "origin_id": origin_id,
             "creation_time": UTCDateTime.now().strftime("%Y-%m-%dT%H:%M:%S.%fZ")
@@ -1395,7 +1404,7 @@ def process_arrival(arrival, source_loc,
     #finding the corresponding stations folder for the network
     network_name = os.path.basename(os.path.dirname(ebank_index_path))
     utdq_paths = get_utdq_paths(network_name)
-    stations_folder = utdq_paths[".utdquake/export/db/stations/.stations"]
+    stations_folder = utdq_paths[".utdquake/export/db"] / "stations" / f".{network_name}"
 
     # stations_folder = os.path.join( stations_folder,".stations")
 
@@ -1586,8 +1595,8 @@ def xxx(origin, df_stations,
                 missing_info ={
                     "network": net,
                     "station": sta,
-                    "latitude": np.NaN,
-                    "longitude": np.NaN,
+                    "latitude": np.nan,
+                    "longitude": np.nan,
                     "status": "missing",
                     "creation_time": UTCDateTime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}
                 missing_info = pd.DataFrame([missing_info])
@@ -1673,8 +1682,8 @@ def xxx(origin, df_stations,
                 missing_info = {
                         "network": net,
                         "station": sta,
-                        "latitude": np.NaN,
-                        "longitude": np.NaN,
+                        "latitude": np.nan,
+                        "longitude": np.nan,
                         "status": "missing",
                         "creation_time": UTCDateTime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}
                 missing_info = pd.DataFrame([missing_info])

utdquake-0.2.3/utdquake/figures.py

@@ -0,0 +1,285 @@
+"""
+Generate and publish the standard per-network figure set used in the
+UTDQuake documentation (:ref:`overview-section`).
+
+Two independent halves:
+
+- ``generate_network_figures``: purely local, no credentials. Runs
+  :class:`utdquake.Network`'s own ``plot_*`` methods and saves the results
+  with the ``{network}_{suffix}.png`` naming already used on the ``figures``
+  branch of https://github.com/ecastillot/UTDQuake.
+- ``publish_network_figures``: uploads to GitHub. Needs a token
+  (``GITHUB_TOKEN`` env var or ``token=``) with ``repo`` scope.
+- ``overview_rst_row``: read-only, no token needed -- the repo is public.
+
+Unlike the Hugging Face Hub, GitHub does not let a non-collaborator open a
+PR without forking first. If the token does not have write access to
+ecastillot/UTDQuake, ``publish_network_figures`` forks the repo under the
+token's account, pushes the figures to that fork's ``figures`` branch, and
+opens a PR back to ecastillot/UTDQuake. If the token does have write access
+(a maintainer's), it commits directly instead.
+"""
+
+from __future__ import annotations
+
+import base64
+import logging
+import time
+from pathlib import Path
+from typing import Dict, Optional
+
+import requests
+
+from .core.utdquake import Network
+
+logger = logging.getLogger(__name__)
+
+GITHUB_OWNER = "ecastillot"
+GITHUB_REPO = "UTDQuake"
+GITHUB_BRANCH = "figures"
+GITHUB_TOKEN_ENV_VAR = "GITHUB_TOKEN"
+GITHUB_API = "https://api.github.com"
+
+# (suffix, Network method name, kwargs for that method, accepts a show= kwarg)
+FIGURE_SPECS = [
+    ("overview", "plot_overview", {}, True),
+    ("stats", "plot_stats", {}, True),
+    ("pick_histograms", "plot_pick_histograms", {}, True),
+    ("phase_count_radar", "plot_phase_count_radar_by_magnitude", {}, True),
+    ("station_location_uncertainty", "plot_station_location_uncertainty", {}, True),
+    ("uncertainty_boxplots", "plot_uncertainty_boxplots", {}, True),
+    ("travel_time_qc", "plot_travel_time_qc", {}, False),
+    ("travel_time_vs_distance", "plot_travel_time_vs_distance", {}, True),
+    ("travel_time_vs_distance_P_zscore", "plot_travel_time_vs_distance_zscore", {"phase": "P"}, True),
+]
+
+
+def generate_network_figures(
+    network: str,
+    outdir: Optional[Path] = None,
+    das: bool = False,
+) -> Dict[str, Path]:
+    """
+    Generate the standard figure set for a network.
+
+    Each figure is generated independently -- one missing a prerequisite
+    (e.g. no travel-time model yet) is logged and skipped rather than
+    failing the whole batch.
+
+    Parameters
+    ----------
+    network : str
+        Network code, e.g. "CM".
+    outdir : Path, optional
+        Directory to save into. Defaults to ``./figures/{network}``.
+    das : bool, optional
+        Whether this is a DAS network. Default False.
+
+    Returns
+    -------
+    dict[str, Path]
+        ``{suffix: saved_path}`` for every figure that succeeded.
+    """
+    net = Network(network, das=das)
+    outdir = Path(outdir) if outdir is not None else Path.cwd() / "figures" / network
+    outdir.mkdir(parents=True, exist_ok=True)
+
+    saved: Dict[str, Path] = {}
+    for suffix, method_name, kwargs, accepts_show in FIGURE_SPECS:
+        savepath = outdir / f"{network}_{suffix}.png"
+        call_kwargs = dict(savepath=str(savepath), **kwargs)
+        if accepts_show:
+            call_kwargs["show"] = False
+        try:
+            getattr(net, method_name)(**call_kwargs)
+            saved[suffix] = savepath
+            logger.info("Generated %s", savepath)
+        except Exception as e:
+            logger.warning("Skipping %s for network %s: %s", suffix, network, e)
+    return saved
+
+
+def _resolve_token(token: Optional[str] = None) -> str:
+    import os
+    token = token or os.environ.get(GITHUB_TOKEN_ENV_VAR)
+    if not token:
+        raise ValueError(
+            f"No GitHub token provided. Set the {GITHUB_TOKEN_ENV_VAR} "
+            "environment variable or pass token=... explicitly."
+        )
+    return token
+
+
+def _gh(method: str, url: str, token: str, **kwargs) -> requests.Response:
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Accept": "application/vnd.github+json",
+        "X-GitHub-Api-Version": "2022-11-28",
+    }
+    resp = requests.request(method, url, headers=headers, timeout=30, **kwargs)
+    if resp.status_code >= 400:
+        raise RuntimeError(f"GitHub API {method} {url} -> {resp.status_code}: {resp.text[:500]}")
+    return resp
+
+
+def _authenticated_user(token: str) -> str:
+    return _gh("GET", f"{GITHUB_API}/user", token).json()["login"]
+
+
+def _has_write_access(token: str, username: str) -> bool:
+    resp = requests.get(
+        f"{GITHUB_API}/repos/{GITHUB_OWNER}/{GITHUB_REPO}/collaborators/{username}/permission",
+        headers={"Authorization": f"Bearer {token}", "Accept": "application/vnd.github+json"},
+        timeout=30,
+    )
+    if resp.status_code != 200:
+        return False
+    return resp.json().get("permission") in ("write", "admin")
+
+
+def _ensure_fork(token: str, username: str) -> None:
+    resp = requests.get(f"{GITHUB_API}/repos/{username}/{GITHUB_REPO}", headers={
+        "Authorization": f"Bearer {token}", "Accept": "application/vnd.github+json"
+    }, timeout=30)
+    if resp.status_code == 200:
+        return
+    _gh("POST", f"{GITHUB_API}/repos/{GITHUB_OWNER}/{GITHUB_REPO}/forks", token)
+    # Forking is async -- poll until the fork is queryable.
+    for _ in range(20):
+        time.sleep(1.5)
+        resp = requests.get(f"{GITHUB_API}/repos/{username}/{GITHUB_REPO}", headers={
+            "Authorization": f"Bearer {token}", "Accept": "application/vnd.github+json"
+        }, timeout=30)
+        if resp.status_code == 200:
+            return
+    raise RuntimeError(f"Timed out waiting for fork {username}/{GITHUB_REPO} to become available.")
+
+
+def _put_file(owner: str, token: str, path_in_repo: str, local_path: Path, branch: str, message: str) -> None:
+    url = f"{GITHUB_API}/repos/{owner}/{GITHUB_REPO}/contents/{path_in_repo}"
+    existing = requests.get(url, params={"ref": branch}, headers={
+        "Authorization": f"Bearer {token}", "Accept": "application/vnd.github+json"
+    }, timeout=30)
+    sha = existing.json().get("sha") if existing.status_code == 200 else None
+
+    content_b64 = base64.b64encode(local_path.read_bytes()).decode("ascii")
+    body = {"message": message, "content": content_b64, "branch": branch}
+    if sha:
+        body["sha"] = sha
+    _gh("PUT", url, token, json=body)
+
+
+def publish_network_figures(
+    network: str,
+    local_dir: Optional[Path] = None,
+    token: Optional[str] = None,
+    create_pr: bool = True,
+    commit_message: Optional[str] = None,
+) -> str:
+    """
+    Publish a network's already-generated figures (see
+    :func:`generate_network_figures`) to the ``figures`` branch of
+    ecastillot/UTDQuake on GitHub.
+
+    Parameters
+    ----------
+    network : str
+        Network code, e.g. "CM".
+    local_dir : Path, optional
+        Directory holding the generated PNGs. Defaults to
+        ``./figures/{network}`` (matching ``generate_network_figures``'s
+        default).
+    token : str, optional
+        GitHub personal access token (``repo`` scope). Defaults to the
+        ``GITHUB_TOKEN`` environment variable.
+    create_pr : bool, optional
+        If True (default) and the token does *not* have write access to
+        ecastillot/UTDQuake, fork the repo under the token's account, push
+        the figures there, and open a PR back to ecastillot/UTDQuake -- no
+        write access to the upstream repo required. If the token *does*
+        have write access, this is ignored and the figures are committed
+        directly to ecastillot/UTDQuake's ``figures`` branch regardless of
+        this flag -- there's no reason to route a maintainer's own push
+        through a fork.
+    commit_message : str, optional
+        Commit message. Defaults to a generated message.
+
+    Returns
+    -------
+    str
+        URL of the resulting commit (or PR, if ``create_pr=True``).
+    """
+    token = _resolve_token(token)
+    local_dir = Path(local_dir) if local_dir is not None else Path.cwd() / "figures" / network
+    pngs = sorted(local_dir.glob(f"{network}_*.png"))
+    if not pngs:
+        raise FileNotFoundError(f"No {network}_*.png files in {local_dir}. Run generate_network_figures first.")
+
+    username = _authenticated_user(token)
+    message = commit_message or f"Add figures for network {network}"
+
+    if create_pr and not _has_write_access(token, username):
+        _ensure_fork(token, username)
+        target_owner = username
+    else:
+        target_owner = GITHUB_OWNER
+        create_pr = False  # already a maintainer -- commit directly
+
+    for png in pngs:
+        _put_file(
+            owner=target_owner,
+            token=token,
+            path_in_repo=f"figures/networks/{network}/{png.name}",
+            local_path=png,
+            branch=GITHUB_BRANCH,
+            message=message,
+        )
+        logger.info("Uploaded %s to %s/%s@%s", png.name, target_owner, GITHUB_REPO, GITHUB_BRANCH)
+
+    if not create_pr:
+        return f"https://github.com/{GITHUB_OWNER}/{GITHUB_REPO}/tree/{GITHUB_BRANCH}/figures/networks/{network}"
+
+    pr = _gh(
+        "POST",
+        f"{GITHUB_API}/repos/{GITHUB_OWNER}/{GITHUB_REPO}/pulls",
+        token,
+        json={
+            "title": message,
+            "head": f"{username}:{GITHUB_BRANCH}",
+            "base": GITHUB_BRANCH,
+            "body": f"Adds the figure set for network `{network}` (see :ref:`upload-dataset-section`).",
+        },
+    ).json()
+    logger.info("Opened PR: %s", pr["html_url"])
+    return pr["html_url"]
+
+
+def overview_rst_row(network: str, das: bool = False) -> str:
+    """
+    Return the ``overview.rst`` list-table row for a network, ready to
+    paste in: the real image row if figures exist on the ``figures``
+    branch, or a "pending" placeholder row if not.
+
+    This makes a single unauthenticated, read-only request -- no token
+    needed, since the repo is public.
+    """
+    raw_url = (
+        f"https://raw.githubusercontent.com/{GITHUB_OWNER}/{GITHUB_REPO}/"
+        f"{GITHUB_BRANCH}/figures/networks/{network}/{network}_overview.png"
+    )
+    folder_url = f"https://github.com/{GITHUB_OWNER}/{GITHUB_REPO}/tree/{GITHUB_BRANCH}/figures/networks/{network}"
+
+    exists = requests.head(raw_url, timeout=15).status_code == 200
+
+    if exists:
+        return (
+            f"   * - {network}\n"
+            f"     - `Open Folder <{folder_url}>`_\n"
+            f"     - .. image:: {raw_url}\n"
+            f"          :width: 200px\n"
+        )
+    return (
+        f"   * - {network}\n"
+        f"     - *pending*\n"
+        f"     - *Figures pending -- see* :ref:`upload-dataset-section`\n"
+    )