utdquake 0.2.2__tar.gz → 0.3.0__tar.gz

{utdquake-0.2.2/utdquake.egg-info → utdquake-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: utdquake
-Version: 0.2.2
+Version: 0.3.0
 Summary: University of Texas at Dallas Earthquake Dataset
 Home-page: https://github.com/ecastillot/UTDQuake
 Author: ecastillot (Emmanuel Castillo)

{utdquake-0.2.2 → utdquake-0.3.0}/utdquake/__init__.py

@@ -9,6 +9,10 @@ includes:
 - `Network`: Class for network-specific access, including EventBank, picks, and stations.
 - `download_snapshot`: Function to download UTDQuake data from Hugging Face.
 - `load`: Function to load datasets from Hugging Face by key.
+- `publish_network`: Function to publish a network's manifests to the shared Hub repo.
+- `remove_network`: Function to remove a network's files from the shared Hub repo.
+- `generate_network_figures`: Function to generate a network's documentation figures.
+- `publish_network_figures`: Function to publish those figures to GitHub.
 
 Usage
 -----
@@ -20,11 +24,17 @@ Usage
 >>> net.events.head()
 
 """
-__version__ = "0.2.2"
+__version__ = "0.3.0"
 
 from .core.utdquake import Dataset, Network
 from .core.data import download_snapshot, load
 from .core.patch import utdquake_obspy_patch
+from .hub import publish_network, remove_network
+from .figures import generate_network_figures, publish_network_figures
 
 utdquake_obspy_patch()
-__all__ = ["Dataset", "Network", "download_snapshot", "load"]
+__all__ = [
+    "Dataset", "Network", "download_snapshot", "load",
+    "publish_network", "remove_network",
+    "generate_network_figures", "publish_network_figures",
+]

{utdquake-0.2.2 → utdquake-0.3.0}/utdquake/bank/bank.py

@@ -73,6 +73,7 @@ class UTDQBank(obsplus.EventBank):
         self.das = das
         self.contributor = os.path.basename(self.bank_path)
         self.db_paths = self.__prepare_paths()
+        self.utdq_paths = self.db_paths
 
     def __prepare_paths(self) -> None:
         """
@@ -297,7 +298,7 @@ class UTDQBank(obsplus.EventBank):
             raise FileNotFoundError(f"Folder not found: {folder_path}")
 
         # Build full glob pattern (e.g., /path/to/folder/*.xml)
-        pattern = os.path.join(folder_path, file_extension)
+        pattern = os.path.join(folder_path, "**", file_extension)
 
         # Retrieve all matching files
         files: List[str] = glob.glob(pattern, recursive=True)
@@ -343,7 +344,8 @@ class UTDQBank(obsplus.EventBank):
     def put_utdq_stations(self, stations, starttime=None, endtime=None,
                         chunk_seconds=86400,
                         das=False,
-                        calculate_d_az=True) -> None:
+                        calculate_d_az=True,
+                        station_sample_size=None) -> None:
         """
         Save station metadata to events in the event bank over a given time range.
 
@@ -367,15 +369,21 @@ class UTDQBank(obsplus.EventBank):
         calculate_d_az : bool, optional
             Whether to calculate distance and azimuth between
             events and stations (default is True).
+        station_sample_size : int, optional
+            At most this many station records (one per arrival) per station
+            database, spread over the events: enough for the station-location
+            summary (calculated position and its spread). None (default)
+            keeps one per arrival. Distances and azimuths of all arrivals
+            are computed either way.
 
         """
         
         ebank_index_path = os.path.join(self.bank_path, ".index.db")
 
         if starttime is None:
-            starttime = self.read_index()["time"].min()
+            starttime = self.read_index()["time"].min() - pd.Timedelta(seconds=1)
         if endtime is None:
-            endtime = self.read_index()["time"].max()
+            endtime = self.read_index()["time"].max() + pd.Timedelta(seconds=1)
 
         iteration = 0
         for catalog_dict  in ut.catalog_generator(self,starttime=starttime,
@@ -397,7 +405,8 @@ class UTDQBank(obsplus.EventBank):
                                                     df_stations=stations,
                                                     calculate_d_az=calculate_d_az,
                                                     ebank_index_path=ebank_index_path,
-                                                    das=das
+                                                    das=das,
+                                                    station_sample_size=station_sample_size,
                                                                 )
 
             new_len_catalog = len(catalog)
@@ -536,7 +545,7 @@ class UTDQBank(obsplus.EventBank):
                             i,
                             total_chunks,
                         )
-                        catalog.apply_utdq_qc()
+                        catalog = catalog.apply_utdq_qc()
 
                     events_df = catalog.utdq_events_to_df()
                     picks_df = catalog.utdq_picks_to_df()
@@ -628,7 +637,7 @@ class UTDQBank(obsplus.EventBank):
                     )
                 """
 
-        df = ut._read_table(self.index_path, query)
+        df = ut._read_table(self.utdq_paths[".utdquake/export/db/stations"], query)
 
         df = sanitize_dataframe(df,
                                 order_cols=PREF_STATIONS_ORDER,

{utdquake-0.2.2 → utdquake-0.3.0}/utdquake/bank/utils.py

@@ -1228,11 +1228,14 @@ def get_stations_summary(stations_folder,
         # --- Collapse into one row ---
         summary = {}
         exclude_cols = {"dist_deg", "esaz", "used", "origin_id", "creation_time"}
+        std_cols = {"calculated_latitude", "calculated_longitude"}
         for col in df.columns:
             if col in exclude_cols:
                 continue
             if pd.api.types.is_float_dtype(df[col]):
                 summary[col] = df[col].mean()
+                if col in std_cols:
+                    summary[f"{col}_std"] = df[col].std()
             elif pd.api.types.is_bool_dtype(df[col]):
                 summary[col] = df[col].any()   # True if any True
             elif pd.api.types.is_integer_dtype(df[col]):
@@ -1242,6 +1245,12 @@ def get_stations_summary(stations_folder,
             else:
                 summary[col] = df[col].iloc[0]  # fallback
 
+        # Number of arrivals that contributed to the calculated position
+        # (one row per origin at this point, since drop_duplicates above
+        # collapses to one entry per origin_id).
+        if "calculated_latitude" in df.columns:
+            summary["calculated_num_entries"] = int(df["calculated_latitude"].notna().sum())
+
         summary["db_path"] = db_path
         summary["creation_time"] = UTCDateTime.now().strftime("%Y-%m-%d %H:%M:%S.%f")
         return summary
@@ -1265,13 +1274,23 @@ def process_arrival(arrival, source_loc,
                     ebank_index_path,
                     # table="/stations/index",
                     calculate_d_az = False,
-                    das=False
+                    das=False,
+                    row_sink=None,
+                    station_lookup=None,
+                    stations_folder=None,
                     ):
     """
     Process an individual arrival: verify pick and station information,
     optionally update distance and azimuth, and add missing or confirmed
     station records to the database.
 
+    With ``row_sink`` (a dict), the station record is appended to
+    ``row_sink[<station db path>]`` instead of being written at once; the
+    caller writes each station database once (see
+    :func:`write_station_rows`). ``station_lookup`` ({(network, station):
+    DataFrame}) and ``stations_folder`` avoid repeating per arrival what is
+    the same for the whole catalog.
+
     Parameters:
         arrival (Arrival): The arrival object to process.
         source_loc (tuple): Tuple of (latitude, longitude) for the event origin.
@@ -1294,15 +1313,15 @@ def process_arrival(arrival, source_loc,
             "network_type": None,
             "station": None,
             "channel": None,
-            "dist_deg": np.NaN,
-            "esaz": np.NaN,
+            "dist_deg": np.nan,
+            "esaz": np.nan,
             "confirmed": False,
-            "confirmed_latitude": np.NaN,
-            "confirmed_longitude": np.NaN,
-            "confirmed_elevation": np.NaN,
+            "confirmed_latitude": np.nan,
+            "confirmed_longitude": np.nan,
+            "confirmed_elevation": np.nan,
             "calculated": False,
-            "calculated_latitude": np.NaN,
-            "calculated_longitude": np.NaN,
+            "calculated_latitude": np.nan,
+            "calculated_longitude": np.nan,
             "used": False,
             "origin_id": origin_id,
             "creation_time": UTCDateTime.now().strftime("%Y-%m-%dT%H:%M:%S.%fZ")
@@ -1323,7 +1342,10 @@ def process_arrival(arrival, source_loc,
     info["channel"] = cha
     phase = arrival.phase
     
-    df_sta = df_stations[(df_stations.network == net) & (df_stations.station == sta)]
+    if station_lookup is not None:
+        df_sta = station_lookup.get((net, sta), df_stations.iloc[0:0])
+    else:
+        df_sta = df_stations[(df_stations.network == net) & (df_stations.station == sta)]
     
     if df_sta is None:
         pass
@@ -1393,18 +1415,18 @@ def process_arrival(arrival, source_loc,
     # stations_folder = os.path.join( os.path.dirname(ebank_index_path),".stations")
 
     #finding the corresponding stations folder for the network
-    network_name = os.path.basename(os.path.dirname(ebank_index_path))
-    utdq_paths = get_utdq_paths(network_name)
-    stations_folder = utdq_paths[".utdquake/export/db/stations/.stations"]
-
-    # stations_folder = os.path.join( stations_folder,".stations")
-
-    os.makedirs(stations_folder, exist_ok=True)
+    if stations_folder is None:
+        stations_folder = get_stations_folder(ebank_index_path)
+        os.makedirs(stations_folder, exist_ok=True)
     if das:
         stations_db_ev_path = os.path.join(stations_folder,f".{net}_{sta}_{cha}.db")
     else:
         stations_db_ev_path = os.path.join(stations_folder,f".{net}_{sta}.db")
 
+    if row_sink is not None:
+        row_sink.setdefault(stations_db_ev_path, []).append(info)
+        return info
+
     logger.info(stations_db_ev_path)
 
     with sqlite3.connect(stations_db_ev_path) as con:
@@ -1420,6 +1442,50 @@ def process_arrival(arrival, source_loc,
 
     return info
 
+def get_stations_folder(ebank_index_path):
+    """The folder of the per-station databases of the network of an event bank."""
+    network_name = os.path.basename(os.path.dirname(ebank_index_path))
+    utdq_paths = get_utdq_paths(network_name)
+    return utdq_paths[".utdquake/export/db"] / "stations" / f".{network_name}"
+
+
+def build_station_lookup(df_stations):
+    """{(network, station): rows of df_stations}, to look stations up once per arrival without filtering."""
+    if df_stations is None or df_stations.empty:
+        return {}
+    return {key: group for key, group in df_stations.groupby(["network", "station"], sort=False)}
+
+
+def write_station_rows(row_sink, station_sample_size=None):
+    """
+    Write the station records collected in ``row_sink`` ({db path: [records]}),
+    one transaction per station database.
+
+    With ``station_sample_size``, each station database keeps at most that
+    many records in total (spread evenly over the events of this call): enough
+    for the station-location summary (the mean and spread of the positions
+    back-projected from the arrivals), without one record per pick. Distances
+    and azimuths of the arrivals themselves are not affected.
+
+    Returns the number of records written.
+    """
+    written = 0
+    for path, rows in row_sink.items():
+        with sqlite3.connect(path) as con:
+            if station_sample_size is not None:
+                try:
+                    have = con.execute('SELECT COUNT(*) FROM "/stations/index"').fetchone()[0]
+                except sqlite3.OperationalError:
+                    have = 0
+                room = max(station_sample_size - have, 0)
+                if room < len(rows):
+                    keep = np.unique(np.linspace(0, len(rows) - 1, room).round().astype(int)) if room else []
+                    rows = [rows[i] for i in keep]
+            if rows:
+                pd.DataFrame(rows).to_sql("/stations/index", con, if_exists="append", index=False)
+                written += len(rows)
+    return written
+
 # ------------------------------------------------------
 def update_origin_quality(origin, distances, esazs, stations_used, used_phase_count):
     """
@@ -1458,7 +1524,10 @@ def parse_origin(origin,
                 df_stations=None,
                 calculate_d_az=True,
                 require_arrivals=True,
-                das=False
+                das=False,
+                row_sink=None,
+                station_lookup=None,
+                stations_folder=None,
                 ):
     """
     Process all arrivals in a seismic Origin: verify picks, update
@@ -1498,7 +1567,10 @@ def parse_origin(origin,
                         df_stations=df_stations,
                         calculate_d_az=calculate_d_az, 
                         ebank_index_path=ebank_index_path, 
-                        das=das
+                        das=das,
+                        row_sink=row_sink,
+                        station_lookup=station_lookup,
+                        stations_folder=stations_folder,
                                                     )
         # print("############### INFO",info)
         if not info["available"]:
@@ -1586,8 +1658,8 @@ def xxx(origin, df_stations,
                 missing_info ={
                     "network": net,
                     "station": sta,
-                    "latitude": np.NaN,
-                    "longitude": np.NaN,
+                    "latitude": np.nan,
+                    "longitude": np.nan,
                     "status": "missing",
                     "creation_time": UTCDateTime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}
                 missing_info = pd.DataFrame([missing_info])
@@ -1673,8 +1745,8 @@ def xxx(origin, df_stations,
                 missing_info = {
                         "network": net,
                         "station": sta,
-                        "latitude": np.NaN,
-                        "longitude": np.NaN,
+                        "latitude": np.nan,
+                        "longitude": np.nan,
                         "status": "missing",
                         "creation_time": UTCDateTime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}
                 missing_info = pd.DataFrame([missing_info])
@@ -1776,7 +1848,8 @@ def random_string(length=8):
 def append_stations_to_catalog(catalog: Catalog, df_stations,
                         ebank_index_path: str, 
                         calculate_d_az=True,
-                        das=False
+                        das=False,
+                        station_sample_size=None,
                         ) -> tuple[Catalog, pd.DataFrame]:
     """
     Append distance and azimuth to each arrival in a catalog,
@@ -1789,13 +1862,27 @@ def append_stations_to_catalog(catalog: Catalog, df_stations,
     df_stations : pandas.DataFrame
         DataFrame containing station metadata with columns:
         ['network', 'station', 'latitude', 'longitude'].
+    station_sample_size : int, optional
+        At most this many records per station database (see
+        :func:`write_station_rows`); None (default) keeps one per arrival.
 
     Returns
     -------
     catalog : obspy.core.event.Catalog
         Modified catalog with distance, azimuth, and quality metrics added.
+
+    Notes
+    -----
+    The station records are collected in memory and each station database is
+    written once per call (one transaction per station instead of one per
+    arrival: the per-arrival writes made this step take hours on a network
+    filesystem). The records themselves are the same.
     """
     random_id = random_string(8)
+    stations_folder = get_stations_folder(ebank_index_path)
+    os.makedirs(stations_folder, exist_ok=True)
+    station_lookup = build_station_lookup(df_stations)
+    row_sink = {}
     logger.info(f"Starting to process {len(catalog.events)} events in catalog {random_id}. (This id is created just for logging purposes)")
 
     for event in catalog:
@@ -1810,7 +1897,10 @@ def append_stations_to_catalog(catalog: Catalog, df_stations,
                                 ebank_index_path=ebank_index_path,
                                 df_stations=df_stations,
                                 calculate_d_az=calculate_d_az,
-                                das=das
+                                das=das,
+                                row_sink=row_sink,
+                                station_lookup=station_lookup,
+                                stations_folder=stations_folder,
                                 )
         if not gd_proc:
             logger.warning(f"Removing event {event.resource_id.id} due to too many missing picks.")
@@ -1818,6 +1908,10 @@ def append_stations_to_catalog(catalog: Catalog, df_stations,
             catalog.events.remove(event)
 
 
+    n_rows = sum(len(r) for r in row_sink.values())
+    written = write_station_rows(row_sink, station_sample_size=station_sample_size)
+    logger.info(f"Wrote {written} of {n_rows} station records to {len(row_sink)} station databases "
+                f"(station_sample_size={station_sample_size}).")
     logger.info(f"Finishing to process {len(catalog.events)} events in catalog {random_id}.")
 
     return catalog