pikobs 3.0.330__tar.gz → 3.0.332__tar.gz

{pikobs-3.0.330 → pikobs-3.0.332}/CHANGELOG.md

@@ -1147,3 +1147,11 @@ Please use the following tags when editing this file:
 ## [3.0.330] 2026-09-01
 ### Added
 - bugs
+
+## [3.0.331] 2026-09-01
+### Added
+- bugs
+
+## [3.0.332] 2026-09-01
+### Added
+- bugs

{pikobs-3.0.330/pikobs.egg-info → pikobs-3.0.332}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pikobs
-Version: 3.0.330
+Version: 3.0.332
 Summary: pikobs
 Home-page: https://gitlab.science.gc.ca/dlo001/pikobs
 Author: David Lobon

pikobs-3.0.332/VERSION

@@ -0,0 +1 @@
+3.0.332

{pikobs-3.0.330 → pikobs-3.0.332}/pikobs/obstimedb/obstimedb.py

@@ -374,50 +374,65 @@ def create_and_populate_moyenne_table(
         # it into its own column. Every other family leaves `channel` as
         # a literal NULL and groups exactly as before -- this is a pure
         # addition, the non-radiance extraction path is unchanged.
+        #
+        # CRITICAL: the per-channel rows must NEVER be the only rows
+        # inserted for a radiance family. The SAME id_obs (profile)
+        # appears in MANY channel rows (one per channel it reports), so
+        # summing Nobsdata/Nobsprofile ACROSS channel rows -- exactly
+        # what every consumer of `moyenne` that isn't channel-aware does
+        # (the main Nobs/Nobsprofile series behind the mosaic, global
+        # time series, and profile summary table) -- silently multiplies
+        # the true profile count by roughly the number of channels
+        # reported (confirmed directly: a 10-profile/15-channel test
+        # case summed to 150, not 10). The fix is to ALSO insert a
+        # SEPARATE, non-channel-split row (channel=NULL) with the
+        # correct, ungrouped-by-channel counts, so every consumer that
+        # filters `WHERE channel IS NULL` (the main series) gets the
+        # right numbers, while `WHERE channel IS NOT NULL` (the
+        # per-channel breakdown) still works exactly as before.
         # ------------------------------------------------------------------
         is_radiance = (VCOTYP == 'CANAL')
         if is_radiance:
-            channel_expr = "vcoord"
-            group_by_clause = f"{group_col}, vcoord"
+            passes = [("NULL", group_col), ("vcoord", f"{group_col}, vcoord")]
         else:
-            channel_expr = "NULL"
-            group_by_clause = group_col
-
-        insert_query = f"""
-            INSERT INTO moyenne (Nobsdata, Nobsprofile, avg_omp, avg_oma, id_stn, channel, date)
-            SELECT
-                count(DISTINCT(id_data)),
-                count(DISTINCT(id_obs)),
-                {avg_omp_expr},
-                {avg_oma_expr},
-                {id_stn_expr},
-                {channel_expr},
-                {date_match.group(1)}
-            FROM db.header
-            NATURAL JOIN db.DATA
-            WHERE obsvalue IS NOT NULL
-                {flag_criteria}
-                {LATLONCRIT}
-                {VCOCRIT}
-                {family_filter}
-            GROUP BY {group_by_clause};
-        """
-
-        try:
-            cursor.execute(insert_query)
-        except sqlite3.OperationalError as err:
-            msg = str(err).lower()
-            fb_query = insert_query
-            for metric in agr_list:
-                if metric in msg:
-                    print(f"[WARNING] Column '{metric}' not found in "
-                          f"{os.path.basename(existing_db_filename)}; storing NULL.",
-                          flush=True)
-                    fb_query = fb_query.replace(f"AVG({metric})", "NULL")
-            if fb_query != insert_query:
-                cursor.execute(fb_query)
-            else:
-                raise
+            passes = [("NULL", group_col)]
+
+        for channel_expr, group_by_clause in passes:
+            insert_query = f"""
+                INSERT INTO moyenne (Nobsdata, Nobsprofile, avg_omp, avg_oma, id_stn, channel, date)
+                SELECT
+                    count(DISTINCT(id_data)),
+                    count(DISTINCT(id_obs)),
+                    {avg_omp_expr},
+                    {avg_oma_expr},
+                    {id_stn_expr},
+                    {channel_expr},
+                    {date_match.group(1)}
+                FROM db.header
+                NATURAL JOIN db.DATA
+                WHERE obsvalue IS NOT NULL
+                    {flag_criteria}
+                    {LATLONCRIT}
+                    {VCOCRIT}
+                    {family_filter}
+                GROUP BY {group_by_clause};
+            """
+
+            try:
+                cursor.execute(insert_query)
+            except sqlite3.OperationalError as err:
+                msg = str(err).lower()
+                fb_query = insert_query
+                for metric in agr_list:
+                    if metric in msg:
+                        print(f"[WARNING] Column '{metric}' not found in "
+                              f"{os.path.basename(existing_db_filename)}; storing NULL.",
+                              flush=True)
+                        fb_query = fb_query.replace(f"AVG({metric})", "NULL")
+                if fb_query != insert_query:
+                    cursor.execute(fb_query)
+                else:
+                    raise
         mem_conn.commit()
 
         try:

{pikobs-3.0.330 → pikobs-3.0.332}/pikobs/obstimedb/obstimedb_plot.py

@@ -379,10 +379,20 @@ def _draw_profile_summary_table(df_all, run_time, name_in='experience'):
     row per (family, id_stn) PLUS one 'ALL' row per family, matching
     the reference report's per-family total row.
     """
+    # NOTE: "Yesterday" and "Last week" are NOT immediately adjacent to
+    # "Today"/"This week" -- confirmed against TWO independent reference
+    # tables given directly by the user (run_time 2026082600/00Z and
+    # 2026083112/12Z), both consistently showing a 1-DAY GAP: Yesterday
+    # ends a full 24h before Today starts, and Last week ends a full
+    # day before This week starts. This was a real bug in an earlier
+    # version of this function (computed both as immediately adjacent,
+    # no gap) -- fixed here to match the confirmed convention exactly.
     today_start = run_time - pd.Timedelta(hours=24)
-    yesterday_start = run_time - pd.Timedelta(hours=48)
+    yesterday_end = run_time - pd.Timedelta(hours=48)
+    yesterday_start = run_time - pd.Timedelta(hours=72)
     week_start = run_time - pd.Timedelta(days=7)
-    lastweek_start = run_time - pd.Timedelta(days=14)
+    lastweek_end = run_time - pd.Timedelta(days=8)
+    lastweek_start = run_time - pd.Timedelta(days=15)
     tenday_start = run_time - pd.Timedelta(days=10)
     run_hour = run_time.hour
 
@@ -400,9 +410,9 @@ def _draw_profile_summary_table(df_all, run_time, name_in='experience'):
         <b>PASS:</b> {run_hour:02d}Z<br><br>
         <b>Definitions of periods</b> (computed for this run):<br>
         Today: date &gt; {_fmtd(today_start)} and date &le; {_fmtd(run_time)}<br>
-        Yesterday: date &gt; {_fmtd(yesterday_start)} and date &le; {_fmtd(today_start)}<br>
+        Yesterday: date &gt; {_fmtd(yesterday_start)} and date &le; {_fmtd(yesterday_end)}<br>
         This week: date &gt; {_fmtd(week_start)} and date &le; {_fmtd(run_time)}<br>
-        Last week: date &gt; {_fmtd(lastweek_start)} and date &le; {_fmtd(week_start)}<br>
+        Last week: date &gt; {_fmtd(lastweek_start)} and date &le; {_fmtd(lastweek_end)}<br>
         Last 10 days (same PASS {run_hour:02d}Z): date &gt; {_fmtd(tenday_start)}
         and date &le; {_fmtd(run_time)}<br><br>
         <b>Percentage difference calculations:</b><br>
@@ -434,9 +444,9 @@ def _draw_profile_summary_table(df_all, run_time, name_in='experience'):
 
     def _build_row(family, id_stn, g):
         today = _period_sum(g, today_start, run_time)
-        yesterday = _period_sum(g, yesterday_start, today_start)
+        yesterday = _period_sum(g, yesterday_start, yesterday_end)
         this_week = _period_sum(g, week_start, run_time)
-        last_week = _period_sum(g, lastweek_start, week_start)
+        last_week = _period_sum(g, lastweek_start, lastweek_end)
         at_run = g.loc[g['date'] == run_time, 'Nprofile'].sum()
         tenday_avg = _period_mean(g, tenday_start, run_time, same_hour=True)
         return {
@@ -881,7 +891,7 @@ def obstimedb_plot(
             select_extra.append(
                 f"SUM(CASE WHEN {col} IS NOT NULL THEN Nobsdata ELSE 0 END) as _wden_{metric}")
         query = ("SELECT date, id_stn, " + ", ".join(select_extra) +
-                 " FROM moyenne GROUP BY date, id_stn;")
+                 " FROM moyenne WHERE channel IS NULL GROUP BY date, id_stn;")
         try:
             df = pd.read_sql_query(query, conn)
         except Exception as e:

{pikobs-3.0.330 → pikobs-3.0.332}/pikobs/script/run_obstimedb.sh

@@ -8,18 +8,18 @@ set -eo pipefail
 # ============================================================================
 # Directory containing the SQLite observation databases for the (single)
 # experiment. obstimedb tracks ONE run over time — no control/reference.
-PATH_EXPERIENCE_FILES="/home/dlo001/data_maestro/ppp7/maestro_archives/NAT910E22S50/monitoring/banco/postalt/"
+PATH_EXPERIENCE_FILES="/home/smco500/.suites/gdps/g2/hub/ppp7/monitoring/banco/postalt/"
 # Label for the experiment data.
 EXPERIENCE_NAME="GDPS-SN"
 # Output directory where the generated results will be saved.
 PATHWORK="/home/dlo001/sites8/pikobs_obstimedb"
 # Start and end dates for the processing window in YYYYMMDDHH format (UTC).
-DATESTART="2022060100"
-DATEEND="2022070100"
+DATESTART="2026082106"
+DATEEND="2026083112"
 # Region to process.
 REGION="Monde"
 # Observation families to process (space-separated list).
-FAMILY="ai  atms  cris  csr  gp  iasi  mwhs2  ro  sc  sf  ssmis  sw  to_amsua to_amsub ua"
+FAMILY="ai atms_allsky ch cris csr gp iasi mwhs2 ro sc sf ssmis sw to_amsua_allsky to_amsub_allsky ua"
 # Bitmask filter criteria.
 FLAGS_CRITERIA="assimilee"
 # Baseline look-back window, in calendar days, for the same synoptic hour

{pikobs-3.0.330 → pikobs-3.0.332/pikobs.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pikobs
-Version: 3.0.330
+Version: 3.0.332
 Summary: pikobs
 Home-page: https://gitlab.science.gc.ca/dlo001/pikobs
 Author: David Lobon

pikobs-3.0.330/VERSION

@@ -1 +0,0 @@
-3.0.330