gw_data 0.3.0__tar.gz → 0.3.2__tar.gz

{gw_data-0.3.0 → gw_data-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gw_data
-Version: 0.3.0
+Version: 0.3.2
 Summary: Gridworks Data Infrastructure
 Author-email: Jessica Millar <jessica.lynn.millar@gmail.com>
 Requires-Python: <3.13.0,>=3.12
@@ -149,41 +149,4 @@ We have column-store compression enabled on the readings table for data older th
 
 ### Useful Queries
 
-The following queries are useful for analyzing TimescaleDB performance:
-
-```
--- Display the size in MB of our two main tables
-SELECT pg_size_pretty(hypertable_size('readings')) as "Readings Table Size", pg_size_pretty(hypertable_size('messages')) as "Messages Table Size";
-```
-
-```
--- Display the size of all database tables in order.
--- This will show each TimescaleDB chunk as a separate table.
-SELECT
-    table_schema || '.' || table_name AS table_full_name,
-    pg_size_pretty(pg_total_relation_size('"' || table_schema || '"."' || table_name || '"')) AS size
-FROM information_schema.tables
-ORDER BY
-    pg_total_relation_size('"' || table_schema || '"."' || table_name || '"') DESC;
-
-```
-
-```
--- Display compression stats for each chunk in the readings table
-SELECT 
-    chunk_name,
-    pg_size_pretty(before_compression_total_bytes) AS size_before,
-    pg_size_pretty(after_compression_total_bytes) AS size_after,
-    100 - (after_compression_total_bytes::float / before_compression_total_bytes * 100) AS compression_ratio_pct
-FROM chunk_compression_stats('readings');
-```
-
-```
--- Get the ID of the policy_compression job so you can manually run `CALL run_job` with it (e.g. after a bulk import).
-SELECT job_id, proc_name, hypertable_name, config 
-FROM timescaledb_information.jobs;
-```
-```
--- Get info (table, time range, etc.) about the TimescaleDB chunks.
-SELECT * FROM timescaledb_information.chunks
-```
+There are some useful queries listed in `useful-queries.sql` in the project root.

{gw_data-0.3.0 → gw_data-0.3.2}/README.md

@@ -134,41 +134,4 @@ We have column-store compression enabled on the readings table for data older th
 
 ### Useful Queries
 
-The following queries are useful for analyzing TimescaleDB performance:
-
-```
--- Display the size in MB of our two main tables
-SELECT pg_size_pretty(hypertable_size('readings')) as "Readings Table Size", pg_size_pretty(hypertable_size('messages')) as "Messages Table Size";
-```
-
-```
--- Display the size of all database tables in order.
--- This will show each TimescaleDB chunk as a separate table.
-SELECT
-    table_schema || '.' || table_name AS table_full_name,
-    pg_size_pretty(pg_total_relation_size('"' || table_schema || '"."' || table_name || '"')) AS size
-FROM information_schema.tables
-ORDER BY
-    pg_total_relation_size('"' || table_schema || '"."' || table_name || '"') DESC;
-
-```
-
-```
--- Display compression stats for each chunk in the readings table
-SELECT 
-    chunk_name,
-    pg_size_pretty(before_compression_total_bytes) AS size_before,
-    pg_size_pretty(after_compression_total_bytes) AS size_after,
-    100 - (after_compression_total_bytes::float / before_compression_total_bytes * 100) AS compression_ratio_pct
-FROM chunk_compression_stats('readings');
-```
-
-```
--- Get the ID of the policy_compression job so you can manually run `CALL run_job` with it (e.g. after a bulk import).
-SELECT job_id, proc_name, hypertable_name, config 
-FROM timescaledb_information.jobs;
-```
-```
--- Get info (table, time range, etc.) about the TimescaleDB chunks.
-SELECT * FROM timescaledb_information.chunks
-```
+There are some useful queries listed in `useful-queries.sql` in the project root.

{gw_data-0.3.0 → gw_data-0.3.2}/alembic/env.py

@@ -10,8 +10,9 @@ from gw_data.config import Settings
 GW_SCHEMA = "gridworks"
 def include_name(name, type_, parent_names):
     if type_ == "schema":
-        # Only track migrations in your chosen schema
         return name == GW_SCHEMA
+    elif type_ == "table":
+        return name not in ['cached_hourly_data']
     return True
 
 # -----------------------------------------------------------------------------

gw_data-0.3.2/alembic/versions/a7f151e8163f_visualizer_functions_and_views.py

@@ -0,0 +1,74 @@
+"""visualizer functions and views
+
+Revision ID: a7f151e8163f
+Revises: 1220f2f941dd
+Create Date: 2026-06-18 10:34:33.737411
+
+"""
+import os
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'a7f151e8163f'
+down_revision: Union[str, Sequence[str], None] = '1220f2f941dd'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    # ### end Alembic commands ###
+
+    op.execute("GRANT UPDATE (last_login, updated_at) ON gridworks.users TO gw_visualizer;")
+
+    op.execute("CREATE FUNCTION gridworks.c2f(temp_c FLOAT) RETURNS FLOAT IMMUTABLE LANGUAGE 'sql' RETURN temp_c * 9.0 / 5.0 + 32;")
+    op.execute("CREATE FUNCTION gridworks.f2c(temp_f FLOAT) RETURNS FLOAT IMMUTABLE LANGUAGE 'sql' RETURN (temp_f - 32) * 5.0 / 9.0;")
+
+    misc_scripts_folder = os.path.join(
+        os.path.dirname(__file__), 
+        '../../src/gw_data/db/scripts/misc/',
+    )
+
+    for script in [
+        'create_view_readings_1hr.sql',
+        'create_policy_view_readings_1hr.sql',
+        'create_udf_retrieve_readings_1s.sql',
+        'create_udf_calc_energy_1hr.sql',
+        'create_udf_calc_hourly_data.sql',
+        'create_table_cached_hourly_data.sql',
+        'create_proc_refresh_cached_hourly_data.sql'
+    ]:
+        with open(os.path.join(misc_scripts_folder, script), 'r') as f:
+            sql = f.read().strip()
+            op.execute(sql)
+        
+
+    op.execute("SELECT add_job('gridworks.refresh_cached_hourly_data', INTERVAL '30 minutes');")
+
+    print("""
+          IMPORTANT: to populate existing data into the `gridworks.readings_1hr` continuous aggregate, you must manually run `refresh_continuous_aggregate`.
+          IMPORTANT: to populate existing data into the `gridworks.cached_hourly_data` table, you will need to run `refresh_all_cached_hourly_data.py`.
+          """)
+
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+
+    # ### end Alembic commands ###
+
+    op.execute("SELECT delete_job((SELECT job_id FROM timescaledb_information.jobs WHERE proc_schema = 'gridworks' AND proc_name = 'refresh_cached_hourly_data'));")
+    op.execute('DROP PROCEDURE gridworks.refresh_cached_hourly_data;')
+    op.execute('DROP TABLE gridworks.cached_hourly_data;')
+    op.execute('DROP FUNCTION gridworks.calc_hourly_data;')
+    op.execute('DROP MATERIALIZED VIEW gridworks.readings_1hr;')
+    op.execute('DROP FUNCTION gridworks.retrieve_readings_1s(timestamp with time zone, timestamp with time zone, text[]);')
+    op.execute('DROP FUNCTION gridworks.calc_energy_1hr(timestamp with time zone, timestamp with time zone);')
+    op.execute('DROP FUNCTION gridworks.c2f(float)')
+    op.execute('DROP FUNCTION gridworks.f2c(float)')

{gw_data-0.3.0 → gw_data-0.3.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "gw_data"
-version = "0.3.0"
+version = "0.3.2"
 description = "Gridworks Data Infrastructure"
 readme = "README.md"
 authors = [

{gw_data-0.3.0 → gw_data-0.3.2}/src/gw_data/db/models/__init__.py

@@ -8,6 +8,7 @@ from gw_data.db.models.customer import CustomerSql
 from gw_data.db.models.installer import InstallerSql
 from gw_data.db.models.user import UserSql
 from gw_data.db.models.installation import InstallationSql
+from gw_data.db.models.user_installation_role import UserInstallationRoleSql
 
 from gw_data.db.models.reading_channel import ReadingChannelSql
 from gw_data.db.models.message import MessageSql
@@ -31,6 +32,7 @@ __all__ = [
     "InstallerSql",
     "UserSql",
     "InstallationSql",
+    "UserInstallationRoleSql",
     "ReadingChannelSql",
     "MessageSql",
     "ReadingSql"

{gw_data-0.3.0 → gw_data-0.3.2}/src/gw_data/db/models/reading.py

@@ -39,7 +39,7 @@ class ReadingSql(Base):
         index=True
     )
     timestamp: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, index=True)
-    value: Mapped[BigInteger] = mapped_column(BigInteger, nullable=False)
+    value: Mapped[int] = mapped_column(BigInteger, nullable=False)
 
     __table_args__ = (
         # TimescaleDB automatically creates this one; we need to include it so Alembic doesn't get confused

{gw_data-0.3.0 → gw_data-0.3.2}/src/gw_data/db/models/user.py

@@ -29,7 +29,7 @@ class UserSql(Base):
     created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
     updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
 
-    installation_roles: Mapped[list[UserInstallationRoleSql]] = relationship(uselist=True, lazy="joined")
+    installation_roles: Mapped[list[UserInstallationRoleSql]] = relationship(uselist=True)
 
     def __repr__(self) -> str:
         return f"User(id={self.id!r}, username={self.username!r}, is_active={self.is_active!r})"

{gw_data-0.3.0 → gw_data-0.3.2}/src/gw_data/db/models/user_installation_role.py

@@ -23,7 +23,6 @@ class UserInstallationRoleSql(Base):
     installations: Mapped[list[InstallationSql]] = relationship(
         "InstallationSql",
         uselist=True,
-        lazy="joined",
         primaryjoin="or_(InstallationSql.id==UserInstallationRoleSql.installation_id, UserInstallationRoleSql.installation_id == None)"
     )
 

gw_data-0.3.2/src/gw_data/db/scripts/misc/adjust_report_message_timestamps.sql

@@ -0,0 +1,59 @@
+-- report.event message parsing had a bug that caused the timestamps to get set in local time rather than UTC.
+-- This script fixes them.
+-- (Unfortunately we can't just UPDATE a timestamp column with TimescaleDB)
+-- This never needed to be run in prod, since the server local time was UTC.
+
+-- SET temp_buffers = '2000MB'; 
+
+BEGIN;
+
+CREATE TEMP TABLE temp_report_messages AS
+SELECT 
+	id, 
+	"timestamp" + INTERVAL '6 hours' AS "timestamp",
+	from_alias, 
+	created_at + INTERVAL '6 hours' AS created_at,
+	persisted_at, 
+	message_type_name, 
+	payload
+FROM gridworks.messages 
+-- where id='85c153a2-ebf6-4a24-83ba-5dba12967d87';
+WHERE message_type_name='report.event';
+
+DELETE FROM gridworks.messages
+-- where id='85c153a2-ebf6-4a24-83ba-5dba12967d87';
+WHERE message_type_name='report.event';
+
+INSERT INTO gridworks.messages SELECT * FROM temp_report_messages;
+
+WITH 
+	q1 AS (
+	    SELECT COUNT(*) as cnt, ROW_NUMBER() OVER () AS rn FROM gridworks.messages
+	),
+	q2 AS (
+	    SELECT *, ROW_NUMBER() OVER () AS rn 
+		FROM gridworks.messages 
+		where id='85c153a2-ebf6-4a24-83ba-5dba12967d87'
+	)
+SELECT 
+    q1.cnt, 
+    q2.*
+FROM q1
+FULL OUTER JOIN q2 ON q1.rn = q2.rn;
+
+-- Fix the OAT and WS readings
+select id from gridworks.reading_channels where name = 'forecast-ws';
+update gridworks.readings
+set value=value * 10
+where channel_id in (
+	'792a01bc-1b95-4434-97e5-30d64c000024',
+	'3310111e-33ab-4c07-bab1-7d4c7d464201',
+	'60bb6806-e5a5-4a06-a60e-c26b0bf77e59',
+	'a349f893-61a7-4c65-8e33-998517d06fb0',
+	'f0ae219e-73d1-4c61-9c02-a13e940b8737',
+	'97a42794-bb6b-4a4d-be6f-1c5c9f955af7'
+)
+
+
+COMMIT
+-- ROLLBACK;

gw_data-0.3.2/src/gw_data/db/scripts/misc/create_policy_view_readings_1hr.sql

@@ -0,0 +1,4 @@
+SELECT add_continuous_aggregate_policy('gridworks.readings_1hr',
+  start_offset => INTERVAL '2 days',
+  end_offset   => INTERVAL '1 hour',
+  schedule_interval => INTERVAL '30 minutes');

gw_data-0.3.2/src/gw_data/db/scripts/misc/create_proc_refresh_cached_hourly_data.sql

@@ -0,0 +1,20 @@
+CREATE OR REPLACE PROCEDURE gridworks.refresh_cached_hourly_data(job_id int, config jsonb)
+LANGUAGE plpgsql
+AS $$
+DECLARE
+    t_start timestamp with time zone := now() - INTERVAL '2 days';
+    t_start_valid timestamp with time zone := t_start + INTERVAL '1 hour';
+    t_end   timestamp with time zone := now();
+BEGIN
+    DELETE FROM gridworks.cached_hourly_data
+    WHERE time_bucket >= t_start_valid AND time_bucket <= t_end;
+
+    INSERT INTO gridworks.cached_hourly_data
+    WITH hourly_data AS MATERIALIZED (
+        SELECT * FROM gridworks.calc_hourly_data(t_start => t_start, t_end => t_end)
+    )
+    SELECT * FROM hourly_data
+    WHERE time_bucket >= t_start_valid;
+END;
+$$;
+

gw_data-0.3.2/src/gw_data/db/scripts/misc/create_table_cached_hourly_data.sql

@@ -0,0 +1,41 @@
+CREATE TABLE gridworks.cached_hourly_data (
+    terminal_asset_alias text,
+    time_bucket timestamp with time zone,
+    hp_kwh_el float,
+    hp_kwh_th float,
+    dist_kwh float,
+    store_change_kwh float,
+    hp_avg_lwt float,
+    hp_avg_ewt float,
+    dist_avg_swt float,
+    dist_avg_rwt float,
+    buffer_depth1_start float,
+    buffer_depth2_start float,
+    buffer_depth3_start float,
+    buffer_depth4_start float,
+    tank1_depth1_start float,
+    tank1_depth2_start float,
+    tank1_depth3_start float,
+    tank1_depth4_start float,
+    tank2_depth1_start float,
+    tank2_depth2_start float,
+    tank2_depth3_start float,
+    tank2_depth4_start float,
+    tank3_depth1_start float,
+    tank3_depth2_start float,
+    tank3_depth3_start float,
+    tank3_depth4_start float,
+    relay_3_pulled_fraction float,
+    relay_5_pulled_fraction float,
+    relay_6_pulled_fraction float,
+    relay_9_pulled_fraction float,
+    zone1_heatcall_fraction float,
+    zone2_heatcall_fraction float,
+    zone3_heatcall_fraction float,
+    zone4_heatcall_fraction float,
+    ws_mph float,
+    oat_f float,
+    total_usd_per_mwh float,
+    buffer_available_kwh float,
+    lmp_usd_per_mwh float    
+)

gw_data-0.3.2/src/gw_data/db/scripts/misc/create_udf_calc_energy_1hr.sql

@@ -0,0 +1,56 @@
+CREATE FUNCTION gridworks.calc_energy_1hr(
+	t_start timestamp with time zone,
+	t_end timestamp with time zone)
+    RETURNS TABLE(
+        terminal_asset_alias text,
+        time_bucket timestamp with time zone,
+        hp_kwh float,
+        dist_kwh float,
+        store_change_kwh float
+    ) 
+    STABLE
+	LANGUAGE 'sql'
+
+AS $BODY$
+
+    SELECT
+        terminal_asset_alias,
+        time_bucket(INTERVAL '1 hour', time_bucket_1s) AS time_bucket_1h,
+        avg(hp_kw) AS hp_kwh,
+        avg(dist_kw) AS dist_kwh,
+        avg(store_change_kw) AS store_change_kwh
+    FROM (
+        SELECT 
+            *,
+            500 * primary_flow_gpm * (hp_lwt_c - hp_ewt_c) * 9 / 5 / 3410 AS hp_kw,
+            500 * dist_flow_gpm * (dist_swt_c - dist_rwt_c) * 9 / 5 / 3410 AS dist_kw,
+            500 * (
+                (1 - store_charge_discharge_0_1) * store_flow_gpm +
+                store_charge_discharge_0_1 * primary_flow_gpm
+            ) * (store_charge_discharge_0_1 * 2 - 1) * (store_hot_pipe_c - store_cold_pipe_c) * 9 / 5 / 3410 AS store_change_kw
+        FROM (
+            SELECT
+                terminal_asset_alias,
+                time_bucket AS time_bucket_1s,
+                (AVG (value) FILTER (WHERE channel_name='primary-flow' AND unit='GpmTimes100') / 100) AS primary_flow_gpm,
+                (AVG (value) FILTER (WHERE channel_name='hp-lwt' AND unit='WaterTempCTimes1000') / 1000) AS hp_lwt_c,
+                (AVG (value) FILTER (WHERE channel_name='hp-ewt' AND unit='WaterTempCTimes1000') / 1000) AS hp_ewt_c,
+                (AVG (value) FILTER (WHERE channel_name='dist-flow' AND unit='GpmTimes100') / 100) AS dist_flow_gpm,
+                (AVG (value) FILTER (WHERE channel_name='dist-swt' AND unit='WaterTempCTimes1000') / 1000) AS dist_swt_c,
+                (AVG (value) FILTER (WHERE channel_name='dist-rwt' AND unit='WaterTempCTimes1000') / 1000) AS dist_rwt_c,
+                (AVG (value) FILTER (WHERE channel_name='store-flow' AND unit='GpmTimes100') / 100) AS store_flow_gpm,
+                (AVG (value) FILTER (WHERE channel_name='store-hot-pipe' AND unit='WaterTempCTimes1000') / 1000) AS store_hot_pipe_c,
+                (AVG (value) FILTER (WHERE channel_name='store-cold-pipe' AND unit='WaterTempCTimes1000') / 1000) AS store_cold_pipe_c,
+                (AVG (value) FILTER (WHERE channel_name='charge-discharge-relay3')) AS store_charge_discharge_0_1
+            FROM gridworks.retrieve_readings_1s(
+                t_start => t_start,
+                t_end => t_end, 
+                channels => ARRAY['hp-ewt','hp-lwt', 'primary-flow', 'dist-swt', 'dist-rwt', 'dist-flow', 'charge-discharge-relay3', 'store-hot-pipe', 'store-cold-pipe', 'store-flow']
+            )
+            GROUP BY terminal_asset_alias,time_bucket
+        )
+    )
+        
+    GROUP BY terminal_asset_alias,time_bucket_1h
+    ORDER BY terminal_asset_alias,time_bucket_1h
+$BODY$;

gw_data-0.3.2/src/gw_data/db/scripts/misc/create_udf_calc_hourly_data.sql

@@ -0,0 +1,162 @@
+CREATE FUNCTION gridworks.calc_hourly_data(
+	t_start timestamp with time zone,
+	t_end timestamp with time zone)
+    RETURNS TABLE(
+        -- NOTE that this table must match the cached_hourly_data table!
+        terminal_asset_alias text,
+        time_bucket timestamp with time zone,
+        hp_kwh_el float,
+        hp_kwh_th float,
+        dist_kwh float,
+        store_change_kwh float,
+        hp_avg_lwt float,
+        hp_avg_ewt float,
+        dist_avg_swt float,
+        dist_avg_rwt float,
+        buffer_depth1_start float,
+        buffer_depth2_start float,
+        buffer_depth3_start float,
+        buffer_depth4_start float,
+        tank1_depth1_start float,
+        tank1_depth2_start float,
+        tank1_depth3_start float,
+        tank1_depth4_start float,
+        tank2_depth1_start float,
+        tank2_depth2_start float,
+        tank2_depth3_start float,
+        tank2_depth4_start float,
+        tank3_depth1_start float,
+        tank3_depth2_start float,
+        tank3_depth3_start float,
+        tank3_depth4_start float,
+        relay_3_pulled_fraction float,
+        relay_5_pulled_fraction float,
+        relay_6_pulled_fraction float,
+        relay_9_pulled_fraction float,
+        zone1_heatcall_fraction float,
+        zone2_heatcall_fraction float,
+        zone3_heatcall_fraction float,
+        zone4_heatcall_fraction float,
+        ws_mph float,
+        oat_f float,
+        total_usd_per_mwh float,
+        buffer_available_kwh float,
+        lmp_usd_per_mwh float
+    )
+    STABLE
+	LANGUAGE 'sql'
+
+
+AS $BODY$
+    SELECT terminal_asset_alias, time_bucket,
+    (hp_idu_w + hp_odu_w) / 1000 AS hp_kwh_el,
+    hp_kwh AS hp_kwh_th,
+    dist_kwh,
+    store_change_kwh,
+    hp_avg_lwt,hp_avg_ewt,
+    dist_avg_swt, dist_avg_rwt,
+    buffer_depth1_start, buffer_depth2_start, buffer_depth3_start, buffer_depth4_start,
+    tank1_depth1_start, tank1_depth2_start, tank1_depth3_start, tank1_depth4_start,
+    tank2_depth1_start, tank2_depth2_start, tank2_depth3_start, tank2_depth4_start,
+    tank3_depth1_start, tank3_depth2_start, tank3_depth3_start, tank3_depth4_start,
+    relay_3_pulled_fraction, relay_5_pulled_fraction, relay_6_pulled_fraction, relay_9_pulled_fraction,
+    zone1_heatcall_fraction, zone2_heatcall_fraction, zone3_heatcall_fraction, zone4_heatcall_fraction,
+    ws_mph, oat_f,
+    total_usd_per_mwh,
+    buffer_available_kwh,
+    lmp_usd_per_mwh
+    FROM
+    (
+        SELECT * FROM (
+            SELECT terminal_asset_alias, time_bucket, hp_kwh, dist_kwh, store_change_kwh
+            FROM gridworks.calc_energy_1hr(t_start => t_start, t_end => t_end)
+        ) AS kwh_values
+        JOIN (
+            SELECT
+                terminal_asset_alias AS avg_value_ta,
+                time_bucket AS avg_value_time_bucket,
+                AVG (avg_value) FILTER (WHERE channel_name='hp-idu-pwr' AND unit='PowerW') AS hp_idu_w,
+                AVG (avg_value) FILTER (WHERE channel_name='hp-odu-pwr' AND unit='PowerW') AS hp_odu_w,
+                gridworks.C2F(AVG (avg_value) FILTER (WHERE channel_name='hp-lwt' AND unit='WaterTempCTimes1000') / 1000) AS hp_avg_lwt,
+                gridworks.C2F(AVG (avg_value) FILTER (WHERE channel_name='hp-ewt' AND unit='WaterTempCTimes1000') / 1000) AS hp_avg_ewt,
+                gridworks.C2F(AVG (avg_value) FILTER (WHERE channel_name='dist-swt' AND unit='WaterTempCTimes1000') / 1000) AS dist_avg_swt,
+                gridworks.C2F(AVG (avg_value) FILTER (WHERE channel_name='dist-rwt' AND unit='WaterTempCTimes1000') / 1000) AS dist_avg_rwt,
+                AVG (avg_value) FILTER (where channel_name like '%-relay3') AS relay_3_pulled_fraction,
+                AVG (avg_value) FILTER (where channel_name like '%-relay5') AS relay_5_pulled_fraction,
+                AVG (avg_value) FILTER (where channel_name like '%-relay6') AS relay_6_pulled_fraction,
+                AVG (avg_value) FILTER (where channel_name like '%-relay9') AS relay_9_pulled_fraction,
+                AVG (avg_value) FILTER (where channel_name like 'zone1-%-heat-call') AS zone1_heatcall_fraction,
+                AVG (avg_value) FILTER (where channel_name like 'zone2-%-heat-call') AS zone2_heatcall_fraction,
+                AVG (avg_value) FILTER (where channel_name like 'zone3-%-heat-call') AS zone3_heatcall_fraction,
+                AVG (avg_value) FILTER (where channel_name like 'zone4-%-heat-call') AS zone4_heatcall_fraction
+            FROM (
+                SELECT 
+                    terminal_asset_alias,
+                    channel_name,
+                    unit,
+                    time_bucket,
+                    interpolated_average(
+                        time_weight,
+                        time_bucket,
+                        '1 hour',
+                        LAG(time_weight) OVER (PARTITION BY terminal_asset_alias,channel_name,unit ORDER BY time_bucket),
+                        LEAD(time_weight) OVER (PARTITION BY terminal_asset_alias,channel_name,unit ORDER BY time_bucket)
+                    ) AS avg_value
+                FROM (
+                    SELECT * FROM gridworks.readings_1hr
+                    WHERE 1=1
+                    AND time_bucket >= t_start
+                    AND time_bucket <= t_end
+                )
+            )
+            GROUP BY time_bucket,terminal_asset_alias
+            ORDER BY time_bucket
+        ) AS avg_values ON kwh_values.time_bucket = avg_value_time_bucket AND kwh_values.terminal_asset_alias = avg_value_ta
+        JOIN (
+            SELECT
+                terminal_asset_alias AS first_value_ta,
+                time_bucket AS first_value_time_bucket,
+                AVG (first_val) FILTER (WHERE channel_name='buffer-depth1' AND unit='FahrenheitX100') / 100 AS buffer_depth1_start,
+                AVG (first_val) FILTER (WHERE channel_name='buffer-depth2' AND unit='FahrenheitX100') / 100 AS buffer_depth2_start,
+                AVG (first_val) FILTER (WHERE channel_name='buffer-depth3' AND unit='FahrenheitX100') / 100 AS buffer_depth3_start,
+                AVG (first_val) FILTER (WHERE channel_name='buffer-depth4' AND unit='FahrenheitX100') / 100 AS buffer_depth4_start,
+                AVG (first_val) FILTER (WHERE channel_name='tank1-depth1' AND unit='FahrenheitX100') / 100 AS tank1_depth1_start,
+                AVG (first_val) FILTER (WHERE channel_name='tank1-depth2' AND unit='FahrenheitX100') / 100 AS tank1_depth2_start,
+                AVG (first_val) FILTER (WHERE channel_name='tank1-depth3' AND unit='FahrenheitX100') / 100 AS tank1_depth3_start,
+                AVG (first_val) FILTER (WHERE channel_name='tank1-depth4' AND unit='FahrenheitX100') / 100 AS tank1_depth4_start,
+                AVG (first_val) FILTER (WHERE channel_name='tank2-depth1' AND unit='FahrenheitX100') / 100 AS tank2_depth1_start,
+                AVG (first_val) FILTER (WHERE channel_name='tank2-depth2' AND unit='FahrenheitX100') / 100 AS tank2_depth2_start,
+                AVG (first_val) FILTER (WHERE channel_name='tank2-depth3' AND unit='FahrenheitX100') / 100 AS tank2_depth3_start,
+                AVG (first_val) FILTER (WHERE channel_name='tank2-depth4' AND unit='FahrenheitX100') / 100 AS tank2_depth4_start,
+                AVG (first_val) FILTER (WHERE channel_name='tank3-depth1' AND unit='FahrenheitX100') / 100 AS tank3_depth1_start,
+                AVG (first_val) FILTER (WHERE channel_name='tank3-depth2' AND unit='FahrenheitX100') / 100 AS tank3_depth2_start,
+                AVG (first_val) FILTER (WHERE channel_name='tank3-depth3' AND unit='FahrenheitX100') / 100 AS tank3_depth3_start,
+                AVG (first_val) FILTER (WHERE channel_name='tank3-depth4' AND unit='FahrenheitX100') / 100 AS tank3_depth4_start,
+                AVG (first_val) FILTER (where channel_name = 'forecast-oat' AND unit='FahrenheitX100') / 100 AS oat_f,
+                AVG (first_val) FILTER (where channel_name = 'forecast-ws' AND unit='MilesPerHourX1000') / 1000 AS ws_mph,
+                AVG (first_val) FILTER (where channel_name = 'total-usd-per-mwh' AND unit='DollarsX1000') / 1000 AS total_usd_per_mwh,
+                AVG (first_val) FILTER (where channel_name = 'lmp-usd-per-mwh' AND unit='DollarsX1000') / 1000 AS lmp_usd_per_mwh,
+                AVG (first_val) FILTER (where channel_name = 'buffer-available-kwh' AND unit='KilowattHoursX1000') / 1000 AS buffer_available_kwh			
+                
+            FROM (
+                SELECT 
+                    terminal_asset_alias,
+                    channel_name,
+                    unit,
+                    time_bucket,
+                    first_val(time_weight) AS first_val
+                FROM (
+                    SELECT * FROM gridworks.readings_1hr
+                    WHERE 1=1
+                    AND time_bucket >= t_start
+                    AND time_bucket <= t_end
+                    ORDER BY time_bucket
+                )
+            )
+            GROUP BY time_bucket,terminal_asset_alias
+            ORDER BY time_bucket
+        ) AS first_values ON kwh_values.time_bucket = first_value_time_bucket AND kwh_values.terminal_asset_alias = first_value_ta
+        OFFSET 0 -- Optimization 
+    )
+    ORDER BY time_bucket
+$BODY$;

gw_data-0.3.2/src/gw_data/db/scripts/misc/create_udf_retrieve_readings_1s.sql

@@ -0,0 +1,40 @@
+CREATE FUNCTION gridworks.retrieve_readings_1s(
+	t_start timestamp with time zone,
+	t_end timestamp with time zone,
+	channels text[])
+    RETURNS TABLE(terminal_asset_alias text, channel_name text, unit text, unit_type text, time_bucket timestamp with time zone, value numeric) 
+    STABLE
+	LANGUAGE 'sql'
+
+AS $BODY$
+	
+	SELECT 
+		time_values.terminal_asset_alias,
+		time_values.channel_name, 
+		time_values.unit, 
+		time_values.unit_type, 
+		time_bucket_gapfill(INTERVAL '1 seconds', time_values.time_bucket) AS time_bucket_gapfilled, 
+		locf(max(time_values.avg_value)) AS last_reading_value
+	FROM (
+		SELECT 
+			rc.terminal_asset_alias,
+			rc.name AS channel_name, 
+			rc.unit, 
+			rc.unit_type, 
+			time_bucket(INTERVAL '1 seconds', r.timestamp) AS time_bucket, 
+			avg(r.value) AS avg_value
+			-- time_weight('LOCF', r.timestamp, r.value) AS time_weight
+		FROM gridworks.readings r
+		JOIN gridworks.reading_channels rc ON rc.id = r.channel_id
+		WHERE 
+			r.timestamp >= t_start --'2026-04-20T04:00:00Z'
+			AND r.timestamp <= t_end --'2026-04-20T07:00:00Z'
+			AND rc.name = ANY(channels)
+		GROUP BY time_bucket, rc.name, rc.unit, rc.unit_type, rc.terminal_asset_alias
+		ORDER BY rc.name, time_bucket
+	) AS time_values
+	WHERE 1=1
+	AND time_values.time_bucket >= t_start
+	AND time_values.time_bucket <= t_end
+	GROUP BY time_values.terminal_asset_alias, time_values.channel_name, time_values.unit, time_values.unit_type, time_bucket_gapfilled
+$BODY$;

gw_data-0.3.2/src/gw_data/db/scripts/misc/create_view_readings_1hr.sql

@@ -0,0 +1,15 @@
+CREATE MATERIALIZED VIEW gridworks.readings_1hr
+WITH (timescaledb.continuous)
+AS
+	SELECT 
+		reading_channels.terminal_asset_alias AS terminal_asset_alias,
+		reading_channels.name AS channel_name, 
+		reading_channels.unit AS unit, 
+		reading_channels.unit_type AS unit_type, 
+		time_bucket(INTERVAL '1 hour', readings.timestamp) AS time_bucket, 
+		time_weight('LOCF', readings.timestamp, readings.value) AS time_weight
+	FROM gridworks.readings 
+	JOIN gridworks.reading_channels ON reading_channels.id = readings.channel_id
+	GROUP BY time_bucket, reading_channels.terminal_asset_alias, reading_channels.name, reading_channels.unit, reading_channels.unit_type 
+	ORDER BY reading_channels.terminal_asset_alias, reading_channels.name, time_bucket
+WITH NO DATA;

gw_data-0.3.2/src/gw_data/db/scripts/misc/refresh_all_cached_hourly_data.py

@@ -0,0 +1,46 @@
+import logging
+from datetime import datetime, timedelta, timezone
+import dotenv
+
+from sqlalchemy import create_engine, text
+from sqlalchemy.orm import sessionmaker
+
+from gw_data.config import Settings
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s")
+logger = logging.getLogger(__name__)
+
+dotenv.load_dotenv()
+settings = Settings()
+engine = create_engine(settings.db_url.get_secret_value(), echo=False)
+
+db_sessionmaker = sessionmaker(bind=engine)
+db_session = db_sessionmaker()
+
+
+# t_start = datetime.now(timezone.utc).replace(minute=0, second=0, microsecond=0)
+t_start = datetime(2026, 6, 1, tzinfo=timezone.utc)
+t_final = datetime(2026, 1, 31, tzinfo=timezone.utc)
+
+# t_start = datetime(2026, 4, 22, tzinfo=timezone.utc)
+# t_final = datetime(2026, 4, 18, tzinfo=timezone.utc)
+
+
+while t_start > t_final:
+    logger.info(f'Refreshing from {t_start.isoformat()}...')
+    t_end = t_start
+    t_start = t_end - timedelta(days=1)
+
+    stmt = text(f"""
+    INSERT INTO gridworks.cached_hourly_data
+    WITH hourly_data AS MATERIALIZED (
+        SELECT * FROM gridworks.calc_hourly_data(t_start => '{(t_start - timedelta(hours=1)).isoformat()}', t_end => '{t_end.isoformat()}')
+    )
+    SELECT * FROM hourly_data
+    WHERE time_bucket >= '{t_start.isoformat()}' AND time_bucket < '{t_end.isoformat()}';
+    """)
+
+    db_result = db_session.execute(stmt)
+    db_session.commit()
+    logger.info(f'Inserted {db_result.rowcount} rows')
+

{gw_data-0.3.0 → gw_data-0.3.2}/src/gw_data/db/scripts/seed_data/homes.csv

@@ -4,3 +4,4 @@
 "fir","{""street"": ""7890 Fir Road"", ""city"": ""Millinocket"", ""state"": ""ME"", ""zip"": ""00130"", ""country"": ""USA"", ""latitude"": 45.64500, ""longitude"": -68.72}","{""first_name"": ""Renee"", ""last_name"": ""Fir"", ""phone"": ""207-754-3256"", ""email"": ""reneefir@example.com""}","{""first_name"": ""Kim"", ""last_name"": ""FirFir"", ""phone"": ""207-210-0452"", ""email"": null}","null",3,"hw1.isone.me.versant.keene.fir","{""status"": ""ok"", ""message"": null, ""acked"": null, ""acked_by"": null, ""acked_at"": null}","""NotListeningToAtn""","127.0.0.1","unknown","{""alpha"": 7.5, ""beta"": -0.13, ""gamma"": 0.0022, ""intermediate_power_kw"": 1.5, ""intermediate_rswt"": 94.0, ""dd_power_kw"": 7.5, ""dd_rswt"": 150.0, ""dd_delta_t"": 20.0}","2026-01-23 07:04:32.477627-06"
 "maple","{""street"": ""101 Maple Road"", ""city"": ""Millinocket"", ""state"": ""ME"", ""zip"": ""04462"", ""country"": ""USA"", ""latitude"": 45.64400, ""longitude"": -68.73}","{""first_name"": ""Herman"", ""last_name"": ""Maple"", ""phone"": ""555-444-3333"", ""email"": null}","null","null",4,"hw1.isone.me.versant.keene.maple","{""status"": ""ok"", ""message"": null, ""acked"": null, ""acked_by"": null, ""acked_at"": null}","""NotListeningToAtn""","127.0.0.1","unknown","{""alpha"": 6.3, ""beta"": -0.09, ""gamma"": 0.0007, ""intermediate_power_kw"": 1.5, ""intermediate_rswt"": 118.0, ""dd_power_kw"": 6.3, ""dd_rswt"": 160.0, ""dd_delta_t"": 20.0}","2026-01-23 07:04:32.477627-06"
 "elm","{""street"": ""202 Elm Lane"", ""city"": ""Millinocket"", ""state"": ""ME"", ""zip"": ""04462"", ""country"": ""USA"", ""latitude"": 45.6800, ""longitude"": -68.71}","{""first_name"": ""Mike"", ""last_name"": ""Elm"", ""phone"": ""555-666-7777"", ""email"": null}","{""first_name"": ""Michelle"", ""last_name"": ""Elm"", ""phone"": null, ""email"": null}","null",5,"hw1.isone.me.versant.keene.elm","{""status"": ""ok"", ""message"": null, ""acked"": null, ""acked_by"": null, ""acked_at"": null}","""NotListeningToAtn""","127.0.0.1","unknown","{""alpha"": 7.0, ""beta"": -0.13, ""gamma"": 0.001, ""intermediate_power_kw"": 1.5, ""intermediate_rswt"": 120.0, ""dd_power_kw"": 7.0, ""dd_rswt"": 150.0, ""dd_delta_t"": 20.0}","2026-01-23 07:04:32.477627-06"
+"spruce","{""street"": ""303 Spruce Blvd"", ""city"": ""Millinocket"", ""state"": ""ME"", ""zip"": ""04462"", ""country"": ""USA"", ""latitude"": 45.6200, ""longitude"": -68.70}","{""first_name"": ""Gerald"", ""last_name"": ""Spruce"", ""phone"": ""555-666-8888"", ""email"": null}","{""first_name"": ""Maryann"", ""last_name"": ""Spruce"", ""phone"": null, ""email"": null}","null",5,"hw1.isone.me.versant.keene.spruce","{""status"": ""ok"", ""message"": null, ""acked"": null, ""acked_by"": null, ""acked_at"": null}","""NotListeningToAtn""","127.0.0.1","unknown","{""alpha"": 7.0, ""beta"": -0.13, ""gamma"": 0.001, ""intermediate_power_kw"": 1.5, ""intermediate_rswt"": 120.0, ""dd_power_kw"": 7.0, ""dd_rswt"": 150.0, ""dd_delta_t"": 20.0}","2026-01-23 07:04:32.477627-06"

gw_data-0.3.2/useful-scripts.sql

@@ -0,0 +1,70 @@
+-- Find readings for specific channels, installations, timeframe
+select rc.terminal_asset_alias,rc.name,r.timestamp,r.value
+from gridworks.readings r
+join gridworks.reading_channels rc on r.channel_id = rc.id
+where 1=1
+	and (rc.name like '%whitewire%'	or rc.name like '%heat-call%')
+	and rc.terminal_asset_alias like '%beech%'
+	and timestamp >= '2026-04-20T04:00:00Z'
+	and timestamp <= '2026-04-20T05:00:00Z'
+order by r.timestamp,rc.terminal_asset_alias
+limit 100;
+
+-- Find channels for an installation
+select * from gridworks.reading_channels
+where terminal_asset_alias like '%beech%'
+order by name,terminal_asset_alias;
+
+-- Find messages for an installation & timeframe
+select * from gridworks.messages
+where 1=1
+and from_alias like '%beech%'
+and timestamp >= '2026-04-20T04:00:00Z'
+and timestamp <= '2026-04-20T05:00:00Z'
+limit 100;
+
+-- Total database size
+SELECT pg_size_pretty(pg_database_size('tsdb')) AS size;
+
+-- Display the size of all database tables in order.
+-- This will show each TimescaleDB chunk as a separate table.
+SELECT
+    table_schema || '.' || table_name AS table_full_name,
+    pg_size_pretty(pg_total_relation_size('"' || table_schema || '"."' || table_name || '"')) AS size
+FROM information_schema.tables
+ORDER BY
+    pg_total_relation_size('"' || table_schema || '"."' || table_name || '"') DESC;
+
+-- Display compression stats for each chunk in the readings table
+SELECT 
+    chunk_name,
+    pg_size_pretty(before_compression_total_bytes) AS size_before,
+    pg_size_pretty(after_compression_total_bytes) AS size_after,
+    100 - (after_compression_total_bytes::float / before_compression_total_bytes * 100) AS compression_ratio_pct
+FROM chunk_compression_stats('gridworks.readings');
+
+-- Get info (table, time range, etc.) about the TimescaleDB chunks.
+SELECT * FROM timescaledb_information.chunks
+
+-- Display the size in MB of our two main tables across all its chunks
+SELECT 
+	pg_size_pretty(hypertable_size('gridworks.readings')) as "Readings Table Size", 
+	pg_size_pretty(hypertable_size('gridworks.messages')) as "Messages Table Size";
+
+-- Detailed size for a full hypertable across all its chunks
+SELECT
+    pg_size_pretty(table_bytes) AS data_size,
+    pg_size_pretty(index_bytes) AS index_size,
+    pg_size_pretty(toast_bytes) AS toast_size,
+    pg_size_pretty(total_bytes) AS total_size
+FROM hypertable_detailed_size('gridworks.readings');
+
+-- Info about timescaledb jobs
+select * from timescaledb_information.jobs;
+
+-- Manually run a job (e.g. run compression after an import).
+-- Fill in job_id based on the query above
+CALL run_job(job_id);
+
+
+

{gw_data-0.3.0 → gw_data-0.3.2}/uv.lock

@@ -103,7 +103,7 @@ wheels = [
 
 [[package]]
 name = "gw-data"
-version = "0.3.0"
+version = "0.3.2"
 source = { editable = "." }
 dependencies = [
     { name = "alembic" },

gw_data-0.3.0/postgres_views.md

@@ -1,44 +0,0 @@
-CREATE VIEW readings_pretty AS
-SELECT
-    dc.name AS name,
-    r.value AS value,
-    dc.telemetry_name AS telemetry_name,
-    timestamp AS time,
-    r.message_id AS message_id
-FROM
-    readings r
-JOIN
-    data_channels dc
-ON
-    r.data_channel_id = dc.id;
-
-
-
-SELECT * FROM preadings WHERE name LIKE '%buffer%' LIMIT 5;
-
-
-CREATE OR REPLACE VIEW msg_pretty AS
-SELECT
-    payload, from_alias, message_type_name, persisted_at AS message_persisted, created_at AS message_created
-FROM messages;
-
-
-
-
-SELECT COUNT(*) from messages;
-
-SELECT COUNT(*) FROM messages WHERE message_type_name = 'power.watts';
-
-SELECT from_alias, message_type_name, message_created FROM msg_pretty WHERE message_type_name LIKE '%status%' LIMIT 5;
-
-SELECT pg_size_pretty(pg_database_size('gridworks')) AS size;
-SELECT * FROM msg_pretty WHERE message_type_name LIKE '%param%';
-
-SELECT id, from_alias, message_type_name, persisted_at FROM messages WHERE message_type_name LIKE '%param%';
-
-DELETE FROM messages WHERE id = 'bf06df07-36a5-4e54-ae64-37dea6041ea8';
-
-SELECT MIN(time) AS earliest, MAX(time) AS latest FROM  readings_pretty;
-SELECT MIN(message_created) AS earliest, MAX(message_created) AS latest FROM  msg_pretty;
-
-SELECT from_alias, message_type_name, message_created FROM msg_pretty WHERE message_type_name LIKE '%batched%' LIMIT 5;