gw_data 0.3.0__py3-none-any.whl

gw_data/__init__.py

@@ -0,0 +1,4 @@
+"""Gridworks Data Library"""
+
+def main() -> None:
+    print("Hello from gw_data!")

gw_data/config.py

@@ -0,0 +1,19 @@
+from typing import Optional
+from pydantic import ConfigDict, SecretStr
+from pydantic_settings import BaseSettings
+
+DEFAULT_ENV_FILE = ".env"
+
+class Settings(BaseSettings):
+    db_url: SecretStr = SecretStr(
+        "postgresql+psycopg://gw_admin@localhost:5432/gridworks"
+    )
+    log_level: str = "INFO"
+    log_dir: str = "~/.local/state/gridworks/gw_data/log"
+    db_echo: bool = False
+
+    model_config = ConfigDict(
+        env_prefix="gw_data_",
+        env_nested_delimiter="__",
+        extra="ignore",
+    )

gw_data/db/models/__init__.py

@@ -0,0 +1,37 @@
+"""List of all the models"""
+
+from gw_data.db.models.position_point import PositionPointSql
+from gw_data.db.models.g_node import GNodeSql
+from gw_data.db.models.connectivity_edge import ConnectivityEdgeSql
+
+from gw_data.db.models.customer import CustomerSql
+from gw_data.db.models.installer import InstallerSql
+from gw_data.db.models.user import UserSql
+from gw_data.db.models.installation import InstallationSql
+
+from gw_data.db.models.reading_channel import ReadingChannelSql
+from gw_data.db.models.message import MessageSql
+from gw_data.db.models.reading import ReadingSql
+
+
+# import position_point
+# import g_node
+# import connectivity_edge
+# import customer
+# import installer
+# import user
+# import data_channel
+# import message
+# import reading
+__all__ = [
+    "PositionPointSql",
+    "GNodeSql",
+    "ConnectivityEdgeSql",
+    "CustomerSql",
+    "InstallerSql",
+    "UserSql",
+    "InstallationSql",
+    "ReadingChannelSql",
+    "MessageSql",
+    "ReadingSql"
+]

gw_data/db/models/_base.py

@@ -0,0 +1,5 @@
+from sqlalchemy import MetaData
+from sqlalchemy.orm import DeclarativeBase
+
+class Base(DeclarativeBase):
+    metadata = MetaData(schema="gridworks")

gw_data/db/models/connectivity_edge.py

@@ -0,0 +1,80 @@
+"""
+SQLAlchemy models for the GridNodeRegistry.
+
+Each SQL row corresponds to a serialized ASL GT snapshot.
+ASL types are used for validation (via the codec) before any insert/update.
+"""
+
+from __future__ import annotations
+import uuid
+
+from datetime import datetime
+
+from sqlalchemy import (
+    Uuid,
+    Enum,
+    DateTime,
+    ForeignKey,
+    UniqueConstraint,
+    func,
+)
+from sqlalchemy.orm import (
+    Mapped,
+    mapped_column
+)
+
+from gw_data.db.models._base import Base
+from gw_data.sema.enums import GNodeStatus
+
+
+class ConnectivityEdgeSql(Base):
+    __tablename__ = "connectivity_edges"
+
+    id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True)
+
+    from_g_node_id: Mapped[uuid.UUID] = mapped_column(
+        ForeignKey("g_nodes.id"), index=True
+    )
+    to_g_node_id: Mapped[uuid.UUID] = mapped_column(
+        ForeignKey("g_nodes.id"), index=True
+    )
+
+    status: Mapped[GNodeStatus] = mapped_column(
+        Enum(GNodeStatus, name="connectivity_edge_status", inherit_schema=True)
+    )
+
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), server_default=func.now()
+    )
+
+    __table_args__ = (
+        UniqueConstraint(
+            "from_g_node_id", "to_g_node_id",
+            name="uq_connectivity_edges_from_to"
+        ),
+    )
+
+    # # -------------------
+    # #  ASL ↔ SQL Helpers
+    # # -------------------
+
+    # def to_gt(self) -> ConnectivityEdgeGt:
+    #     return ConnectivityEdgeGt(
+    #         id=self.id,
+    #         from_g_node_id=self.from_g_node_id,
+    #         to_g_node_id=self.to_g_node_id,
+    #         from_g_node_alias=self.from_g_node_alias,
+    #         to_g_node_alias=self.to_g_node_alias,
+    #         status=self.status,
+    #     )
+
+    # @staticmethod
+    # def from_gt(gt: ConnectivityEdgeGt) -> "ConnectivityEdgeSql":
+    #     return ConnectivityEdgeSql(
+    #         id=gt.id,
+    #         from_g_node_id=gt.from_g_node_id,
+    #         to_g_node_id=gt.to_g_node_id,
+    #         from_g_node_alias=gt.from_g_node_alias,
+    #         to_g_node_alias=gt.to_g_node_alias,
+    #         status=gt.status,
+    #     )

gw_data/db/models/customer.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+import uuid
+
+from sqlalchemy import Uuid
+from sqlalchemy.dialects.postgresql import JSONB
+
+from sqlalchemy.orm import (
+    Mapped,
+    mapped_column
+)
+
+from gw_data.db.models._base import Base
+
+class CustomerSql(Base):
+    __tablename__ = "customers"
+    id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True)
+    primary_contact: Mapped[JSONB] = mapped_column(JSONB, nullable=False)
+    secondary_contact: Mapped[JSONB] = mapped_column(JSONB, nullable=False)

gw_data/db/models/g_node.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Optional
+import uuid
+
+from sqlalchemy import (
+    func,
+    Uuid,
+    String,
+    Enum,
+    DateTime,
+    ForeignKey,
+)
+from sqlalchemy.orm import (
+    Mapped,
+    mapped_column,
+    relationship,
+)
+
+from gw_data.db.models._base import Base
+from gw_data.db.models.position_point import PositionPointSql
+from gw_data.sema.enums import BaseGNodeClass, GNodeStatus
+
+class GNodeSql(Base):
+    __tablename__ = "g_nodes"
+
+    id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True)
+    alias: Mapped[str] = mapped_column(String, index=True, unique=True)
+    prev_alias: Mapped[Optional[str]] = mapped_column(String, nullable=True)
+
+    base_class: Mapped[BaseGNodeClass] = mapped_column(
+        Enum(BaseGNodeClass, name="base_g_node_class", inherit_schema=True),
+        nullable=True
+    )
+
+    g_node_class: Mapped[str] = mapped_column(String)
+
+    status: Mapped[GNodeStatus] = mapped_column(
+        Enum(GNodeStatus, name="g_node_status", inherit_schema=True)
+    )
+
+    position_point_id: Mapped[Optional[uuid.UUID]] = mapped_column(
+        ForeignKey("position_points.id"), nullable=True
+    )
+    position_point: Mapped[Optional[PositionPointSql]] = relationship()
+
+    display_name: Mapped[Optional[str]] = mapped_column(String, nullable=True)
+
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), server_default=func.now()
+    )
+
+    # # -------------------
+    # #  ASL ↔ SQL Helpers
+    # # -------------------
+
+    # def to_gt(self) -> GNodeGt:
+    #     """Serialize SQL row → ASL GT."""
+    #     return GNodeGt(
+    #         g_node_id=self.id,
+    #         alias=self.alias,
+    #         base_class=self.base_class,
+    #         g_node_class=self.g_node_class,
+    #         status=self.status,
+    #         prev_alias=self.prev_alias,
+    #         position_point_id=self.position_point_id,
+    #         display_name=self.display_name,
+    #     )
+
+    # @staticmethod
+    # def from_gt(gt: GNodeGt) -> "GNodeSql":
+    #     """Create SQL model from an ASL GT instance (already validated)."""
+    #     return GNodeSql(
+    #         id=gt.g_node_id,
+    #         alias=gt.alias,
+    #         prev_alias=gt.prev_alias,
+    #         base_class=gt.base_class,
+    #         g_node_class=gt.g_node_class,
+    #         status=gt.status,
+    #         position_point_id=gt.position_point_id,
+    #         display_name=gt.display_name,
+    #     )

gw_data/db/models/installation.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+import uuid
+
+from sqlalchemy import (
+    String,
+    Uuid,
+    ForeignKey
+)
+from sqlalchemy.dialects.postgresql import JSONB
+
+from sqlalchemy.orm import (
+    Mapped,
+    mapped_column,
+    relationship
+)
+
+from gw_data.db.models._base import Base
+from gw_data.db.models.customer import CustomerSql
+from gw_data.db.models.g_node import GNodeSql
+from gw_data.db.models.installer import InstallerSql
+
+class InstallationSql(Base):
+    __tablename__ = "installations"
+
+    id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True)
+    g_node_id: Mapped[str] = mapped_column(
+        ForeignKey("g_nodes.id"),
+        nullable=False
+    )
+    g_node: Mapped[GNodeSql] = relationship(lazy="joined")
+
+    display_name: Mapped[str] = mapped_column(String, nullable=False)
+    customer_id: Mapped[uuid.UUID] = mapped_column(
+        ForeignKey("customers.id"),
+        nullable=False
+    )
+    customer: Mapped[CustomerSql] = relationship()
+
+    installer_id: Mapped[uuid.UUID] = mapped_column(
+        ForeignKey("installers.id"),
+        nullable=False
+    )
+    installer: Mapped[InstallerSql] = relationship()
+
+    address: Mapped[dict] = mapped_column(JSONB, nullable=False)
+    alert_status: Mapped[dict] = mapped_column(JSONB, nullable=False)
+    hardware_layout: Mapped[dict] = mapped_column(JSONB, nullable=False)
+    representation_status: Mapped[dict] = mapped_column(JSONB, nullable=True)
+    house_parameters: Mapped[dict] = mapped_column(JSONB, nullable=True)
+    scada_ip_address: Mapped[str] = mapped_column(String, nullable=True)
+    scada_git_commit: Mapped[str] = mapped_column(String, nullable=True)
+

gw_data/db/models/installer.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+import uuid
+
+from sqlalchemy import Uuid
+from sqlalchemy.dialects.postgresql import JSONB
+
+from sqlalchemy.orm import (
+    Mapped,
+    mapped_column,
+)
+
+from gw_data.db.models._base import Base
+
+class InstallerSql(Base):
+    __tablename__ = "installers"
+    id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True)
+    info: Mapped[JSONB] = mapped_column(JSONB, nullable=False)

gw_data/db/models/message.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+from datetime import datetime
+import uuid
+
+from sqlalchemy import (
+    Uuid,
+    String,
+    DateTime,
+    Index
+)
+from sqlalchemy.dialects.postgresql import JSONB
+
+from sqlalchemy.orm import (
+    Mapped,
+    mapped_column,
+)
+
+from gw_data.db.models._base import Base
+
+class MessageSql(Base):
+    __tablename__ = "messages"
+    id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True)
+    timestamp: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, primary_key=True, index=True)
+
+    # This is not a foreign key because we may receive messages from nodes that are not yet in the database
+    from_alias: Mapped[str] = mapped_column(String, nullable=False)
+
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=True)
+    persisted_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+
+    message_type_name: Mapped[str] = mapped_column(String, nullable=False)
+    payload: Mapped[dict] = mapped_column(JSONB, nullable=False)
+
+    __table_args__ = (
+        # TimescaleDB automatically creates this one; we need to include it so Alembic doesn't get confused
+        Index("messages_timestamp_idx", timestamp.desc()),
+        Index(
+            "ix_from_type_message",
+            "from_alias",
+            "message_type_name",
+            "persisted_at"
+        ),
+        {
+            'timescaledb_hypertable': {
+                'time_column_name': 'timestamp',
+            }
+        }
+    )
+
+    def test_fn(self):
+        self.abc.defe = 3

gw_data/db/models/position_point.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from datetime import datetime
+import uuid
+
+from sqlalchemy import (
+    func,
+    Uuid,
+    DateTime,
+)
+from sqlalchemy.orm import (
+    Mapped,
+    mapped_column
+)
+
+from gw_data.db.models._base import Base
+from gw_data.sema.types import PositionPointGt
+
+class PositionPointSql(Base):
+    __tablename__ = "position_points"
+
+    id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True)
+    latitude_micro_deg: Mapped[int] = mapped_column()
+    longitude_micro_deg: Mapped[int] = mapped_column()
+
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), server_default=func.now()
+    )
+
+    def to_gt(self) -> PositionPointGt:
+        """Serialize database row → ASL GT."""
+        return PositionPointGt(
+            id=str(self.id),
+            latitude_micro_deg=self.latitude_micro_deg,
+            longitude_micro_deg=self.longitude_micro_deg,
+        )
+
+    @staticmethod
+    def from_gt(gt: PositionPointGt) -> "PositionPointSql":
+        """Create SQL row from ASL GT after full ASL validation."""
+        return PositionPointSql(
+            id=uuid.UUID(gt.id),
+            latitude_micro_deg=gt.latitude_micro_deg,
+            longitude_micro_deg=gt.longitude_micro_deg,
+        )
+

gw_data/db/models/reading.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+from datetime import datetime
+import uuid
+
+from sqlalchemy import (
+    BigInteger,
+    DateTime,
+    ForeignKey,
+    Index,
+    UniqueConstraint
+)
+
+from sqlalchemy.orm import (
+    Mapped,
+    mapped_column,
+    relationship,
+)
+
+from gw_data.db.models._base import Base
+from gw_data.db.models.reading_channel import ReadingChannelSql
+
+class ReadingSql(Base):
+    __tablename__ = "readings"
+    channel_id: Mapped[uuid.UUID] = mapped_column(
+        ForeignKey("reading_channels.id"),
+        nullable=False
+    )
+    channel: Mapped[ReadingChannelSql] = relationship()
+
+    # This is not a foreign key for two reasons:
+    #   1. TimescaleDB does not allow foreign keys between hypertables
+    #   2. We may conceivably have readings that do not have an associated message
+    #       (e.g., from a server cron job or an admin action)
+    #
+    # Also note that since they are both hypertables, any queries that map between readings and messages 
+    # should correlate based on time to improve performance.
+    message_id: Mapped[uuid.UUID] = mapped_column(
+        nullable=False,
+        index=True
+    )
+    timestamp: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, index=True)
+    value: Mapped[BigInteger] = mapped_column(BigInteger, nullable=False)
+
+    __table_args__ = (
+        # TimescaleDB automatically creates this one; we need to include it so Alembic doesn't get confused
+        Index("readings_timestamp_idx", timestamp.desc()),
+        UniqueConstraint("channel_id", "timestamp", name="readings_channel_id_timestamp_key")
+    )
+
+    # This table does not need a true primary key -- but SQLAlchemy requires one.
+    # So we define a synthetic PK out of two columns that should always be unique. 
+    __mapper_args__ = {
+        "primary_key": [channel_id, timestamp]
+    }
+

gw_data/db/models/reading_channel.py

@@ -0,0 +1,62 @@
+from datetime import datetime
+import uuid
+
+from sqlalchemy import (
+    Index,
+    Uuid,
+    Boolean,
+    String,
+    DateTime,
+    UniqueConstraint
+)
+
+from sqlalchemy.orm import (
+    Mapped,
+    mapped_column,
+)
+
+from gw_data.db.models._base import Base
+
+class ReadingChannelSql(Base):
+    """
+    Reading Channel.
+
+    A reading channel is a source of data for readings, with metadata common to the readings.
+    """
+
+    __tablename__ = "reading_channels"
+    id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True)
+    name: Mapped[str] = mapped_column(String, nullable=False)
+    terminal_asset_alias: Mapped[str] = mapped_column(String, nullable=False)
+    display_name: Mapped[str] = mapped_column(String, nullable=False)
+    unit: Mapped[str] = mapped_column(String, nullable=False)
+    unit_type: Mapped[str] = mapped_column(String, nullable=False)
+    channel_type: Mapped[str] = mapped_column(String, nullable=False)
+    deactivated_date: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=True)
+
+    __table_args__ = (
+        Index("terminal_asset_alias", "name"),
+        UniqueConstraint(
+            "terminal_asset_alias",
+            "name",
+            "deactivated_date",
+            name="unique_name_terminal_asset_deactivated_date",
+            postgresql_nulls_not_distinct=True 
+        )
+    )
+
+    def __repr__(self):
+        active_status = f"deactivated {self.deactivated_date.isoformat()}" if self.self.deactivated_date is not None else "active"
+        return (
+            f"<ReadingChannelSql(name='{self.name}', "
+            f"unit='{self.unit}', terminal asset={self.terminal_asset_alias.split(".")[-2]}, [{active_status}])>"
+            
+        )
+
+    def __str__(self):
+        active_status = f"deactivated {self.deactivated_date.isoformat()}" if self.self.deactivated_date is not None else "active"
+        return (
+            f"ReadingChannelSql(name:{self.name},"
+            f"unit: {self.unit}, terminal asset: {self.terminal_asset_alias.split(".")[-2]}, [{active_status}])"
+        )
+

gw_data/db/models/user.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+from datetime import datetime
+import uuid
+
+from sqlalchemy import (
+    String,
+    DateTime,
+    Boolean,
+    Uuid,
+)
+from sqlalchemy.sql import func
+
+from sqlalchemy.orm import (
+    Mapped,
+    mapped_column,
+    relationship,
+)
+
+from gw_data.db.models._base import Base
+from gw_data.db.models.user_installation_role import UserInstallationRoleSql
+
+class UserSql(Base):
+    __tablename__ = "users"
+    id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True)
+    username: Mapped[str] = mapped_column(String(50), nullable=False, unique=True)
+    hashed_password: Mapped[str] = mapped_column(String(255), nullable=False)
+    is_active: Mapped[bool] = mapped_column(Boolean, default=True)
+    last_login: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+
+    installation_roles: Mapped[list[UserInstallationRoleSql]] = relationship(uselist=True, lazy="joined")
+
+    def __repr__(self) -> str:
+        return f"User(id={self.id!r}, username={self.username!r}, is_active={self.is_active!r})"

gw_data/db/models/user_installation_role.py

@@ -0,0 +1,38 @@
+import uuid
+
+from sqlalchemy import ForeignKey, String, UniqueConstraint
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from gw_data.db.models._base import Base
+from gw_data.db.models.installation import InstallationSql
+
+
+class UserInstallationRoleSql(Base):
+    __tablename__ = "user_installation_roles"
+    user_id: Mapped[uuid.UUID] = mapped_column(
+        ForeignKey("users.id"),
+        nullable=False
+    )
+    role: Mapped[str] = mapped_column(String, nullable=False)
+    installation_id: Mapped[uuid.UUID] = mapped_column(
+        ForeignKey("installations.id"),
+        nullable=True
+    )
+
+    # A null installation_id associates the role to ALL installations (e.g. for admin users)
+    installations: Mapped[list[InstallationSql]] = relationship(
+        "InstallationSql",
+        uselist=True,
+        lazy="joined",
+        primaryjoin="or_(InstallationSql.id==UserInstallationRoleSql.installation_id, UserInstallationRoleSql.installation_id == None)"
+    )
+
+    __table_args__ = (
+        UniqueConstraint("user_id", "installation_id", name="user_installation_roles_user_installation_key"),
+    )
+
+    # This table does not need a true primary key -- but SQLAlchemy requires one.
+    # So we define a synthetic PK out of two columns that should always be unique. 
+    __mapper_args__ = {
+        "primary_key": [user_id, installation_id]
+    }

gw_data/db/scripts/0_db_create.psql

@@ -0,0 +1,4 @@
+-- In prod this will already be created.
+CREATE DATABASE tsdb;
+\c tsdb
+CREATE EXTENSION IF NOT EXISTS timescaledb;

gw_data/db/scripts/1_db_user_setup.psql

@@ -0,0 +1,16 @@
+-- The following needs to be done by a powerful role (like postgresql or tsdbadmin), and doesn't need a specific database connection
+
+-- Basic security config
+REVOKE CONNECT ON DATABASE tsdb FROM PUBLIC;
+
+CREATE USER gw_admin;
+GRANT ALL ON DATABASE tsdb TO gw_admin;
+\password gw_admin
+
+CREATE USER gw_journalkeeper;
+GRANT CONNECT ON DATABASE tsdb TO gw_journalkeeper;
+\password gw_journalkeeper
+
+CREATE USER gw_visualizer;
+GRANT CONNECT ON DATABASE tsdb TO gw_visualizer;
+\password gw_visualizer

gw_data/db/scripts/2_db_schema_setup.sql

@@ -0,0 +1,11 @@
+-- The following should be done by gw_admin while connected to tsdb
+
+CREATE SCHEMA gridworks;
+
+GRANT USAGE ON SCHEMA gridworks TO gw_journalkeeper;
+ALTER DEFAULT PRIVILEGES IN SCHEMA gridworks GRANT SELECT,INSERT,UPDATE,DELETE ON TABLES TO gw_journalkeeper;
+ALTER DEFAULT PRIVILEGES IN SCHEMA gridworks GRANT ALL PRIVILEGES ON SEQUENCES TO gw_journalkeeper;
+
+GRANT USAGE ON SCHEMA gridworks TO gw_visualizer;
+ALTER DEFAULT PRIVILEGES IN SCHEMA gridworks GRANT SELECT ON TABLES TO gw_visualizer;
+ALTER DEFAULT PRIVILEGES IN SCHEMA gridworks GRANT ALL PRIVILEGES ON SEQUENCES TO gw_visualizer;

gw_data/db/scripts/3_db_alembic_upgrade.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+.venv/bin/python3 .venv/bin/alembic upgrade head