gw_data 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gw_data/__init__.py +4 -0
- gw_data/config.py +19 -0
- gw_data/db/__init__.py +0 -0
- gw_data/db/models/__init__.py +37 -0
- gw_data/db/models/_base.py +5 -0
- gw_data/db/models/connectivity_edge.py +80 -0
- gw_data/db/models/customer.py +18 -0
- gw_data/db/models/g_node.py +83 -0
- gw_data/db/models/installation.py +52 -0
- gw_data/db/models/installer.py +17 -0
- gw_data/db/models/message.py +51 -0
- gw_data/db/models/position_point.py +46 -0
- gw_data/db/models/reading.py +55 -0
- gw_data/db/models/reading_channel.py +62 -0
- gw_data/db/models/user.py +35 -0
- gw_data/db/models/user_installation_role.py +38 -0
- gw_data/db/scripts/0_db_create.psql +4 -0
- gw_data/db/scripts/1_db_user_setup.psql +16 -0
- gw_data/db/scripts/2_db_schema_setup.sql +11 -0
- gw_data/db/scripts/3_db_alembic_upgrade.sh +2 -0
- gw_data/db/scripts/4_db_seed.py +129 -0
- gw_data/db/scripts/_XX_drop_all.sql +13 -0
- gw_data/db/scripts/seed_data/homes.csv +6 -0
- gw_data/db/session.py +0 -0
- gw_data/sema/__init__.py +15 -0
- gw_data/sema/base.py +184 -0
- gw_data/sema/codec.py +168 -0
- gw_data/sema/definitions/enums/base.g.node.class/000.yaml +55 -0
- gw_data/sema/definitions/enums/g.node.status/000.yaml +35 -0
- gw_data/sema/definitions/formats/left.right.dot.yaml +27 -0
- gw_data/sema/definitions/formats/uuid4.str.yaml +29 -0
- gw_data/sema/definitions/registry.yaml +77 -0
- gw_data/sema/definitions/types/g.node.gt/004.yaml +169 -0
- gw_data/sema/definitions/types/position.point.gt/000.yaml +86 -0
- gw_data/sema/enums/__init__.py +7 -0
- gw_data/sema/enums/base_g_node_class.py +29 -0
- gw_data/sema/enums/g_node_status.py +28 -0
- gw_data/sema/enums/gw_str_enum.py +97 -0
- gw_data/sema/enums/old_versions/__init__.py +0 -0
- gw_data/sema/indexes/dependency_closure.yaml +19 -0
- gw_data/sema/indexes/local_names.yaml +6 -0
- gw_data/sema/indexes/lookup.yaml +35 -0
- gw_data/sema/indexes/reverse_dependencies.yaml +21 -0
- gw_data/sema/indexes/seed_expanded.yaml +29 -0
- gw_data/sema/indexes/versions.yaml +31 -0
- gw_data/sema/property_format.py +56 -0
- gw_data/sema/tests/test_property_format.py +47 -0
- gw_data/sema/types/__init__.py +7 -0
- gw_data/sema/types/g_node_gt.py +106 -0
- gw_data/sema/types/old_versions/__init__.py +0 -0
- gw_data/sema/types/position_point_gt.py +31 -0
- gw_data-0.3.0.dist-info/METADATA +189 -0
- gw_data-0.3.0.dist-info/RECORD +55 -0
- gw_data-0.3.0.dist-info/WHEEL +4 -0
- gw_data-0.3.0.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import re
|
|
2
|
+
import uuid
|
|
3
|
+
from typing import Annotated
|
|
4
|
+
|
|
5
|
+
from pydantic import BeforeValidator
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
# --- patterns ---
|
|
9
|
+
LEFT_RIGHT_DOT_PATTERN = re.compile(
|
|
10
|
+
r"^[a-z][a-z0-9]*(\.[a-z0-9]+)*$"
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
UUID4_STR_PATTERN = re.compile(
|
|
14
|
+
r"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
# --- methods ---
|
|
19
|
+
def is_left_right_dot(v: str) -> str:
|
|
20
|
+
if not isinstance(v, str):
|
|
21
|
+
raise ValueError(f"<{v}>: LeftRightDot must be a string.")
|
|
22
|
+
|
|
23
|
+
if not LEFT_RIGHT_DOT_PATTERN.fullmatch(v):
|
|
24
|
+
raise ValueError(f"<{v}>: Fails LeftRightDot format.")
|
|
25
|
+
|
|
26
|
+
return v
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def is_uuid4_str(v: str) -> str:
|
|
30
|
+
if not isinstance(v, str):
|
|
31
|
+
raise ValueError(f"<{v}>: uuid4.str must be a string.")
|
|
32
|
+
|
|
33
|
+
if not UUID4_STR_PATTERN.fullmatch(v):
|
|
34
|
+
raise ValueError(f"<{v}>: Fails uuid4.str format.")
|
|
35
|
+
|
|
36
|
+
try:
|
|
37
|
+
u = uuid.UUID(v)
|
|
38
|
+
except Exception as e:
|
|
39
|
+
raise ValueError(f"Invalid UUID4: {v} <{e}>") from e
|
|
40
|
+
if u.version != 4:
|
|
41
|
+
raise ValueError(
|
|
42
|
+
f"{v} is valid uid, but of version {u.version}. Fails UuidCanonicalTextual"
|
|
43
|
+
)
|
|
44
|
+
return str(u)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
# --- annotated types ---
|
|
48
|
+
LeftRightDot = Annotated[
|
|
49
|
+
str,
|
|
50
|
+
BeforeValidator(is_left_right_dot),
|
|
51
|
+
]
|
|
52
|
+
|
|
53
|
+
UUID4Str = Annotated[
|
|
54
|
+
str,
|
|
55
|
+
BeforeValidator(is_uuid4_str),
|
|
56
|
+
]
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
import pytest
|
|
5
|
+
import yaml
|
|
6
|
+
from pydantic import TypeAdapter, ValidationError
|
|
7
|
+
|
|
8
|
+
from gw_data.sema import property_format
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
PACKAGE_ROOT = Path(__file__).resolve().parents[1]
|
|
12
|
+
DEFINITIONS_DIR = PACKAGE_ROOT / "definitions"
|
|
13
|
+
FORMAT_SCHEMA_PATHS = [
|
|
14
|
+
DEFINITIONS_DIR / "formats" / "left.right.dot.yaml",
|
|
15
|
+
DEFINITIONS_DIR / "formats" / "uuid4.str.yaml",
|
|
16
|
+
]
|
|
17
|
+
RUNTIME_FORMAT_TYPES: dict[str, Any] = {
|
|
18
|
+
"left.right.dot": property_format.LeftRightDot,
|
|
19
|
+
"uuid4.str": property_format.UUID4Str,
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def load_yaml(path: Path) -> dict[str, Any]:
|
|
24
|
+
return yaml.safe_load(path.read_text())
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def format_adapter(format_name: str) -> TypeAdapter:
|
|
28
|
+
return TypeAdapter(RUNTIME_FORMAT_TYPES[format_name])
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@pytest.mark.parametrize("schema_path", FORMAT_SCHEMA_PATHS)
|
|
32
|
+
def test_property_format_schema_examples_validate(schema_path: Path) -> None:
|
|
33
|
+
schema = load_yaml(schema_path)
|
|
34
|
+
adapter = format_adapter(schema["title"])
|
|
35
|
+
|
|
36
|
+
for example in schema.get("examples", []):
|
|
37
|
+
assert adapter.validate_python(example) == example
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@pytest.mark.parametrize("schema_path", FORMAT_SCHEMA_PATHS)
|
|
41
|
+
def test_property_format_schema_counterexamples_fail(schema_path: Path) -> None:
|
|
42
|
+
schema = load_yaml(schema_path)
|
|
43
|
+
adapter = format_adapter(schema["title"])
|
|
44
|
+
|
|
45
|
+
for counterexample in schema.get("counterexamples", []):
|
|
46
|
+
with pytest.raises((TypeError, ValueError, ValidationError)):
|
|
47
|
+
adapter.validate_python(counterexample)
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
from typing import Literal, Self
|
|
2
|
+
from pydantic import model_validator
|
|
3
|
+
from gw_data.sema.base import SemaType
|
|
4
|
+
from gw_data.sema.enums import BaseGNodeClass
|
|
5
|
+
from gw_data.sema.enums import GNodeStatus
|
|
6
|
+
from gw_data.sema.property_format import LeftRightDot
|
|
7
|
+
from gw_data.sema.property_format import UUID4Str
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class GNodeGt(SemaType):
|
|
11
|
+
"""Sema: https://schemas.electricity.works/types/g.node.gt/004"""
|
|
12
|
+
|
|
13
|
+
g_node_id: UUID4Str
|
|
14
|
+
alias: LeftRightDot
|
|
15
|
+
base_class: BaseGNodeClass
|
|
16
|
+
g_node_class: str
|
|
17
|
+
status: GNodeStatus
|
|
18
|
+
prev_alias: LeftRightDot | None = None
|
|
19
|
+
position_point_id: UUID4Str | None = None
|
|
20
|
+
display_name: str | None = None
|
|
21
|
+
type_name: Literal["g.node.gt"] = "g.node.gt"
|
|
22
|
+
version: Literal["004"] = "004"
|
|
23
|
+
|
|
24
|
+
@model_validator(mode="after")
|
|
25
|
+
def check_axiom_1(self) -> Self:
|
|
26
|
+
"""
|
|
27
|
+
Axiom 1: ClassConsistency
|
|
28
|
+
a. If BaseClass is not Logical, GNodeClass SHALL equal the string value of BaseClass.
|
|
29
|
+
b. If BaseClass is Logical, GNodeClass SHALL NOT equal any value of base.g.node.class other than Logical.
|
|
30
|
+
"""
|
|
31
|
+
if self.base_class != BaseGNodeClass.Logical:
|
|
32
|
+
if self.g_node_class != self.base_class.value:
|
|
33
|
+
raise ValueError(
|
|
34
|
+
f"Axiom 1 failed: Physical GNodes must align BaseClass and GNodeClass. "
|
|
35
|
+
f"Expected GNodeClass='{self.base_class.value}' "
|
|
36
|
+
f"but got '{self.g_node_class}'."
|
|
37
|
+
)
|
|
38
|
+
elif (
|
|
39
|
+
self.g_node_class in BaseGNodeClass.values()
|
|
40
|
+
and self.g_node_class != BaseGNodeClass.Logical.value
|
|
41
|
+
):
|
|
42
|
+
raise ValueError(
|
|
43
|
+
"Axiom 1 failed: Logical GNodes must not use a non-Logical "
|
|
44
|
+
"base.g.node.class value as GNodeClass."
|
|
45
|
+
)
|
|
46
|
+
return self
|
|
47
|
+
|
|
48
|
+
@model_validator(mode="after")
|
|
49
|
+
def check_axiom_2(self) -> Self:
|
|
50
|
+
"""
|
|
51
|
+
Axiom 2: PhysicalGNodeLocations
|
|
52
|
+
If BaseClass != Logical, PositionPointId SHALL NOT be null.
|
|
53
|
+
"""
|
|
54
|
+
if self.base_class != BaseGNodeClass.Logical:
|
|
55
|
+
if self.position_point_id is None:
|
|
56
|
+
raise ValueError(
|
|
57
|
+
"Axiom 2 failed: Physical GNodes must have a PositionPointId. "
|
|
58
|
+
f"BaseClass='{self.base_class.value}' has no location."
|
|
59
|
+
)
|
|
60
|
+
return self
|
|
61
|
+
|
|
62
|
+
@model_validator(mode="after")
|
|
63
|
+
def check_axiom_3(self) -> Self:
|
|
64
|
+
"""
|
|
65
|
+
Axiom 3: AliasTransitionConsistency
|
|
66
|
+
If PrevAlias is not null, it SHALL differ from Alias.
|
|
67
|
+
"""
|
|
68
|
+
if self.prev_alias is not None and self.prev_alias == self.alias:
|
|
69
|
+
raise ValueError(
|
|
70
|
+
"Axiom 3 failed: PrevAlias must differ from Alias when present."
|
|
71
|
+
)
|
|
72
|
+
return self
|
|
73
|
+
|
|
74
|
+
@model_validator(mode="after")
|
|
75
|
+
def check_axiom_4(self) -> Self:
|
|
76
|
+
"""
|
|
77
|
+
Axiom 4: GNodeClassNamespacing
|
|
78
|
+
GNodeClass SHALL be a non-empty string containing no whitespace
|
|
79
|
+
"""
|
|
80
|
+
if not self.g_node_class:
|
|
81
|
+
raise ValueError("Axiom 4 failed: GNodeClass must be non-empty.")
|
|
82
|
+
if any(char.isspace() for char in self.g_node_class):
|
|
83
|
+
raise ValueError("Axiom 4 failed: GNodeClass must not contain whitespace.")
|
|
84
|
+
return self
|
|
85
|
+
|
|
86
|
+
@model_validator(mode="after")
|
|
87
|
+
def check_axiom_5(self) -> Self:
|
|
88
|
+
"""
|
|
89
|
+
Axiom 5: AliasSuffixSemantics
|
|
90
|
+
a. Alias SHALL end with ".ta" if and only if GNodeClass is "TerminalAsset".
|
|
91
|
+
b. Alias SHALL end with ".scada" if and only if GNodeClass is "Scada".
|
|
92
|
+
"""
|
|
93
|
+
alias_has_ta_suffix = self.alias.endswith(".ta")
|
|
94
|
+
if (self.g_node_class == "TerminalAsset") != alias_has_ta_suffix:
|
|
95
|
+
raise ValueError(
|
|
96
|
+
'Axiom 5 failed: Alias must end with ".ta" if and only if '
|
|
97
|
+
'GNodeClass is "TerminalAsset".'
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
alias_has_scada_suffix = self.alias.endswith(".scada")
|
|
101
|
+
if (self.g_node_class == "Scada") != alias_has_scada_suffix:
|
|
102
|
+
raise ValueError(
|
|
103
|
+
'Axiom 5 failed: Alias must end with ".scada" if and only if '
|
|
104
|
+
'GNodeClass is "Scada".'
|
|
105
|
+
)
|
|
106
|
+
return self
|
|
File without changes
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
from typing import Literal, Self
|
|
2
|
+
from pydantic import StrictInt, model_validator
|
|
3
|
+
from gw_data.sema.base import SemaType
|
|
4
|
+
from gw_data.sema.property_format import UUID4Str
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class PositionPointGt(SemaType):
|
|
8
|
+
"""Sema: https://schemas.electricity.works/types/position.point.gt/000"""
|
|
9
|
+
|
|
10
|
+
id: UUID4Str
|
|
11
|
+
latitude_micro_deg: StrictInt
|
|
12
|
+
longitude_micro_deg: StrictInt
|
|
13
|
+
type_name: Literal["position.point.gt"] = "position.point.gt"
|
|
14
|
+
version: Literal["000"] = "000"
|
|
15
|
+
|
|
16
|
+
@model_validator(mode="after")
|
|
17
|
+
def check_axiom_1(self) -> Self:
|
|
18
|
+
"""
|
|
19
|
+
Axiom 1: ValidEarthCoordinates
|
|
20
|
+
LatitudeMicroDeg SHALL be between -90,000,000 and 90,000,000 inclusive.
|
|
21
|
+
LongitudeMicroDeg SHALL be between -180,000,000 and 180,000,000 inclusive.
|
|
22
|
+
"""
|
|
23
|
+
if not -90_000_000 <= self.latitude_micro_deg <= 90_000_000:
|
|
24
|
+
raise ValueError(
|
|
25
|
+
f"Latitude {self.latitude_micro_deg / 1_000_000}° out of range [-90, 90]"
|
|
26
|
+
)
|
|
27
|
+
if not -180_000_000 <= self.longitude_micro_deg <= 180_000_000:
|
|
28
|
+
raise ValueError(
|
|
29
|
+
f"Longitude {self.longitude_micro_deg / 1_000_000}° out of range [-180, 180]"
|
|
30
|
+
)
|
|
31
|
+
return self
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: gw_data
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Gridworks Data Infrastructure
|
|
5
|
+
Author-email: Jessica Millar <jessica.lynn.millar@gmail.com>
|
|
6
|
+
Requires-Python: <3.13.0,>=3.12
|
|
7
|
+
Requires-Dist: alembic>=1.17.2
|
|
8
|
+
Requires-Dist: fastapi>=0.123.0
|
|
9
|
+
Requires-Dist: psycopg[binary]>=3.1
|
|
10
|
+
Requires-Dist: pydantic-settings>=2.12.0
|
|
11
|
+
Requires-Dist: sqlalchemy-timescaledb>=0.4.1
|
|
12
|
+
Requires-Dist: sqlalchemy>=2.0.44
|
|
13
|
+
Requires-Dist: uvicorn[standard]>=0.38.0
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
|
|
16
|
+
# Gridworks Data
|
|
17
|
+
|
|
18
|
+
This project contains code related to working with databases in the GridWorks ecosystem.
|
|
19
|
+
|
|
20
|
+
Our platform is PostgreSQL, with the TimescaleDB (+toolkit) extensions for efficiently handling time-series data.
|
|
21
|
+
|
|
22
|
+
## Prerequisites
|
|
23
|
+
|
|
24
|
+
* **Docker Engine** — to run PostgreSQL+TimescaleDB locally.
|
|
25
|
+
(If you have previously set up the RabbitMQ server from `gridworks-base`, you should already have Docker.)
|
|
26
|
+
* **`psql`** PostgreSQL command-line client (any modern version; tested with 14.x and 18.x).
|
|
27
|
+
Install via your package manager (e.g. `brew install libpq && brew link --force libpq` on macOS, `sudo apt install postgresql-client` on Linux). Check with `psql --version`.
|
|
28
|
+
*(Optional: `pgAdmin` as a GUI for inspecting the database.)*
|
|
29
|
+
* **Python ≥ 3.12** and `uv` (per the rest of the GridWorks toolchain).
|
|
30
|
+
|
|
31
|
+
## Database Setup
|
|
32
|
+
|
|
33
|
+
The following steps will get you set up to run with the database, either locally or on a managed Tiger Cloud instance.
|
|
34
|
+
|
|
35
|
+
### Local Pre-requisite: PostgreSQL+TimescaleDB container
|
|
36
|
+
|
|
37
|
+
If running locally, you'll need to start by pulling the official TimescaleDB image and starting a container. **You need the `-ha` variant** for PostgreSQL v18 and TimescaleDB v2.25 — without `-ha` you'll get the "lite" version of TimescaleDB which is missing required functionality.
|
|
38
|
+
|
|
39
|
+
If you don't already have a PostgreSQL listener on port 5432, map `5432:5432`. If you do (native install, or another Docker postgres), pick a free port and map `<HOST_PORT>:5432` instead (e.g. `5433:5432`).
|
|
40
|
+
|
|
41
|
+
Pick a `POSTGRES_PASSWORD` — this becomes the password for the built-in `postgres` superuser; record it locally (e.g. in `.env`), don't commit it.
|
|
42
|
+
|
|
43
|
+
Concrete `docker run` example (uses 5433 and a placeholder password):
|
|
44
|
+
|
|
45
|
+
docker run -d \
|
|
46
|
+
--name gw-data-pg \
|
|
47
|
+
-e POSTGRES_PASSWORD=changeme \
|
|
48
|
+
-p 5433:5432 \
|
|
49
|
+
timescale/timescaledb-ha:pg18-ts2.25
|
|
50
|
+
|
|
51
|
+
Verify the container is healthy: `docker ps` should show it up, and `PGPASSWORD=changeme psql -h 127.0.0.1 -p 5433 -U postgres -c "SELECT version();"` should return PostgreSQL 18.x.
|
|
52
|
+
|
|
53
|
+
Detailed instructions and alternative configurations: https://www.tigerdata.com/docs/self-hosted/latest/install/installation-docker
|
|
54
|
+
|
|
55
|
+
### Database Setup
|
|
56
|
+
|
|
57
|
+
This repo includes a numbered sequence of scripts in the `src/gw_data/db/scripts` folder that will get you up and running.
|
|
58
|
+
|
|
59
|
+
#### 0. (Local-Only) Create the Database
|
|
60
|
+
|
|
61
|
+
This step is not required (or even possible) on a Tiger Cloud managed server.
|
|
62
|
+
|
|
63
|
+
Run `0_db_create.sql` as the `postgres` user to create the `tsdb` database and add the `timescaledb` extension.
|
|
64
|
+
|
|
65
|
+
psql -d "postgresql://127.0.0.1:<PORT>/" -U postgres -f src/gw_data/db/scripts/0_server_init.psql
|
|
66
|
+
|
|
67
|
+
* Replace `<PORT>` with the host port you mapped to 5432 (e.g. `5433`).
|
|
68
|
+
* You'll be prompted for the `postgres` user password (i.e., the one you previously set as `POSTGRES_PASSWORD` in your `docker run` command).
|
|
69
|
+
|
|
70
|
+
#### 1. Create the Users
|
|
71
|
+
|
|
72
|
+
Run `1_db_user_setup.psql` as follows to create the users we need. This script can be run as the `postgres` user locally, or the `tsdbadmin` user in Tiger Cloud.
|
|
73
|
+
|
|
74
|
+
psql -d "postgresql://<SERVER>:<PORT>/" -U <postgres|tsdbadmin> -f src/gw_data/db/scripts/1_db_user_setup.psql
|
|
75
|
+
|
|
76
|
+
* Replace `<SERVER>` with the database server address (e.g., `127.0.0.1` when running locally).
|
|
77
|
+
* Again, replace `<PORT>` with your mapped host port.
|
|
78
|
+
* Again, you'll be prompted for the `postgres` user password.
|
|
79
|
+
|
|
80
|
+
This script creates three user roles: `gw_admin` (full ownership), `gw_journalkeeper` (insert/update/delete), and `gw_visualizer` (select-only).
|
|
81
|
+
|
|
82
|
+
**Note: the script uses `psql`'s interactive `\password` meta-command three times** (once for each user). It must be run **interactively** — it will prompt for each password as it runs and cannot be piped or run via CI:
|
|
83
|
+
|
|
84
|
+
You'll be prompted for new passwords for each of the users. Pick whatever you like and record them somewhere.
|
|
85
|
+
|
|
86
|
+
For non-interactive setups (CI, scripted bootstraps), apply the equivalent SQL with explicit passwords; see the script as the canonical reference.
|
|
87
|
+
|
|
88
|
+
**Reset shortcut:** if your local DB ever gets into a weird state, you can wipe it and start over with:
|
|
89
|
+
|
|
90
|
+
psql -d "postgresql://127.0.0.1:<PORT>/" -U postgres -f src/gw_data/db/scripts/_XX_drop_all.sql
|
|
91
|
+
|
|
92
|
+
#### 2. Create the `gridworks` Schema
|
|
93
|
+
|
|
94
|
+
Run `2_db_schema_setup.sql` as follows to create our private `gridworks` schema and apply appropriate permissions. This should be run as the `gw_admin` user.
|
|
95
|
+
|
|
96
|
+
psql -d "postgresql://<SERVER>:<PORT>/" -U gw_admin -f src/gw_data/db/scripts/2_db_schema_setup.psql
|
|
97
|
+
|
|
98
|
+
* Again, replace `<SERVER>` with the database server address.
|
|
99
|
+
* Again, replace `<PORT>` with your mapped host port.
|
|
100
|
+
* This time you'll be prompted for the `gw_admin` user password.
|
|
101
|
+
|
|
102
|
+
#### 3. Run the Alembic Migrations
|
|
103
|
+
|
|
104
|
+
Next we need to run the database migrations we've defined with Alembic to create the tables, etc. that we need.
|
|
105
|
+
|
|
106
|
+
But first we need to update our .env file. Copy `template.env` (at the repo root) to `.env`:
|
|
107
|
+
|
|
108
|
+
cp template.env .env
|
|
109
|
+
|
|
110
|
+
Then, edit `.env` as follows:
|
|
111
|
+
* Replace `<SERVER>` with the database server address (e.g., `127.0.0.1` when running locally).
|
|
112
|
+
* Replace `<%PASSWORD%>` in `GW_DATA_DB_URL` with the `gw_admin` password you just set.
|
|
113
|
+
* Replace `<%PORT%>` with the host port you mapped to 5432 (e.g. `5433`).
|
|
114
|
+
|
|
115
|
+
Now we can run `3_db_alembic_upgrade.sh` as follows:
|
|
116
|
+
|
|
117
|
+
`sh src/gw_data/db/scripts/3_db_alembic_upgrade.sh`
|
|
118
|
+
|
|
119
|
+
This should create 12 tables in the `gridworks` schema: `alembic_version`, `connectivity_edges`, `customers`, `g_nodes`, `installations`, `installers`, `messages`, `position_points`, `reading_channels`, `readings`, `user_installation_roles`, `users`.
|
|
120
|
+
|
|
121
|
+
Verify with `psql -d "postgresql://<SERVER>:<PORT>/tsdb" -U gw_admin -c "\dt gridworks.*"`.
|
|
122
|
+
|
|
123
|
+
#### 4. Seed the Database
|
|
124
|
+
|
|
125
|
+
With the database created, `gw_admin` ready, and `.env` filled in, you can seed some initial data:
|
|
126
|
+
|
|
127
|
+
uv run python ./src/gw_data/db/scripts/1_db_seed.py
|
|
128
|
+
|
|
129
|
+
**Note: the seed script is also interactive** — it uses Python's `getpass` to prompt for passwords for two seeded users (`admin` and `beech-user`). It must be run from a real terminal (no piping). The seed populates a small set of dev users, a customer, an installation, and one g_node.
|
|
130
|
+
|
|
131
|
+
In the future we will have a more comprehensive seeding process that will ingest some actual message data.
|
|
132
|
+
|
|
133
|
+
## Best Practices for Databases
|
|
134
|
+
|
|
135
|
+
The following are some best practices that we should follow when at all possible:
|
|
136
|
+
|
|
137
|
+
* Primary Keys should be UUIDs, stored as the DB-native UUID type
|
|
138
|
+
* Dates/Times should be stored as `TIMESTAMPTZ`
|
|
139
|
+
* Migrations should be encouraged and done frequently to provide new functionality. Database schema is always temporary.
|
|
140
|
+
* Each application that uses that database should have its own dedicated user, with the minimal set of permissions. (In particular, `postgres` should never be used at all, and `gw_admin` should only be used for migrations and other tasks that require full ownership of the database.)
|
|
141
|
+
|
|
142
|
+
## TimescaleDB Performance
|
|
143
|
+
|
|
144
|
+
TimescaleDB does two main things to improve performance:
|
|
145
|
+
1. Separates time-series tables (which it calls "hypertables") into "chunks", each of which cover a certain timeframe.
|
|
146
|
+
2. Allows time-series data to be stored in column order with compression, which vastly improves performance for data that changes very little over time. This compression happens in a scheduled job, and only for data older than a configured threshold.
|
|
147
|
+
|
|
148
|
+
We have column-store compression enabled on the readings table for data older than 2 weeks, with compression segmented by the channel ID.
|
|
149
|
+
|
|
150
|
+
### Useful Queries
|
|
151
|
+
|
|
152
|
+
The following queries are useful for analyzing TimescaleDB performance:
|
|
153
|
+
|
|
154
|
+
```
|
|
155
|
+
-- Display the size in MB of our two main tables
|
|
156
|
+
SELECT pg_size_pretty(hypertable_size('readings')) as "Readings Table Size", pg_size_pretty(hypertable_size('messages')) as "Messages Table Size";
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
```
|
|
160
|
+
-- Display the size of all database tables in order.
|
|
161
|
+
-- This will show each TimescaleDB chunk as a separate table.
|
|
162
|
+
SELECT
|
|
163
|
+
table_schema || '.' || table_name AS table_full_name,
|
|
164
|
+
pg_size_pretty(pg_total_relation_size('"' || table_schema || '"."' || table_name || '"')) AS size
|
|
165
|
+
FROM information_schema.tables
|
|
166
|
+
ORDER BY
|
|
167
|
+
pg_total_relation_size('"' || table_schema || '"."' || table_name || '"') DESC;
|
|
168
|
+
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
```
|
|
172
|
+
-- Display compression stats for each chunk in the readings table
|
|
173
|
+
SELECT
|
|
174
|
+
chunk_name,
|
|
175
|
+
pg_size_pretty(before_compression_total_bytes) AS size_before,
|
|
176
|
+
pg_size_pretty(after_compression_total_bytes) AS size_after,
|
|
177
|
+
100 - (after_compression_total_bytes::float / before_compression_total_bytes * 100) AS compression_ratio_pct
|
|
178
|
+
FROM chunk_compression_stats('readings');
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
```
|
|
182
|
+
-- Get the ID of the policy_compression job so you can manually run `CALL run_job` with it (e.g. after a bulk import).
|
|
183
|
+
SELECT job_id, proc_name, hypertable_name, config
|
|
184
|
+
FROM timescaledb_information.jobs;
|
|
185
|
+
```
|
|
186
|
+
```
|
|
187
|
+
-- Get info (table, time range, etc.) about the TimescaleDB chunks.
|
|
188
|
+
SELECT * FROM timescaledb_information.chunks
|
|
189
|
+
```
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
gw_data/__init__.py,sha256=-7-KRmuqK2V9knk1lhmA14zxCaGkPh3bn3R4fY_KCVQ,83
|
|
2
|
+
gw_data/config.py,sha256=AKSIR8VAkAkpjDpTYZXeTpWMF6m3pNeMqKxyDQM6Mbs,517
|
|
3
|
+
gw_data/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
+
gw_data/db/session.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
|
+
gw_data/db/models/__init__.py,sha256=AXAOpY2Lb6aI6q7ih4ja2wq269Ueur-zrNaqLIZtI5U,970
|
|
6
|
+
gw_data/db/models/_base.py,sha256=fOkXv_2UayrGDivGKC_YBgceIe4_eXrZagP39sdyyCg,149
|
|
7
|
+
gw_data/db/models/connectivity_edge.py,sha256=BnJohJN5DQo-_77R6rP58lqn0RAmxyjeEPoo26FE7BQ,2125
|
|
8
|
+
gw_data/db/models/customer.py,sha256=2oNiHCiWzzxe2MeVob0WJfaowAVi7KuWmuOXgxXhm1o,503
|
|
9
|
+
gw_data/db/models/g_node.py,sha256=449nA1tUGqCVgTMD8XEmSZGYLgCzQXhuE0-gslgDT0g,2494
|
|
10
|
+
gw_data/db/models/installation.py,sha256=tVlKHmuycbKB8ikz_TqV1uI0fYxzYwLnCOPiKXYnWng,1651
|
|
11
|
+
gw_data/db/models/installer.py,sha256=RTgrSigB3CmIn4G96aGLxt6av0-wuKT2FZoe5AdpMCI,419
|
|
12
|
+
gw_data/db/models/message.py,sha256=Y95eot3PCDS-3hJSY2zyth4YXCw99v97SM0g2kboJtM,1564
|
|
13
|
+
gw_data/db/models/position_point.py,sha256=To-MW8nL--OjUQoSLlwfhBlZ17UnvMe-hfu7DBopb-A,1260
|
|
14
|
+
gw_data/db/models/reading.py,sha256=Py63YJUYMiaBCYzN0VlrWxWXEd5-Jh5VSSd79aKGTB8,1855
|
|
15
|
+
gw_data/db/models/reading_channel.py,sha256=LCFUbg7WiBuVtho_8b2e5iyaEbAwAkPutFq2_GLt6HE,2055
|
|
16
|
+
gw_data/db/models/user.py,sha256=iUubEAgKmOaCH6gGnucby0fYepjyw3MpT-vAJ1rinzg,1278
|
|
17
|
+
gw_data/db/models/user_installation_role.py,sha256=zg955VQ8ve-zWKoUB2Dfd4J4Gjq-PAwFKYz0BCHNjsY,1377
|
|
18
|
+
gw_data/db/scripts/0_db_create.psql,sha256=_wrqfVQvFlvCwjBl1y71W9O5WSY0TkVuiDq5Se5ASgc,114
|
|
19
|
+
gw_data/db/scripts/1_db_user_setup.psql,sha256=__pukcQPFJVr7EpRgyIi8RsN7mW-Yv03SmPVu6dxqZs,497
|
|
20
|
+
gw_data/db/scripts/2_db_schema_setup.sql,sha256=zxR4ydpoE2Cz_8qtYKh1xGLPuye1SWMR4GwuisOp-bo,592
|
|
21
|
+
gw_data/db/scripts/3_db_alembic_upgrade.sh,sha256=CHidb3tTgi2A1Rbvm82kHEKxSUKCQGrkfufaC5_Uql8,58
|
|
22
|
+
gw_data/db/scripts/4_db_seed.py,sha256=2lH4FF4wv27TTKBeZBA4imiR9qoLJOoKxNPUeYKn1u4,4159
|
|
23
|
+
gw_data/db/scripts/_XX_drop_all.sql,sha256=0jgtgbwoGGrXXDuxTsLRFDqse8oUieEcpVNBiogo6Ik,335
|
|
24
|
+
gw_data/db/scripts/seed_data/homes.csv,sha256=Gwd5LyeIPehQpxXYvymh-a4R_Zl7EewQU5xc2vSmMvE,3925
|
|
25
|
+
gw_data/sema/__init__.py,sha256=CXGduMLuY_hLIdK5DIbrkhVTk5aoIynw-pLrpGWe2F4,226
|
|
26
|
+
gw_data/sema/base.py,sha256=SJa48J-jjP2g6JNbfVE65nOzYE4jbqTV32KuiMVKoNg,5656
|
|
27
|
+
gw_data/sema/codec.py,sha256=bYfxp7sCHw2vY3vjNg65kAyiJD13w1MB4hJscwIlinE,5042
|
|
28
|
+
gw_data/sema/property_format.py,sha256=d1_FCA6IRYLBTC7OS2LrPQ7XuOdoeJxihTHSCRdZiE4,1278
|
|
29
|
+
gw_data/sema/definitions/registry.yaml,sha256=DWJAVnShj4Cj3pb0hpgRhjRrtdbAxYtJ513JU57RqQU,2371
|
|
30
|
+
gw_data/sema/definitions/enums/base.g.node.class/000.yaml,sha256=POg5Lhx1xW6W_3ln0BuvsEXftNOQb-AwUQyPMXSMZsU,2442
|
|
31
|
+
gw_data/sema/definitions/enums/g.node.status/000.yaml,sha256=LXZOBfDEWBPSCWa98gYMNYwfXHhZvinMJA058qDsM0M,990
|
|
32
|
+
gw_data/sema/definitions/formats/left.right.dot.yaml,sha256=CV2maPQOFL2AauBMVu7Q67svccjM-y2CSIIm5z2QGs8,915
|
|
33
|
+
gw_data/sema/definitions/formats/uuid4.str.yaml,sha256=8GOl3d3458yA8S8u_XuMtIghP_QpCjGTZiSDA32MXK8,1116
|
|
34
|
+
gw_data/sema/definitions/types/g.node.gt/004.yaml,sha256=8Cct3vUkSBo4zG_STZRQ9Nt2Zv1CrQIyRs9H4SvUJ68,5990
|
|
35
|
+
gw_data/sema/definitions/types/position.point.gt/000.yaml,sha256=sAp-WMC8pPIRdIqBKSeOtK0CsjUSPFIURBEGVb_tRUI,2976
|
|
36
|
+
gw_data/sema/enums/__init__.py,sha256=Mq56Pns7CFzY8X2kdRmdHydM9K9eFCTiPaGnsynIAtc,177
|
|
37
|
+
gw_data/sema/enums/base_g_node_class.py,sha256=t3JdII3YEGMtSfGyr067iQP8SZnvEdsXy_n9QZGQ_jk,662
|
|
38
|
+
gw_data/sema/enums/g_node_status.py,sha256=EBjTbZF-HOem8KOFM5gs8viZ5iP-n0r_DR6gtsByghI,612
|
|
39
|
+
gw_data/sema/enums/gw_str_enum.py,sha256=tVPX7GYLBQUKeRbZJpYN1hX9z8zs0LvuhrCVdMKHDQg,2486
|
|
40
|
+
gw_data/sema/enums/old_versions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
41
|
+
gw_data/sema/indexes/dependency_closure.yaml,sha256=c7O3DEPZfPc8nKltdc7AoDamxJsww0iVR7BFG39yfio,344
|
|
42
|
+
gw_data/sema/indexes/local_names.yaml,sha256=hllGKsdQdt7Jvme2Sn0KnYYBX1jTdwmg51l2P0atIBw,146
|
|
43
|
+
gw_data/sema/indexes/lookup.yaml,sha256=sKvFa3bimhh_s5XqKju84qQdRc2m9wTReTFYJK8iEdA,1054
|
|
44
|
+
gw_data/sema/indexes/reverse_dependencies.yaml,sha256=lou8E9J22bXV4H9fO6m_EbH4oipMpJCg5cajqhzslK4,360
|
|
45
|
+
gw_data/sema/indexes/seed_expanded.yaml,sha256=uGE_Mc1wyQWK7f75MQApPt1MMIsTNBqPtpqIBnWS7y4,772
|
|
46
|
+
gw_data/sema/indexes/versions.yaml,sha256=-C_BCttuPiYW1FoopjYfSmxXnbghyEqYb9BWCgyGXgg,703
|
|
47
|
+
gw_data/sema/tests/test_property_format.py,sha256=zuEFL_c13vkWOIQOjX_DYmALqrydxivt7x99K-8RXi4,1484
|
|
48
|
+
gw_data/sema/types/__init__.py,sha256=KedWRca6ffpOZJ_a9qs68C1T0OQAEdKzqRPARL14TIA,167
|
|
49
|
+
gw_data/sema/types/g_node_gt.py,sha256=MBLTppzjfsKtC-dAuXrlKi_TnpKyDMcWEGDowX8Iw4k,4171
|
|
50
|
+
gw_data/sema/types/position_point_gt.py,sha256=9_bysVuJw0g4QkmDVRkmnCwFox5H5gG6DopkQ8JanQ8,1208
|
|
51
|
+
gw_data/sema/types/old_versions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
52
|
+
gw_data-0.3.0.dist-info/METADATA,sha256=hwWQE8z3H_qPugIeWk2-nJ4BTfMxtW1BXhseSiffXh4,9478
|
|
53
|
+
gw_data-0.3.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
|
|
54
|
+
gw_data-0.3.0.dist-info/entry_points.txt,sha256=O1SfIojdL-7hyB4sh94VyonbsMnXtu7sYRruz03HldQ,41
|
|
55
|
+
gw_data-0.3.0.dist-info/RECORD,,
|