gw_data 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. gw_data-0.3.0/.github/workflows/release.yml +58 -0
  2. gw_data-0.3.0/.gitignore +89 -0
  3. gw_data-0.3.0/PKG-INFO +189 -0
  4. gw_data-0.3.0/README.md +174 -0
  5. gw_data-0.3.0/alembic/README +1 -0
  6. gw_data-0.3.0/alembic/env.py +87 -0
  7. gw_data-0.3.0/alembic/script.py.mako +28 -0
  8. gw_data-0.3.0/alembic/versions/1220f2f941dd_initial_schema.py +200 -0
  9. gw_data-0.3.0/alembic.ini +41 -0
  10. gw_data-0.3.0/postgres_views.md +44 -0
  11. gw_data-0.3.0/pyproject.toml +30 -0
  12. gw_data-0.3.0/sema_seed_request.yaml +7 -0
  13. gw_data-0.3.0/src/gw_data/__init__.py +4 -0
  14. gw_data-0.3.0/src/gw_data/config.py +19 -0
  15. gw_data-0.3.0/src/gw_data/db/__init__.py +0 -0
  16. gw_data-0.3.0/src/gw_data/db/models/__init__.py +37 -0
  17. gw_data-0.3.0/src/gw_data/db/models/_base.py +5 -0
  18. gw_data-0.3.0/src/gw_data/db/models/connectivity_edge.py +80 -0
  19. gw_data-0.3.0/src/gw_data/db/models/customer.py +18 -0
  20. gw_data-0.3.0/src/gw_data/db/models/g_node.py +83 -0
  21. gw_data-0.3.0/src/gw_data/db/models/installation.py +52 -0
  22. gw_data-0.3.0/src/gw_data/db/models/installer.py +17 -0
  23. gw_data-0.3.0/src/gw_data/db/models/message.py +51 -0
  24. gw_data-0.3.0/src/gw_data/db/models/position_point.py +46 -0
  25. gw_data-0.3.0/src/gw_data/db/models/reading.py +55 -0
  26. gw_data-0.3.0/src/gw_data/db/models/reading_channel.py +62 -0
  27. gw_data-0.3.0/src/gw_data/db/models/user.py +35 -0
  28. gw_data-0.3.0/src/gw_data/db/models/user_installation_role.py +38 -0
  29. gw_data-0.3.0/src/gw_data/db/scripts/0_db_create.psql +4 -0
  30. gw_data-0.3.0/src/gw_data/db/scripts/1_db_user_setup.psql +16 -0
  31. gw_data-0.3.0/src/gw_data/db/scripts/2_db_schema_setup.sql +11 -0
  32. gw_data-0.3.0/src/gw_data/db/scripts/3_db_alembic_upgrade.sh +2 -0
  33. gw_data-0.3.0/src/gw_data/db/scripts/4_db_seed.py +129 -0
  34. gw_data-0.3.0/src/gw_data/db/scripts/_XX_drop_all.sql +13 -0
  35. gw_data-0.3.0/src/gw_data/db/scripts/seed_data/homes.csv +6 -0
  36. gw_data-0.3.0/src/gw_data/db/session.py +0 -0
  37. gw_data-0.3.0/src/gw_data/sema/__init__.py +15 -0
  38. gw_data-0.3.0/src/gw_data/sema/base.py +184 -0
  39. gw_data-0.3.0/src/gw_data/sema/codec.py +168 -0
  40. gw_data-0.3.0/src/gw_data/sema/definitions/enums/base.g.node.class/000.yaml +55 -0
  41. gw_data-0.3.0/src/gw_data/sema/definitions/enums/g.node.status/000.yaml +35 -0
  42. gw_data-0.3.0/src/gw_data/sema/definitions/formats/left.right.dot.yaml +27 -0
  43. gw_data-0.3.0/src/gw_data/sema/definitions/formats/uuid4.str.yaml +29 -0
  44. gw_data-0.3.0/src/gw_data/sema/definitions/registry.yaml +77 -0
  45. gw_data-0.3.0/src/gw_data/sema/definitions/types/g.node.gt/004.yaml +169 -0
  46. gw_data-0.3.0/src/gw_data/sema/definitions/types/position.point.gt/000.yaml +86 -0
  47. gw_data-0.3.0/src/gw_data/sema/enums/__init__.py +7 -0
  48. gw_data-0.3.0/src/gw_data/sema/enums/base_g_node_class.py +29 -0
  49. gw_data-0.3.0/src/gw_data/sema/enums/g_node_status.py +28 -0
  50. gw_data-0.3.0/src/gw_data/sema/enums/gw_str_enum.py +97 -0
  51. gw_data-0.3.0/src/gw_data/sema/enums/old_versions/__init__.py +0 -0
  52. gw_data-0.3.0/src/gw_data/sema/indexes/dependency_closure.yaml +19 -0
  53. gw_data-0.3.0/src/gw_data/sema/indexes/local_names.yaml +6 -0
  54. gw_data-0.3.0/src/gw_data/sema/indexes/lookup.yaml +35 -0
  55. gw_data-0.3.0/src/gw_data/sema/indexes/reverse_dependencies.yaml +21 -0
  56. gw_data-0.3.0/src/gw_data/sema/indexes/seed_expanded.yaml +29 -0
  57. gw_data-0.3.0/src/gw_data/sema/indexes/versions.yaml +31 -0
  58. gw_data-0.3.0/src/gw_data/sema/property_format.py +56 -0
  59. gw_data-0.3.0/src/gw_data/sema/tests/test_property_format.py +47 -0
  60. gw_data-0.3.0/src/gw_data/sema/types/__init__.py +7 -0
  61. gw_data-0.3.0/src/gw_data/sema/types/g_node_gt.py +106 -0
  62. gw_data-0.3.0/src/gw_data/sema/types/old_versions/__init__.py +0 -0
  63. gw_data-0.3.0/src/gw_data/sema/types/position_point_gt.py +31 -0
  64. gw_data-0.3.0/template.env +18 -0
  65. gw_data-0.3.0/uv.lock +485 -0
@@ -0,0 +1,58 @@
1
+ name: Release
2
+
3
+ # Publish gw_data to PyPI when a push to main carries a new version.
4
+ #
5
+ # The version in pyproject.toml is the trigger: salsify/detect-and-tag
6
+ # compares `uv version --short` against existing git tags. If the version
7
+ # is new, it creates the `vX.Y.Z` tag and the build/publish steps run;
8
+ # if the version already has a tag, those steps are skipped (no
9
+ # re-publish, which PyPI forbids anyway). So the release ritual is just
10
+ # "bump the version, merge to main."
11
+ #
12
+ # Modeled on gridworks-base/.github/workflows/release.yml, trimmed to the
13
+ # PyPI publish path (no TestPyPI dev-release, no release-drafter). Requires
14
+ # a single repo secret: PYPI_TOKEN.
15
+
16
+ on:
17
+ push:
18
+ branches:
19
+ - main
20
+
21
+ jobs:
22
+ release:
23
+ name: Release
24
+ runs-on: ubuntu-latest
25
+ permissions:
26
+ contents: write # salsify/detect-and-tag pushes the vX.Y.Z tag
27
+ steps:
28
+ - name: Check out the repository
29
+ uses: actions/checkout@v4
30
+ with:
31
+ fetch-depth: 2
32
+
33
+ - name: Install uv
34
+ uses: astral-sh/setup-uv@v6
35
+ with:
36
+ python-version: "3.12"
37
+ enable-cache: true
38
+
39
+ - name: Check if there is a parent commit
40
+ id: check-parent-commit
41
+ run: |
42
+ echo "sha=$(git rev-parse --verify --quiet HEAD^)" >> "$GITHUB_OUTPUT"
43
+
44
+ - name: Detect and tag new version
45
+ id: check-version
46
+ if: steps.check-parent-commit.outputs.sha
47
+ uses: salsify/action-detect-and-tag-new-version@v2.0.3
48
+ with:
49
+ version-command: |
50
+ uv version --short
51
+
52
+ - name: Build package
53
+ if: steps.check-version.outputs.tag
54
+ run: uv build
55
+
56
+ - name: Publish package on PyPI
57
+ if: steps.check-version.outputs.tag
58
+ run: uv publish --token ${{ secrets.PYPI_TOKEN }}
@@ -0,0 +1,89 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[codz]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # scratch files
10
+ scratch*
11
+
12
+ # Distribution / packaging
13
+ .Python
14
+ build/
15
+ develop-eggs/
16
+ dist/
17
+ downloads/
18
+ eggs/
19
+ .eggs/
20
+ lib/
21
+ lib64/
22
+ parts/
23
+ sdist/
24
+ var/
25
+ wheels/
26
+ share/python-wheels/
27
+ *.egg-info/
28
+ .installed.cfg
29
+ *.egg
30
+ MANIFEST
31
+
32
+ *.DS_STORE
33
+ *.manifest
34
+ *.spec
35
+
36
+ # Installer logs
37
+ pip-log.txt
38
+ pip-delete-this-directory.txt
39
+
40
+ # Unit test / coverage reports
41
+ htmlcov/
42
+ .tox/
43
+ .nox/
44
+ .coverage
45
+ .coverage.*
46
+ .cache
47
+ nosetests.xml
48
+ coverage.xml
49
+ *.cover
50
+ *.py.cover
51
+ .hypothesis/
52
+ .pytest_cache/
53
+ cover/
54
+
55
+ .pdm-python
56
+ .pdm-build/
57
+
58
+
59
+ # Environments
60
+ .env
61
+ .envrc
62
+ .venv
63
+ env/
64
+ venv/
65
+ ENV/
66
+ env.bak/
67
+ venv.bak/
68
+
69
+ # mypy
70
+ .mypy_cache/
71
+ .dmypy.json
72
+ dmypy.json
73
+
74
+ # Pyre type checker
75
+ .pyre/
76
+
77
+ # pytype static type analyzer
78
+ .pytype/
79
+
80
+ # Cython debug symbols
81
+ cython_debug/
82
+
83
+
84
+ .idea/
85
+
86
+ .vscode/
87
+
88
+ # Ruff stuff:
89
+ .ruff_cache/
gw_data-0.3.0/PKG-INFO ADDED
@@ -0,0 +1,189 @@
1
+ Metadata-Version: 2.4
2
+ Name: gw_data
3
+ Version: 0.3.0
4
+ Summary: Gridworks Data Infrastructure
5
+ Author-email: Jessica Millar <jessica.lynn.millar@gmail.com>
6
+ Requires-Python: <3.13.0,>=3.12
7
+ Requires-Dist: alembic>=1.17.2
8
+ Requires-Dist: fastapi>=0.123.0
9
+ Requires-Dist: psycopg[binary]>=3.1
10
+ Requires-Dist: pydantic-settings>=2.12.0
11
+ Requires-Dist: sqlalchemy-timescaledb>=0.4.1
12
+ Requires-Dist: sqlalchemy>=2.0.44
13
+ Requires-Dist: uvicorn[standard]>=0.38.0
14
+ Description-Content-Type: text/markdown
15
+
16
+ # Gridworks Data
17
+
18
+ This project contains code related to working with databases in the GridWorks ecosystem.
19
+
20
+ Our platform is PostgreSQL, with the TimescaleDB (+toolkit) extensions for efficiently handling time-series data.
21
+
22
+ ## Prerequisites
23
+
24
+ * **Docker Engine** — to run PostgreSQL+TimescaleDB locally.
25
+ (If you have previously set up the RabbitMQ server from `gridworks-base`, you should already have Docker.)
26
+ * **`psql`** PostgreSQL command-line client (any modern version; tested with 14.x and 18.x).
27
+ Install via your package manager (e.g. `brew install libpq && brew link --force libpq` on macOS, `sudo apt install postgresql-client` on Linux). Check with `psql --version`.
28
+ *(Optional: `pgAdmin` as a GUI for inspecting the database.)*
29
+ * **Python ≥ 3.12** and `uv` (per the rest of the GridWorks toolchain).
30
+
31
+ ## Database Setup
32
+
33
+ The following steps will get you set up to run with the database, either locally or on a managed Tiger Cloud instance.
34
+
35
+ ### Local Pre-requisite: PostgreSQL+TimescaleDB container
36
+
37
+ If running locally, you'll need to start by pulling the official TimescaleDB image and starting a container. **You need the `-ha` variant** for PostgreSQL v18 and TimescaleDB v2.25 — without `-ha` you'll get the "lite" version of TimescaleDB which is missing required functionality.
38
+
39
+ If you don't already have a PostgreSQL listener on port 5432, map `5432:5432`. If you do (native install, or another Docker postgres), pick a free port and map `<HOST_PORT>:5432` instead (e.g. `5433:5432`).
40
+
41
+ Pick a `POSTGRES_PASSWORD` — this becomes the password for the built-in `postgres` superuser; record it locally (e.g. in `.env`), don't commit it.
42
+
43
+ Concrete `docker run` example (uses 5433 and a placeholder password):
44
+
45
+ docker run -d \
46
+ --name gw-data-pg \
47
+ -e POSTGRES_PASSWORD=changeme \
48
+ -p 5433:5432 \
49
+ timescale/timescaledb-ha:pg18-ts2.25
50
+
51
+ Verify the container is healthy: `docker ps` should show it up, and `PGPASSWORD=changeme psql -h 127.0.0.1 -p 5433 -U postgres -c "SELECT version();"` should return PostgreSQL 18.x.
52
+
53
+ Detailed instructions and alternative configurations: https://www.tigerdata.com/docs/self-hosted/latest/install/installation-docker
54
+
55
+ ### Database Setup
56
+
57
+ This repo includes a numbered sequence of scripts in the `src/gw_data/db/scripts` folder that will get you up and running.
58
+
59
+ #### 0. (Local-Only) Create the Database
60
+
61
+ This step is not required (or even possible) on a Tiger Cloud managed server.
62
+
63
+ Run `0_db_create.sql` as the `postgres` user to create the `tsdb` database and add the `timescaledb` extension.
64
+
65
+ psql -d "postgresql://127.0.0.1:<PORT>/" -U postgres -f src/gw_data/db/scripts/0_server_init.psql
66
+
67
+ * Replace `<PORT>` with the host port you mapped to 5432 (e.g. `5433`).
68
+ * You'll be prompted for the `postgres` user password (i.e., the one you previously set as `POSTGRES_PASSWORD` in your `docker run` command).
69
+
70
+ #### 1. Create the Users
71
+
72
+ Run `1_db_user_setup.psql` as follows to create the users we need. This script can be run as the `postgres` user locally, or the `tsdbadmin` user in Tiger Cloud.
73
+
74
+ psql -d "postgresql://<SERVER>:<PORT>/" -U <postgres|tsdbadmin> -f src/gw_data/db/scripts/1_db_user_setup.psql
75
+
76
+ * Replace `<SERVER>` with the database server address (e.g., `127.0.0.1` when running locally).
77
+ * Again, replace `<PORT>` with your mapped host port.
78
+ * Again, you'll be prompted for the `postgres` user password.
79
+
80
+ This script creates three user roles: `gw_admin` (full ownership), `gw_journalkeeper` (insert/update/delete), and `gw_visualizer` (select-only).
81
+
82
+ **Note: the script uses `psql`'s interactive `\password` meta-command three times** (once for each user). It must be run **interactively** — it will prompt for each password as it runs and cannot be piped or run via CI:
83
+
84
+ You'll be prompted for new passwords for each of the users. Pick whatever you like and record them somewhere.
85
+
86
+ For non-interactive setups (CI, scripted bootstraps), apply the equivalent SQL with explicit passwords; see the script as the canonical reference.
87
+
88
+ **Reset shortcut:** if your local DB ever gets into a weird state, you can wipe it and start over with:
89
+
90
+ psql -d "postgresql://127.0.0.1:<PORT>/" -U postgres -f src/gw_data/db/scripts/_XX_drop_all.sql
91
+
92
+ #### 2. Create the `gridworks` Schema
93
+
94
+ Run `2_db_schema_setup.sql` as follows to create our private `gridworks` schema and apply appropriate permissions. This should be run as the `gw_admin` user.
95
+
96
+ psql -d "postgresql://<SERVER>:<PORT>/" -U gw_admin -f src/gw_data/db/scripts/2_db_schema_setup.psql
97
+
98
+ * Again, replace `<SERVER>` with the database server address.
99
+ * Again, replace `<PORT>` with your mapped host port.
100
+ * This time you'll be prompted for the `gw_admin` user password.
101
+
102
+ #### 3. Run the Alembic Migrations
103
+
104
+ Next we need to run the database migrations we've defined with Alembic to create the tables, etc. that we need.
105
+
106
+ But first we need to update our .env file. Copy `template.env` (at the repo root) to `.env`:
107
+
108
+ cp template.env .env
109
+
110
+ Then, edit `.env` as follows:
111
+ * Replace `<SERVER>` with the database server address (e.g., `127.0.0.1` when running locally).
112
+ * Replace `<%PASSWORD%>` in `GW_DATA_DB_URL` with the `gw_admin` password you just set.
113
+ * Replace `<%PORT%>` with the host port you mapped to 5432 (e.g. `5433`).
114
+
115
+ Now we can run `3_db_alembic_upgrade.sh` as follows:
116
+
117
+ `sh src/gw_data/db/scripts/3_db_alembic_upgrade.sh`
118
+
119
+ This should create 12 tables in the `gridworks` schema: `alembic_version`, `connectivity_edges`, `customers`, `g_nodes`, `installations`, `installers`, `messages`, `position_points`, `reading_channels`, `readings`, `user_installation_roles`, `users`.
120
+
121
+ Verify with `psql -d "postgresql://<SERVER>:<PORT>/tsdb" -U gw_admin -c "\dt gridworks.*"`.
122
+
123
+ #### 4. Seed the Database
124
+
125
+ With the database created, `gw_admin` ready, and `.env` filled in, you can seed some initial data:
126
+
127
+ uv run python ./src/gw_data/db/scripts/1_db_seed.py
128
+
129
+ **Note: the seed script is also interactive** — it uses Python's `getpass` to prompt for passwords for two seeded users (`admin` and `beech-user`). It must be run from a real terminal (no piping). The seed populates a small set of dev users, a customer, an installation, and one g_node.
130
+
131
+ In the future we will have a more comprehensive seeding process that will ingest some actual message data.
132
+
133
+ ## Best Practices for Databases
134
+
135
+ The following are some best practices that we should follow when at all possible:
136
+
137
+ * Primary Keys should be UUIDs, stored as the DB-native UUID type
138
+ * Dates/Times should be stored as `TIMESTAMPTZ`
139
+ * Migrations should be encouraged and done frequently to provide new functionality. Database schema is always temporary.
140
+ * Each application that uses that database should have its own dedicated user, with the minimal set of permissions. (In particular, `postgres` should never be used at all, and `gw_admin` should only be used for migrations and other tasks that require full ownership of the database.)
141
+
142
+ ## TimescaleDB Performance
143
+
144
+ TimescaleDB does two main things to improve performance:
145
+ 1. Separates time-series tables (which it calls "hypertables") into "chunks", each of which cover a certain timeframe.
146
+ 2. Allows time-series data to be stored in column order with compression, which vastly improves performance for data that changes very little over time. This compression happens in a scheduled job, and only for data older than a configured threshold.
147
+
148
+ We have column-store compression enabled on the readings table for data older than 2 weeks, with compression segmented by the channel ID.
149
+
150
+ ### Useful Queries
151
+
152
+ The following queries are useful for analyzing TimescaleDB performance:
153
+
154
+ ```
155
+ -- Display the size in MB of our two main tables
156
+ SELECT pg_size_pretty(hypertable_size('readings')) as "Readings Table Size", pg_size_pretty(hypertable_size('messages')) as "Messages Table Size";
157
+ ```
158
+
159
+ ```
160
+ -- Display the size of all database tables in order.
161
+ -- This will show each TimescaleDB chunk as a separate table.
162
+ SELECT
163
+ table_schema || '.' || table_name AS table_full_name,
164
+ pg_size_pretty(pg_total_relation_size('"' || table_schema || '"."' || table_name || '"')) AS size
165
+ FROM information_schema.tables
166
+ ORDER BY
167
+ pg_total_relation_size('"' || table_schema || '"."' || table_name || '"') DESC;
168
+
169
+ ```
170
+
171
+ ```
172
+ -- Display compression stats for each chunk in the readings table
173
+ SELECT
174
+ chunk_name,
175
+ pg_size_pretty(before_compression_total_bytes) AS size_before,
176
+ pg_size_pretty(after_compression_total_bytes) AS size_after,
177
+ 100 - (after_compression_total_bytes::float / before_compression_total_bytes * 100) AS compression_ratio_pct
178
+ FROM chunk_compression_stats('readings');
179
+ ```
180
+
181
+ ```
182
+ -- Get the ID of the policy_compression job so you can manually run `CALL run_job` with it (e.g. after a bulk import).
183
+ SELECT job_id, proc_name, hypertable_name, config
184
+ FROM timescaledb_information.jobs;
185
+ ```
186
+ ```
187
+ -- Get info (table, time range, etc.) about the TimescaleDB chunks.
188
+ SELECT * FROM timescaledb_information.chunks
189
+ ```
@@ -0,0 +1,174 @@
1
+ # Gridworks Data
2
+
3
+ This project contains code related to working with databases in the GridWorks ecosystem.
4
+
5
+ Our platform is PostgreSQL, with the TimescaleDB (+toolkit) extensions for efficiently handling time-series data.
6
+
7
+ ## Prerequisites
8
+
9
+ * **Docker Engine** — to run PostgreSQL+TimescaleDB locally.
10
+ (If you have previously set up the RabbitMQ server from `gridworks-base`, you should already have Docker.)
11
+ * **`psql`** PostgreSQL command-line client (any modern version; tested with 14.x and 18.x).
12
+ Install via your package manager (e.g. `brew install libpq && brew link --force libpq` on macOS, `sudo apt install postgresql-client` on Linux). Check with `psql --version`.
13
+ *(Optional: `pgAdmin` as a GUI for inspecting the database.)*
14
+ * **Python ≥ 3.12** and `uv` (per the rest of the GridWorks toolchain).
15
+
16
+ ## Database Setup
17
+
18
+ The following steps will get you set up to run with the database, either locally or on a managed Tiger Cloud instance.
19
+
20
+ ### Local Pre-requisite: PostgreSQL+TimescaleDB container
21
+
22
+ If running locally, you'll need to start by pulling the official TimescaleDB image and starting a container. **You need the `-ha` variant** for PostgreSQL v18 and TimescaleDB v2.25 — without `-ha` you'll get the "lite" version of TimescaleDB which is missing required functionality.
23
+
24
+ If you don't already have a PostgreSQL listener on port 5432, map `5432:5432`. If you do (native install, or another Docker postgres), pick a free port and map `<HOST_PORT>:5432` instead (e.g. `5433:5432`).
25
+
26
+ Pick a `POSTGRES_PASSWORD` — this becomes the password for the built-in `postgres` superuser; record it locally (e.g. in `.env`), don't commit it.
27
+
28
+ Concrete `docker run` example (uses 5433 and a placeholder password):
29
+
30
+ docker run -d \
31
+ --name gw-data-pg \
32
+ -e POSTGRES_PASSWORD=changeme \
33
+ -p 5433:5432 \
34
+ timescale/timescaledb-ha:pg18-ts2.25
35
+
36
+ Verify the container is healthy: `docker ps` should show it up, and `PGPASSWORD=changeme psql -h 127.0.0.1 -p 5433 -U postgres -c "SELECT version();"` should return PostgreSQL 18.x.
37
+
38
+ Detailed instructions and alternative configurations: https://www.tigerdata.com/docs/self-hosted/latest/install/installation-docker
39
+
40
+ ### Database Setup
41
+
42
+ This repo includes a numbered sequence of scripts in the `src/gw_data/db/scripts` folder that will get you up and running.
43
+
44
+ #### 0. (Local-Only) Create the Database
45
+
46
+ This step is not required (or even possible) on a Tiger Cloud managed server.
47
+
48
+ Run `0_db_create.sql` as the `postgres` user to create the `tsdb` database and add the `timescaledb` extension.
49
+
50
+ psql -d "postgresql://127.0.0.1:<PORT>/" -U postgres -f src/gw_data/db/scripts/0_server_init.psql
51
+
52
+ * Replace `<PORT>` with the host port you mapped to 5432 (e.g. `5433`).
53
+ * You'll be prompted for the `postgres` user password (i.e., the one you previously set as `POSTGRES_PASSWORD` in your `docker run` command).
54
+
55
+ #### 1. Create the Users
56
+
57
+ Run `1_db_user_setup.psql` as follows to create the users we need. This script can be run as the `postgres` user locally, or the `tsdbadmin` user in Tiger Cloud.
58
+
59
+ psql -d "postgresql://<SERVER>:<PORT>/" -U <postgres|tsdbadmin> -f src/gw_data/db/scripts/1_db_user_setup.psql
60
+
61
+ * Replace `<SERVER>` with the database server address (e.g., `127.0.0.1` when running locally).
62
+ * Again, replace `<PORT>` with your mapped host port.
63
+ * Again, you'll be prompted for the `postgres` user password.
64
+
65
+ This script creates three user roles: `gw_admin` (full ownership), `gw_journalkeeper` (insert/update/delete), and `gw_visualizer` (select-only).
66
+
67
+ **Note: the script uses `psql`'s interactive `\password` meta-command three times** (once for each user). It must be run **interactively** — it will prompt for each password as it runs and cannot be piped or run via CI:
68
+
69
+ You'll be prompted for new passwords for each of the users. Pick whatever you like and record them somewhere.
70
+
71
+ For non-interactive setups (CI, scripted bootstraps), apply the equivalent SQL with explicit passwords; see the script as the canonical reference.
72
+
73
+ **Reset shortcut:** if your local DB ever gets into a weird state, you can wipe it and start over with:
74
+
75
+ psql -d "postgresql://127.0.0.1:<PORT>/" -U postgres -f src/gw_data/db/scripts/_XX_drop_all.sql
76
+
77
+ #### 2. Create the `gridworks` Schema
78
+
79
+ Run `2_db_schema_setup.sql` as follows to create our private `gridworks` schema and apply appropriate permissions. This should be run as the `gw_admin` user.
80
+
81
+ psql -d "postgresql://<SERVER>:<PORT>/" -U gw_admin -f src/gw_data/db/scripts/2_db_schema_setup.psql
82
+
83
+ * Again, replace `<SERVER>` with the database server address.
84
+ * Again, replace `<PORT>` with your mapped host port.
85
+ * This time you'll be prompted for the `gw_admin` user password.
86
+
87
+ #### 3. Run the Alembic Migrations
88
+
89
+ Next we need to run the database migrations we've defined with Alembic to create the tables, etc. that we need.
90
+
91
+ But first we need to update our .env file. Copy `template.env` (at the repo root) to `.env`:
92
+
93
+ cp template.env .env
94
+
95
+ Then, edit `.env` as follows:
96
+ * Replace `<SERVER>` with the database server address (e.g., `127.0.0.1` when running locally).
97
+ * Replace `<%PASSWORD%>` in `GW_DATA_DB_URL` with the `gw_admin` password you just set.
98
+ * Replace `<%PORT%>` with the host port you mapped to 5432 (e.g. `5433`).
99
+
100
+ Now we can run `3_db_alembic_upgrade.sh` as follows:
101
+
102
+ `sh src/gw_data/db/scripts/3_db_alembic_upgrade.sh`
103
+
104
+ This should create 12 tables in the `gridworks` schema: `alembic_version`, `connectivity_edges`, `customers`, `g_nodes`, `installations`, `installers`, `messages`, `position_points`, `reading_channels`, `readings`, `user_installation_roles`, `users`.
105
+
106
+ Verify with `psql -d "postgresql://<SERVER>:<PORT>/tsdb" -U gw_admin -c "\dt gridworks.*"`.
107
+
108
+ #### 4. Seed the Database
109
+
110
+ With the database created, `gw_admin` ready, and `.env` filled in, you can seed some initial data:
111
+
112
+ uv run python ./src/gw_data/db/scripts/1_db_seed.py
113
+
114
+ **Note: the seed script is also interactive** — it uses Python's `getpass` to prompt for passwords for two seeded users (`admin` and `beech-user`). It must be run from a real terminal (no piping). The seed populates a small set of dev users, a customer, an installation, and one g_node.
115
+
116
+ In the future we will have a more comprehensive seeding process that will ingest some actual message data.
117
+
118
+ ## Best Practices for Databases
119
+
120
+ The following are some best practices that we should follow when at all possible:
121
+
122
+ * Primary Keys should be UUIDs, stored as the DB-native UUID type
123
+ * Dates/Times should be stored as `TIMESTAMPTZ`
124
+ * Migrations should be encouraged and done frequently to provide new functionality. Database schema is always temporary.
125
+ * Each application that uses that database should have its own dedicated user, with the minimal set of permissions. (In particular, `postgres` should never be used at all, and `gw_admin` should only be used for migrations and other tasks that require full ownership of the database.)
126
+
127
+ ## TimescaleDB Performance
128
+
129
+ TimescaleDB does two main things to improve performance:
130
+ 1. Separates time-series tables (which it calls "hypertables") into "chunks", each of which cover a certain timeframe.
131
+ 2. Allows time-series data to be stored in column order with compression, which vastly improves performance for data that changes very little over time. This compression happens in a scheduled job, and only for data older than a configured threshold.
132
+
133
+ We have column-store compression enabled on the readings table for data older than 2 weeks, with compression segmented by the channel ID.
134
+
135
+ ### Useful Queries
136
+
137
+ The following queries are useful for analyzing TimescaleDB performance:
138
+
139
+ ```
140
+ -- Display the size in MB of our two main tables
141
+ SELECT pg_size_pretty(hypertable_size('readings')) as "Readings Table Size", pg_size_pretty(hypertable_size('messages')) as "Messages Table Size";
142
+ ```
143
+
144
+ ```
145
+ -- Display the size of all database tables in order.
146
+ -- This will show each TimescaleDB chunk as a separate table.
147
+ SELECT
148
+ table_schema || '.' || table_name AS table_full_name,
149
+ pg_size_pretty(pg_total_relation_size('"' || table_schema || '"."' || table_name || '"')) AS size
150
+ FROM information_schema.tables
151
+ ORDER BY
152
+ pg_total_relation_size('"' || table_schema || '"."' || table_name || '"') DESC;
153
+
154
+ ```
155
+
156
+ ```
157
+ -- Display compression stats for each chunk in the readings table
158
+ SELECT
159
+ chunk_name,
160
+ pg_size_pretty(before_compression_total_bytes) AS size_before,
161
+ pg_size_pretty(after_compression_total_bytes) AS size_after,
162
+ 100 - (after_compression_total_bytes::float / before_compression_total_bytes * 100) AS compression_ratio_pct
163
+ FROM chunk_compression_stats('readings');
164
+ ```
165
+
166
+ ```
167
+ -- Get the ID of the policy_compression job so you can manually run `CALL run_job` with it (e.g. after a bulk import).
168
+ SELECT job_id, proc_name, hypertable_name, config
169
+ FROM timescaledb_information.jobs;
170
+ ```
171
+ ```
172
+ -- Get info (table, time range, etc.) about the TimescaleDB chunks.
173
+ SELECT * FROM timescaledb_information.chunks
174
+ ```
@@ -0,0 +1 @@
1
+ Generic single-database configuration.
@@ -0,0 +1,87 @@
1
+ from logging.config import fileConfig
2
+
3
+ import dotenv
4
+ from sqlalchemy import engine_from_config, pool
5
+
6
+ from alembic import context
7
+ from gw_data.config import Settings
8
+
9
+ # Custom code to use the gridworks schema
10
+ GW_SCHEMA = "gridworks"
11
+ def include_name(name, type_, parent_names):
12
+ if type_ == "schema":
13
+ # Only track migrations in your chosen schema
14
+ return name == GW_SCHEMA
15
+ return True
16
+
17
+ # -----------------------------------------------------------------------------
18
+ # Alembic Config
19
+ # -----------------------------------------------------------------------------
20
+ config = context.config
21
+
22
+ # Interpret config file for Python logging.
23
+ if config.config_file_name is not None:
24
+ fileConfig(config.config_file_name)
25
+
26
+
27
+ # -----------------------------------------------------------------------------
28
+ # Database URL (from our pydantic Settings)
29
+ # -----------------------------------------------------------------------------
30
+ dotenv.load_dotenv()
31
+ settings = Settings()
32
+ db_url = settings.db_url.get_secret_value()
33
+ config.set_main_option("sqlalchemy.url", db_url)
34
+
35
+ from gw_data.db.models._base import Base
36
+ target_metadata = Base.metadata
37
+
38
+
39
+ # -----------------------------------------------------------------------------
40
+ # Run migrations
41
+ # -----------------------------------------------------------------------------
42
+ def run_migrations_offline():
43
+ """Run migrations in 'offline' mode."""
44
+ context.configure(
45
+ url=db_url,
46
+ target_metadata=target_metadata,
47
+ literal_binds=True,
48
+ dialect_opts={"paramstyle": "named"},
49
+
50
+ # Use our custom schema (and scan non-default schemas to find it, and use our filter function)
51
+ version_table_schema=GW_SCHEMA,
52
+ include_schemas=True,
53
+ include_name=include_name,
54
+ )
55
+
56
+ with context.begin_transaction():
57
+ context.run_migrations()
58
+
59
+
60
+ def run_migrations_online():
61
+ """Run migrations in 'online' mode."""
62
+
63
+ connectable = engine_from_config(
64
+ config.get_section(config.config_ini_section, {}),
65
+ prefix="sqlalchemy.",
66
+ poolclass=pool.NullPool,
67
+ )
68
+
69
+ with connectable.connect() as connection:
70
+ context.configure(
71
+ connection=connection,
72
+ target_metadata=target_metadata,
73
+
74
+ # Use our custom schema (and scan non-default schemas to find it, and use our filter function)
75
+ version_table_schema=GW_SCHEMA,
76
+ include_schemas=True,
77
+ include_name=include_name,
78
+ )
79
+
80
+ with context.begin_transaction():
81
+ context.run_migrations()
82
+
83
+
84
+ if context.is_offline_mode():
85
+ run_migrations_offline()
86
+ else:
87
+ run_migrations_online()
@@ -0,0 +1,28 @@
1
+ """${message}
2
+
3
+ Revision ID: ${up_revision}
4
+ Revises: ${down_revision | comma,n}
5
+ Create Date: ${create_date}
6
+
7
+ """
8
+ from typing import Sequence, Union
9
+
10
+ from alembic import op
11
+ import sqlalchemy as sa
12
+ ${imports if imports else ""}
13
+
14
+ # revision identifiers, used by Alembic.
15
+ revision: str = ${repr(up_revision)}
16
+ down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)}
17
+ branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
18
+ depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
19
+
20
+
21
+ def upgrade() -> None:
22
+ """Upgrade schema."""
23
+ ${upgrades if upgrades else "pass"}
24
+
25
+
26
+ def downgrade() -> None:
27
+ """Downgrade schema."""
28
+ ${downgrades if downgrades else "pass"}