renovara-sql 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,253 @@
1
+
2
+ # Created by https://www.gitignore.io/api/osx,linux,python,windows,pycharm,visualstudiocode
3
+
4
+ ### Linux ###
5
+ *~
6
+
7
+ # temporary files which can be created if a process still has a handle open of a deleted file
8
+ .fuse_hidden*
9
+
10
+ # KDE directory preferences
11
+ .directory
12
+
13
+ # Linux trash folder which might appear on any partition or disk
14
+ .Trash-*
15
+
16
+ # .nfs files are created when an open file is removed but is still being accessed
17
+ .nfs*
18
+
19
+ ### OSX ###
20
+ *.DS_Store
21
+ .AppleDouble
22
+ .LSOverride
23
+
24
+ # Icon must end with two \r
25
+ Icon
26
+
27
+ # Thumbnails
28
+ ._*
29
+
30
+ # Files that might appear in the root of a volume
31
+ .DocumentRevisions-V100
32
+ .fseventsd
33
+ .Spotlight-V100
34
+ .TemporaryItems
35
+ .Trashes
36
+ .VolumeIcon.icns
37
+ .com.apple.timemachine.donotpresent
38
+
39
+ # Directories potentially created on remote AFP share
40
+ .AppleDB
41
+ .AppleDesktop
42
+ Network Trash Folder
43
+ Temporary Items
44
+ .apdisk
45
+
46
+ ### PyCharm ###
47
+ # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
48
+ # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
49
+
50
+ # User-specific stuff:
51
+ .idea/**/workspace.xml
52
+ .idea/**/tasks.xml
53
+ .idea/dictionaries
54
+
55
+ # Sensitive or high-churn files:
56
+ .idea/**/dataSources/
57
+ .idea/**/dataSources.ids
58
+ .idea/**/dataSources.xml
59
+ .idea/**/dataSources.local.xml
60
+ .idea/**/sqlDataSources.xml
61
+ .idea/**/dynamic.xml
62
+ .idea/**/uiDesigner.xml
63
+
64
+ # Gradle:
65
+ .idea/**/gradle.xml
66
+ .idea/**/libraries
67
+
68
+ # CMake
69
+ cmake-build-debug/
70
+
71
+ # Mongo Explorer plugin:
72
+ .idea/**/mongoSettings.xml
73
+
74
+ ## File-based project format:
75
+ *.iws
76
+
77
+ ## Plugin-specific files:
78
+
79
+ # IntelliJ
80
+ /out/
81
+
82
+ # mpeltonen/sbt-idea plugin
83
+ .idea_modules/
84
+
85
+ # JIRA plugin
86
+ atlassian-ide-plugin.xml
87
+
88
+ # Cursive Clojure plugin
89
+ .idea/replstate.xml
90
+
91
+ # Ruby plugin and RubyMine
92
+ /.rakeTasks
93
+
94
+ # Crashlytics plugin (for Android Studio and IntelliJ)
95
+ com_crashlytics_export_strings.xml
96
+ crashlytics.properties
97
+ crashlytics-build.properties
98
+ fabric.properties
99
+
100
+ ### PyCharm Patch ###
101
+ # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
102
+
103
+ # *.iml
104
+ # modules.xml
105
+ # .idea/misc.xml
106
+ # *.ipr
107
+
108
+ # Sonarlint plugin
109
+ .idea/sonarlint
110
+
111
+ ### Python ###
112
+ # Byte-compiled / optimized / DLL files
113
+ __pycache__/
114
+ *.py[cod]
115
+ *$py.class
116
+
117
+ # C extensions
118
+ *.so
119
+
120
+ # Distribution / packaging
121
+ .Python
122
+ build/
123
+ develop-eggs/
124
+ dist/
125
+ downloads/
126
+ eggs/
127
+ .eggs/
128
+ lib/
129
+ lib64/
130
+ parts/
131
+ sdist/
132
+ var/
133
+ wheels/
134
+ *.egg-info/
135
+ .installed.cfg
136
+ *.egg
137
+
138
+ # PyInstaller
139
+ # Usually these files are written by a python script from a template
140
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
141
+ *.manifest
142
+ *.spec
143
+
144
+ # Installer logs
145
+ pip-log.txt
146
+ pip-delete-this-directory.txt
147
+
148
+ # Unit test / coverage reports
149
+ htmlcov/
150
+ .tox/
151
+ .coverage
152
+ .coverage.*
153
+ .cache
154
+ .pytest_cache/
155
+ nosetests.xml
156
+ coverage.xml
157
+ *.cover
158
+ .hypothesis/
159
+
160
+ # Translations
161
+ *.mo
162
+ *.pot
163
+
164
+ # Flask stuff:
165
+ instance/
166
+ .webassets-cache
167
+
168
+ # Scrapy stuff:
169
+ .scrapy
170
+
171
+ # Sphinx documentation
172
+ docs/_build/
173
+
174
+ # PyBuilder
175
+ target/
176
+
177
+ # Jupyter Notebook
178
+ .ipynb_checkpoints
179
+
180
+ # pyenv
181
+ .python-version
182
+
183
+ # celery beat schedule file
184
+ celerybeat-schedule.*
185
+
186
+ # SageMath parsed files
187
+ *.sage.py
188
+
189
+ # Environments
190
+ .env
191
+ .venv
192
+ env/
193
+ venv/
194
+ ENV/
195
+ env.bak/
196
+ venv.bak/
197
+ .envrc
198
+
199
+ # Spyder project settings
200
+ .spyderproject
201
+ .spyproject
202
+
203
+ # Rope project settings
204
+ .ropeproject
205
+
206
+ # mkdocs documentation
207
+ /site
208
+
209
+ # mypy
210
+ .mypy_cache/
211
+
212
+ ### VisualStudioCode ###
213
+ .vscode/*
214
+ !.vscode/settings.json
215
+ !.vscode/tasks.json
216
+ !.vscode/launch.json
217
+ !.vscode/extensions.json
218
+ .history
219
+
220
+ ### Windows ###
221
+ # Windows thumbnail cache files
222
+ Thumbs.db
223
+ ehthumbs.db
224
+ ehthumbs_vista.db
225
+
226
+ # Folder config file
227
+ Desktop.ini
228
+
229
+ # Recycle Bin used on file shares
230
+ $RECYCLE.BIN/
231
+
232
+ # Windows Installer files
233
+ *.cab
234
+ *.msi
235
+ *.msm
236
+ *.msp
237
+
238
+ # Windows shortcuts
239
+ *.lnk
240
+
241
+ # Build folder
242
+
243
+ */build/*
244
+
245
+ # The MCP stack's own SAM build and cache dirs (see samconfig.toml [mcp])
246
+ .aws-sam/mcp-build/
247
+ .aws-sam/mcp-cache/
248
+
249
+ # End of https://www.gitignore.io/api/osx,linux,python,windows,pycharm,visualstudiocode
250
+
251
+ # The SQL API's staged packages and download cache (scripts/build_sql.py)
252
+ .aws-sam/sql/
253
+ .aws-sam/sql-cache/
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Scoptico Pty Ltd (trading as Renovara)
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,119 @@
1
+ Metadata-Version: 2.5
2
+ Name: renovara-sql
3
+ Version: 0.2.0
4
+ Summary: Python client for the Renovara SQL API: DuckDB SQL over Australian NEM and east coast gas market data from AEMO
5
+ Project-URL: Homepage, https://renovara.co/sql-api
6
+ Project-URL: Documentation, https://renovara.co/how-renovara-sql-works
7
+ Project-URL: API description, https://sql.api.renovara.co/v1/openapi.json
8
+ Author-email: "Scoptico Pty Ltd (Renovara)" <info@renovara.co>
9
+ License-Expression: MIT
10
+ License-File: LICENSE
11
+ Keywords: aemo,australia,duckdb,electricity,energy,gas,nem,sql
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: Operating System :: OS Independent
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Topic :: Scientific/Engineering
16
+ Requires-Python: >=3.9
17
+ Provides-Extra: pandas
18
+ Requires-Dist: pandas; extra == 'pandas'
19
+ Requires-Dist: pyarrow; extra == 'pandas'
20
+ Description-Content-Type: text/markdown
21
+
22
+ # Renovara SQL API — Python client
23
+
24
+ Read-only SQL over Australian National Electricity Market (NEM) and east
25
+ coast gas market data from AEMO, from your own code. The SQL is DuckDB's.
26
+
27
+ ```bash
28
+ pip install renovara-sql
29
+ ```
30
+
31
+ ## See what the data is — no credentials needed
32
+
33
+ ```python
34
+ from renovara_sql import Client
35
+
36
+ public = Client("https://sql.api.renovara.co/v1")
37
+ print(public.guide()[:2000]) # which table answers what, and the traps
38
+ for t in public.tables(): # every table, one line each
39
+ print(t["table"], "-", t["description"][:80])
40
+ print(public.table("silver_dispatchis_reports_dispatch_price")["columns"][:5])
41
+ ```
42
+
43
+ ## Credentials
44
+
45
+ Your Renovara credentials page gives you a file:
46
+
47
+ ```json
48
+ {
49
+ "api_url": "https://sql.api.renovara.co/v1/",
50
+ "token_url": "https://auth.api.renovara.co/oauth2/token",
51
+ "client_id": "…",
52
+ "client_secret": "…",
53
+ "scope": "renovara-sql/query"
54
+ }
55
+ ```
56
+
57
+ Keep it private: anyone holding it can query as your organisation. It is an
58
+ OAuth client-credentials pair, exchanged for an access token that lasts an
59
+ hour; the client keeps the token for the hour and renews it for you.
60
+
61
+ ## Run queries
62
+
63
+ ```python
64
+ from renovara_sql import Client
65
+
66
+ client = Client.from_file("~/.config/renovara-sql/acme.json")
67
+ result = client.query("""
68
+ SELECT REGIONID, round(avg(RRP), 2) AS avg_price
69
+ FROM silver_dispatchis_reports_dispatch_price
70
+ WHERE SETTLEMENTDATE >= current_timestamp + INTERVAL 10 HOUR - INTERVAL 7 DAY
71
+ AND INTERVENTION = 0
72
+ GROUP BY 1 ORDER BY 1
73
+ """)
74
+ print(result.column_names, result.rows)
75
+ print(result.usage) # your plan, query time used, statements today
76
+ frame = result.to_pandas() # needs pandas; pyarrow for exact types
77
+ result.download("prices.parquet") # the full result, any size
78
+ ```
79
+
80
+ `python quickstart.py <credentials file>` runs a short tour.
81
+
82
+ ## What to know
83
+
84
+ - **DuckDB SQL**, one `SELECT` per request (or `EXPLAIN SELECT …` for the
85
+ plan). `client.guide()` lists the differences from Databricks SQL. Tables
86
+ answer to their bare names or to `external_data.nemweb.<table>`.
87
+ - **Times**: `SETTLEMENTDATE` and the other NEM timestamps are AEST market
88
+ time (UTC+10, no daylight saving) already. Don't shift them. "Now" in market
89
+ time is `current_timestamp + INTERVAL 10 HOUR`.
90
+ - **Long queries**: `query()` returns when the query finishes, polling if the
91
+ server needs more than ~20 s. A statement stops at 300 s. For control, use
92
+ `submit(sql)`, `status(statement_id)` and `cancel(statement_id)`.
93
+ - **Results**: up to 10,000 rows come back inline as `result.rows`; every
94
+ result is also a Parquet file at `result.parquet_url` for an hour. Over
95
+ 5,000,000 rows a statement is refused: aggregate in SQL. Statements and
96
+ results last 24 hours.
97
+ - **Retries are safe**: `query()` sends an idempotency key, so a retry never
98
+ runs a statement twice.
99
+ - **Errors** raise `QueryFailed` (a statement that ran and failed: `code` is
100
+ `QUERY_REJECTED`, `SQL_ERROR`, `QUERY_TIMEOUT`, `RESULT_TOO_LARGE` or
101
+ `CANCELLED`) or `RenovaraError` (a refused request, e.g. `DAILY_LIMIT`,
102
+ `ALLOWANCE_USED`). Each has a `docs_url`.
103
+
104
+ ## Without the client
105
+
106
+ ```bash
107
+ TOKEN=$(curl -s -u "$CLIENT_ID:$CLIENT_SECRET" -d grant_type=client_credentials \
108
+ -d scope=renovara-sql/query "$TOKEN_URL" | jq -r .access_token)
109
+ # Keep $TOKEN for its hour: each one issued is billed, so not one per query.
110
+
111
+ curl -s -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" \
112
+ -H "Idempotency-Key: $(uuidgen)" \
113
+ -d '{"sql": "SELECT count(*) FROM silver_dispatchis_reports_dispatch_price", "wait_seconds": 20}' \
114
+ "${API_URL}statements"
115
+ # -> {"statement_id": "…", "state": "SUCCEEDED", "result": {"columns": …, "rows": …}}
116
+ # or "state": "RUNNING": then GET ${API_URL}statements/<statement_id> until it isn't
117
+ ```
118
+
119
+ The API describes itself: `GET ${API_URL}` and `GET ${API_URL}openapi.json`.
@@ -0,0 +1,98 @@
1
+ # Renovara SQL API — Python client
2
+
3
+ Read-only SQL over Australian National Electricity Market (NEM) and east
4
+ coast gas market data from AEMO, from your own code. The SQL is DuckDB's.
5
+
6
+ ```bash
7
+ pip install renovara-sql
8
+ ```
9
+
10
+ ## See what the data is — no credentials needed
11
+
12
+ ```python
13
+ from renovara_sql import Client
14
+
15
+ public = Client("https://sql.api.renovara.co/v1")
16
+ print(public.guide()[:2000]) # which table answers what, and the traps
17
+ for t in public.tables(): # every table, one line each
18
+ print(t["table"], "-", t["description"][:80])
19
+ print(public.table("silver_dispatchis_reports_dispatch_price")["columns"][:5])
20
+ ```
21
+
22
+ ## Credentials
23
+
24
+ Your Renovara credentials page gives you a file:
25
+
26
+ ```json
27
+ {
28
+ "api_url": "https://sql.api.renovara.co/v1/",
29
+ "token_url": "https://auth.api.renovara.co/oauth2/token",
30
+ "client_id": "…",
31
+ "client_secret": "…",
32
+ "scope": "renovara-sql/query"
33
+ }
34
+ ```
35
+
36
+ Keep it private: anyone holding it can query as your organisation. It is an
37
+ OAuth client-credentials pair, exchanged for an access token that lasts an
38
+ hour; the client keeps the token for the hour and renews it for you.
39
+
40
+ ## Run queries
41
+
42
+ ```python
43
+ from renovara_sql import Client
44
+
45
+ client = Client.from_file("~/.config/renovara-sql/acme.json")
46
+ result = client.query("""
47
+ SELECT REGIONID, round(avg(RRP), 2) AS avg_price
48
+ FROM silver_dispatchis_reports_dispatch_price
49
+ WHERE SETTLEMENTDATE >= current_timestamp + INTERVAL 10 HOUR - INTERVAL 7 DAY
50
+ AND INTERVENTION = 0
51
+ GROUP BY 1 ORDER BY 1
52
+ """)
53
+ print(result.column_names, result.rows)
54
+ print(result.usage) # your plan, query time used, statements today
55
+ frame = result.to_pandas() # needs pandas; pyarrow for exact types
56
+ result.download("prices.parquet") # the full result, any size
57
+ ```
58
+
59
+ `python quickstart.py <credentials file>` runs a short tour.
60
+
61
+ ## What to know
62
+
63
+ - **DuckDB SQL**, one `SELECT` per request (or `EXPLAIN SELECT …` for the
64
+ plan). `client.guide()` lists the differences from Databricks SQL. Tables
65
+ answer to their bare names or to `external_data.nemweb.<table>`.
66
+ - **Times**: `SETTLEMENTDATE` and the other NEM timestamps are AEST market
67
+ time (UTC+10, no daylight saving) already. Don't shift them. "Now" in market
68
+ time is `current_timestamp + INTERVAL 10 HOUR`.
69
+ - **Long queries**: `query()` returns when the query finishes, polling if the
70
+ server needs more than ~20 s. A statement stops at 300 s. For control, use
71
+ `submit(sql)`, `status(statement_id)` and `cancel(statement_id)`.
72
+ - **Results**: up to 10,000 rows come back inline as `result.rows`; every
73
+ result is also a Parquet file at `result.parquet_url` for an hour. Over
74
+ 5,000,000 rows a statement is refused: aggregate in SQL. Statements and
75
+ results last 24 hours.
76
+ - **Retries are safe**: `query()` sends an idempotency key, so a retry never
77
+ runs a statement twice.
78
+ - **Errors** raise `QueryFailed` (a statement that ran and failed: `code` is
79
+ `QUERY_REJECTED`, `SQL_ERROR`, `QUERY_TIMEOUT`, `RESULT_TOO_LARGE` or
80
+ `CANCELLED`) or `RenovaraError` (a refused request, e.g. `DAILY_LIMIT`,
81
+ `ALLOWANCE_USED`). Each has a `docs_url`.
82
+
83
+ ## Without the client
84
+
85
+ ```bash
86
+ TOKEN=$(curl -s -u "$CLIENT_ID:$CLIENT_SECRET" -d grant_type=client_credentials \
87
+ -d scope=renovara-sql/query "$TOKEN_URL" | jq -r .access_token)
88
+ # Keep $TOKEN for its hour: each one issued is billed, so not one per query.
89
+
90
+ curl -s -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" \
91
+ -H "Idempotency-Key: $(uuidgen)" \
92
+ -d '{"sql": "SELECT count(*) FROM silver_dispatchis_reports_dispatch_price", "wait_seconds": 20}' \
93
+ "${API_URL}statements"
94
+ # -> {"statement_id": "…", "state": "SUCCEEDED", "result": {"columns": …, "rows": …}}
95
+ # or "state": "RUNNING": then GET ${API_URL}statements/<statement_id> until it isn't
96
+ ```
97
+
98
+ The API describes itself: `GET ${API_URL}` and `GET ${API_URL}openapi.json`.
@@ -0,0 +1,43 @@
1
+ # The Renovara SQL API's Python client, published to PyPI as renovara-sql.
2
+ #
3
+ # cd clients/python && python -m build # -> dist/renovara_sql-<version>*
4
+ #
5
+ # One module, standard library only; pandas/pyarrow are optional extras for
6
+ # Result.to_pandas(). The version is renovara_sql.__version__ (a test keeps
7
+ # them equal). Published by .github/workflows/publish-sql-client.yml on a
8
+ # `renovara-sql-v<version>` tag, through PyPI trusted publishing (no token).
9
+ [build-system]
10
+ requires = ["hatchling>=1.27"]
11
+ build-backend = "hatchling.build"
12
+
13
+ [project]
14
+ name = "renovara-sql"
15
+ version = "0.2.0"
16
+ description = "Python client for the Renovara SQL API: DuckDB SQL over Australian NEM and east coast gas market data from AEMO"
17
+ readme = "README.md"
18
+ license = "MIT"
19
+ license-files = ["LICENSE"]
20
+ requires-python = ">=3.9"
21
+ authors = [{ name = "Scoptico Pty Ltd (Renovara)", email = "info@renovara.co" }]
22
+ keywords = ["nem", "aemo", "electricity", "energy", "australia", "sql", "duckdb", "gas"]
23
+ classifiers = [
24
+ "Programming Language :: Python :: 3",
25
+ "Intended Audience :: Science/Research",
26
+ "Topic :: Scientific/Engineering",
27
+ "Operating System :: OS Independent",
28
+ ]
29
+ dependencies = []
30
+
31
+ [project.optional-dependencies]
32
+ pandas = ["pandas", "pyarrow"]
33
+
34
+ [project.urls]
35
+ Homepage = "https://renovara.co/sql-api"
36
+ Documentation = "https://renovara.co/how-renovara-sql-works"
37
+ "API description" = "https://sql.api.renovara.co/v1/openapi.json"
38
+
39
+ [tool.hatch.build.targets.wheel]
40
+ include = ["renovara_sql.py"]
41
+
42
+ [tool.hatch.build.targets.sdist]
43
+ include = ["renovara_sql.py", "README.md", "quickstart.py", "LICENSE"]
@@ -0,0 +1,65 @@
1
+ """Renovara SQL API quickstart.
2
+
3
+ python quickstart.py ~/.config/renovara-sql/sandbox-acme.json
4
+
5
+ Shows the things a customer does: see what the data is, run a query, and run
6
+ a long one while watching its progress.
7
+ """
8
+ import sys
9
+ import time
10
+
11
+ from renovara_sql import Client, QueryFailed
12
+
13
+ client = Client.from_file(sys.argv[1])
14
+
15
+ # 1. What can I query, and what do the columns mean? (No credentials needed.)
16
+ print("Tables:", ", ".join(t["table"] for t in client.tables()))
17
+ price = client.table("silver_dispatchis_reports_dispatch_price")
18
+ rrp = next(c for c in price["columns"] if c["name"] == "RRP")
19
+ print(f"RRP: {rrp['description'][:100]}")
20
+
21
+ # 2. A quick query: one call, result inline.
22
+ started = time.perf_counter()
23
+ result = client.query("""
24
+ SELECT REGIONID, round(avg(RRP), 2) AS avg_price, count(*) AS intervals
25
+ FROM silver_dispatchis_reports_dispatch_price
26
+ WHERE SETTLEMENTDATE >= TIMESTAMP '2026-10-20' AND SETTLEMENTDATE < TIMESTAMP '2026-10-27'
27
+ AND INTERVENTION = 0
28
+ GROUP BY 1 ORDER BY 1
29
+ """)
30
+ print(f"\nAverage price, week of 20 October ({time.perf_counter() - started:.1f} s):")
31
+ print(result.column_names)
32
+ for row in result.rows:
33
+ print(row)
34
+ if result.usage:
35
+ u = result.usage
36
+ allowed = u["query_seconds_allowed"]
37
+ print(f"Plan {u['plan']}: {u['query_seconds_used']:,} s of query time used"
38
+ + (f" of {allowed:,}" if allowed else "") + f"; {u.get('statements_today')} today")
39
+
40
+ # 3. A longer query: the same call, which polls if the server needs longer.
41
+ started = time.perf_counter()
42
+ result = client.query(
43
+ """
44
+ SELECT year(SETTLEMENTDATE) AS year, REGIONID, round(avg(TOTALDEMAND)) AS avg_demand_mw
45
+ FROM external_data.nemweb.silver_dispatchis_reports_dispatch_regionsum
46
+ WHERE INTERVENTION = 0
47
+ GROUP BY ALL ORDER BY ALL
48
+ """,
49
+ on_progress=lambda s: print(f" ... {s['state']}"),
50
+ )
51
+ print(f"\nYearly average demand, 2019-2026: {result.row_count} rows "
52
+ f"({time.perf_counter() - started:.1f} s)")
53
+
54
+ # Errors come back with a code and DuckDB's own message.
55
+ try:
56
+ client.query("DROP TABLE silver_dispatchis_reports_dispatch_price")
57
+ except QueryFailed as e:
58
+ print(f"\nRefused, as it should be: {e.code}: {e.message}")
59
+
60
+ # pandas, if installed (pyarrow gives exact types).
61
+ try:
62
+ frame = result.to_pandas()
63
+ print("\n", frame.head())
64
+ except ImportError:
65
+ print("\n(pandas not installed; skipping to_pandas())")
@@ -0,0 +1,284 @@
1
+ """Python client for the Renovara SQL API.
2
+
3
+ One file, standard library only. pandas (and pyarrow, for exact column types)
4
+ are needed only for Result.to_pandas().
5
+
6
+ from renovara_sql import Client
7
+
8
+ client = Client.from_file("~/.config/renovara-sql/acme.json")
9
+ result = client.query("SELECT REGIONID, avg(RRP) FROM silver_dispatchis_reports_dispatch_price "
10
+ "WHERE SETTLEMENTDATE >= current_timestamp + INTERVAL 10 HOUR - INTERVAL 7 DAY "
11
+ "GROUP BY 1")
12
+ print(result.rows)
13
+
14
+ Queries are DuckDB SQL, not Databricks SQL: read client.guide() first, which
15
+ lists the differences. NEM timestamps (SETTLEMENTDATE and friends) are AEST
16
+ market time already; the server's session zone is UTC so they are shown and
17
+ compared unshifted.
18
+
19
+ The guide and the tables' descriptions need no credentials:
20
+
21
+ Client("https://sql.api.renovara.co/v1").table("silver_dispatchis_reports_dispatch_price")
22
+
23
+ How a query runs: query() submits it and the server waits up to 20 s. Most
24
+ queries finish inside that and come back in one call. A longer one comes back
25
+ with a statement ID, and query() polls until it finishes. Use submit(),
26
+ status() and cancel() to do that yourself.
27
+ """
28
+ from __future__ import annotations
29
+
30
+ import base64
31
+ import io
32
+ import json
33
+ import os
34
+ import time
35
+ import urllib.error
36
+ import urllib.parse
37
+ import urllib.request
38
+ import uuid
39
+ from dataclasses import dataclass, field
40
+ from pathlib import Path
41
+
42
+ __version__ = "0.2.0"
43
+
44
+ TERMINAL = {"SUCCEEDED", "FAILED"}
45
+ # Retry-After longer than this is a limit to report, not a pause to sit out.
46
+ MAX_RETRY_WAIT_S = 60
47
+
48
+
49
+ class RenovaraError(Exception):
50
+ """A request the API refused, with its error code, message and docs link."""
51
+
52
+ def __init__(self, status: int, code: str, message: str, docs_url: str | None = None,
53
+ retry_after: float | None = None):
54
+ super().__init__(f"{code}: {message}" if code else message)
55
+ self.status, self.code, self.message = status, code, message
56
+ self.docs_url, self.retry_after = docs_url, retry_after
57
+
58
+
59
+ class QueryFailed(RenovaraError):
60
+ """The query ran and failed: rejected, a SQL error, timed out or cancelled."""
61
+
62
+ def __init__(self, statement_id: str, code: str, message: str, docs_url: str | None = None):
63
+ super().__init__(200, code, message, docs_url)
64
+ self.statement_id = statement_id
65
+
66
+
67
+ @dataclass
68
+ class Result:
69
+ statement_id: str
70
+ columns: list[dict]
71
+ row_count: int
72
+ parquet_url: str
73
+ rows: list[list] | None = None
74
+ query_ms: int | None = None
75
+ usage: dict | None = None
76
+ raw: dict = field(default_factory=dict, repr=False)
77
+
78
+ @property
79
+ def column_names(self) -> list[str]:
80
+ return [c["name"] for c in self.columns]
81
+
82
+ def download(self, path: str | os.PathLike) -> Path:
83
+ """Save the full result as Parquet. The link works for an hour."""
84
+ path = Path(path)
85
+ with urllib.request.urlopen(self.parquet_url, timeout=300) as response:
86
+ path.write_bytes(response.read())
87
+ return path
88
+
89
+ def to_pandas(self):
90
+ """The result as a DataFrame.
91
+
92
+ With pyarrow installed this reads the Parquet file, keeping exact
93
+ types. Without it, it builds the frame from the inline rows and
94
+ parses timestamp columns.
95
+ """
96
+ import pandas as pd
97
+
98
+ try:
99
+ import pyarrow # noqa: F401
100
+ except ImportError:
101
+ if self.rows is None:
102
+ raise RuntimeError(
103
+ f"This result has {self.row_count:,} rows, too many to return inline. "
104
+ "Install pyarrow to read its Parquet file.") from None
105
+ frame = pd.DataFrame(self.rows, columns=self.column_names)
106
+ for column in self.columns:
107
+ if column["type"].startswith(("TIMESTAMP", "DATE")):
108
+ frame[column["name"]] = pd.to_datetime(frame[column["name"]])
109
+ return frame
110
+ with urllib.request.urlopen(self.parquet_url, timeout=300) as response:
111
+ return pd.read_parquet(io.BytesIO(response.read()))
112
+
113
+
114
+ class Client:
115
+ def __init__(self, api_url: str, token_url: str | None = None, client_id: str | None = None,
116
+ client_secret: str | None = None, scope: str = "renovara-sql/query",
117
+ timeout: float = 30):
118
+ self.api_url = api_url.rstrip("/") + "/"
119
+ self.token_url = token_url
120
+ self.client_id = client_id
121
+ self._client_secret = client_secret
122
+ self.scope = scope
123
+ self.timeout = timeout
124
+ self._token: str | None = None
125
+ self._token_expires = 0.0
126
+
127
+ @classmethod
128
+ def from_file(cls, path: str | os.PathLike) -> "Client":
129
+ """Read the credentials file from your Renovara credentials page."""
130
+ data = json.loads(Path(path).expanduser().read_text())
131
+ return cls(data["api_url"], data["token_url"], data["client_id"],
132
+ data["client_secret"], data.get("scope", "renovara-sql/query"))
133
+
134
+ @classmethod
135
+ def from_env(cls) -> "Client":
136
+ """RENOVARA_SQL_API_URL, _TOKEN_URL, _CLIENT_ID and _CLIENT_SECRET."""
137
+ e = os.environ
138
+ return cls(e["RENOVARA_SQL_API_URL"], e["RENOVARA_SQL_TOKEN_URL"],
139
+ e["RENOVARA_SQL_CLIENT_ID"], e["RENOVARA_SQL_CLIENT_SECRET"])
140
+
141
+ # -- auth ------------------------------------------------------------
142
+
143
+ def _access_token(self) -> str:
144
+ """An OAuth client-credentials token, kept for its hour.
145
+
146
+ Renewed a minute before it expires, not per request: each token
147
+ Renovara's sign-in service issues is billed, so a fresh one per query
148
+ would cost more than the query.
149
+ """
150
+ if self._token and time.time() < self._token_expires - 60:
151
+ return self._token
152
+ if not (self.token_url and self.client_id and self._client_secret):
153
+ raise RenovaraError(401, "UNAUTHORIZED",
154
+ "Running queries needs credentials: Client.from_file(...) with "
155
+ "the file from your Renovara credentials page.")
156
+ basic = base64.b64encode(f"{self.client_id}:{self._client_secret}".encode()).decode()
157
+ request = urllib.request.Request(
158
+ self.token_url, method="POST",
159
+ data=urllib.parse.urlencode({"grant_type": "client_credentials",
160
+ "scope": self.scope}).encode(),
161
+ headers={"Authorization": f"Basic {basic}",
162
+ "Content-Type": "application/x-www-form-urlencoded"})
163
+ try:
164
+ with urllib.request.urlopen(request, timeout=self.timeout) as response:
165
+ body = json.load(response)
166
+ except urllib.error.HTTPError as e:
167
+ raise RenovaraError(e.code, "AUTH_FAILED",
168
+ f"Could not get an access token: {e.read().decode()[:300]}") from None
169
+ self._token = body["access_token"]
170
+ self._token_expires = time.time() + int(body.get("expires_in", 3600))
171
+ return self._token
172
+
173
+ # -- HTTP ------------------------------------------------------------
174
+
175
+ def _call(self, method: str, path: str, body: dict | None = None, timeout: float | None = None,
176
+ headers: dict | None = None, auth: bool = True, text: bool = False):
177
+ for attempt in range(5):
178
+ sent = {"User-Agent": f"renovara-sql-python/{__version__}", **(headers or {})}
179
+ if body is not None:
180
+ sent["Content-Type"] = "application/json"
181
+ if auth:
182
+ sent["Authorization"] = f"Bearer {self._access_token()}"
183
+ request = urllib.request.Request(
184
+ self.api_url + path, method=method,
185
+ data=json.dumps(body).encode() if body is not None else None, headers=sent)
186
+ try:
187
+ with urllib.request.urlopen(request, timeout=timeout or self.timeout) as response:
188
+ return response.read().decode() if text else json.load(response)
189
+ except urllib.error.HTTPError as e:
190
+ raw = e.read().decode(errors="replace")
191
+ try:
192
+ error = json.loads(raw).get("error") or {}
193
+ except ValueError:
194
+ error = {}
195
+ error = error if isinstance(error, dict) else {}
196
+ retry_after = float(e.headers.get("Retry-After") or 0) or None
197
+ if e.code == 401 and auth and attempt == 0:
198
+ self._token = None # expired or revoked mid-flight: renew once
199
+ continue
200
+ retryable = e.code in (502, 503, 504) or (
201
+ e.code == 429 and (retry_after or 1) <= MAX_RETRY_WAIT_S)
202
+ if retryable and attempt < 4:
203
+ time.sleep(retry_after or 2 ** attempt)
204
+ continue
205
+ raise RenovaraError(e.code, error.get("code", "HTTP_ERROR"),
206
+ error.get("message", raw[:300] or e.reason),
207
+ error.get("docs_url"), retry_after) from None
208
+ raise RenovaraError(429, "THROTTLED", "Too many requests; try again shortly.")
209
+
210
+ # -- what the data is (no credentials needed) ----------------------------
211
+
212
+ def guide(self, domain: str = "nem") -> str:
213
+ """The analyst guide (markdown): which table answers what, the traps,
214
+ worked SQL, and how DuckDB differs from the Databricks SQL it quotes.
215
+ domain is "nem" (electricity) or "gas"."""
216
+ return self._call("GET", f"guide?domain={urllib.parse.quote(domain)}", auth=False, text=True)
217
+
218
+ def tables(self) -> list[dict]:
219
+ """Every table you can query, each with a one-line description."""
220
+ return self._call("GET", "tables", auth=False)["tables"]
221
+
222
+ def table(self, name: str) -> dict:
223
+ """A table's columns with their meaning, primary key and measured coverage."""
224
+ return self._call("GET", f"tables/{urllib.parse.quote(name)}", auth=False)
225
+
226
+ def reference(self, path: str, domain: str = "nem") -> dict:
227
+ """A file the guide points to, by its references/ path."""
228
+ query = urllib.parse.urlencode({"path": path, "domain": domain})
229
+ return self._call("GET", f"reference?{query}", auth=False)
230
+
231
+ # -- running SQL -----------------------------------------------------
232
+
233
+ def submit(self, sql: str, wait_seconds: float = 0, idempotency_key: str | None = None) -> dict:
234
+ """Start a query. With wait_seconds > 0 (max 25) the server waits for it first.
235
+
236
+ With an idempotency_key, sending the same key again returns the first
237
+ statement instead of running the query twice.
238
+ """
239
+ headers = {"Idempotency-Key": idempotency_key} if idempotency_key else None
240
+ return self._call("POST", "statements", {"sql": sql, "wait_seconds": wait_seconds},
241
+ timeout=max(self.timeout, wait_seconds + 10), headers=headers)
242
+
243
+ def status(self, statement_id: str) -> dict:
244
+ """A statement's state, and its result once it has SUCCEEDED."""
245
+ return self._call("GET", f"statements/{statement_id}")
246
+
247
+ def cancel(self, statement_id: str) -> dict:
248
+ """Stop a pending or running statement. Its state becomes FAILED, CANCELLED."""
249
+ return self._call("POST", f"statements/{statement_id}/cancel", {})
250
+
251
+ def query(self, sql: str, timeout: float = 900, on_progress=None) -> Result:
252
+ """Run a query and return its result, polling if it takes a while.
253
+
254
+ Safe to retry: the submit carries an idempotency key, so a retry after
255
+ a dropped connection returns the same statement. on_progress(statement)
256
+ is called after each poll, if given.
257
+ """
258
+ statement = self.submit(sql, wait_seconds=20, idempotency_key=uuid.uuid4().hex)
259
+ usage = statement.get("usage")
260
+ deadline = time.monotonic() + timeout
261
+ pause = 1.0
262
+ while statement["state"] not in TERMINAL:
263
+ if time.monotonic() > deadline:
264
+ raise TimeoutError(f"Statement {statement['statement_id']} still "
265
+ f"{statement['state']} after {timeout:.0f} s; "
266
+ "poll it later with status(), or cancel() it.")
267
+ time.sleep(pause)
268
+ pause = min(pause * 1.5, 5.0)
269
+ statement = self.status(statement["statement_id"])
270
+ if on_progress:
271
+ on_progress(statement)
272
+ return self._result(statement, usage)
273
+
274
+ @staticmethod
275
+ def _result(statement: dict, usage: dict | None = None) -> Result:
276
+ if statement["state"] == "FAILED":
277
+ error = statement.get("error") or {}
278
+ raise QueryFailed(statement["statement_id"], error.get("code", "FAILED"),
279
+ error.get("message", "The query failed."), error.get("docs_url"))
280
+ result = statement["result"]
281
+ return Result(statement_id=statement["statement_id"], columns=result["columns"],
282
+ row_count=result["row_count"], parquet_url=result["parquet_url"],
283
+ rows=result.get("rows"), query_ms=statement.get("query_ms"),
284
+ usage=usage or statement.get("usage"), raw=statement)