onehouse-python-sdk 0.2.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/PKG-INFO +136 -4
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/README.md +134 -2
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/pyproject.toml +1 -1
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/src/onehouse_python_sdk/__init__.py +1 -1
- onehouse_python_sdk-0.4.0/src/onehouse_python_sdk/_oauth.py +293 -0
- onehouse_python_sdk-0.4.0/src/onehouse_python_sdk/_session.py +169 -0
- onehouse_python_sdk-0.4.0/src/onehouse_python_sdk/ai_gateway/__init__.py +29 -0
- onehouse_python_sdk-0.4.0/src/onehouse_python_sdk/ai_gateway/discovery.py +143 -0
- onehouse_python_sdk-0.4.0/src/onehouse_python_sdk/ai_gateway/exceptions.py +11 -0
- onehouse_python_sdk-0.4.0/src/onehouse_python_sdk/ai_gateway/tokens.py +223 -0
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/src/onehouse_python_sdk/resources/client.py +621 -12
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/src/onehouse_python_sdk/resources/credentials.py +41 -12
- onehouse_python_sdk-0.4.0/src/onehouse_python_sdk/resources/oauth.py +258 -0
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/src/onehouse_python_sdk/resources/sql/builder.py +8 -0
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/src/onehouse_python_sdk/resources/sql/commands.py +632 -19
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/src/onehouse_python_sdk/resources/transport.py +51 -9
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/.gitignore +0 -0
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/src/onehouse_python_sdk/base.py +0 -0
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/src/onehouse_python_sdk/exceptions.py +0 -0
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/src/onehouse_python_sdk/lakebase/__init__.py +0 -0
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/src/onehouse_python_sdk/lakebase/auth/__init__.py +0 -0
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/src/onehouse_python_sdk/lakebase/auth/_ssl.py +0 -0
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/src/onehouse_python_sdk/lakebase/auth/azure_oauth.py +0 -0
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/src/onehouse_python_sdk/lakebase/auth/browser_auth.py +0 -0
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/src/onehouse_python_sdk/lakebase/auth/local_auth_server.py +0 -0
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/src/onehouse_python_sdk/lakebase/auth/oidc_device_flow.py +0 -0
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/src/onehouse_python_sdk/lakebase/auth/saml.py +0 -0
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/src/onehouse_python_sdk/lakebase/client.py +0 -0
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/src/onehouse_python_sdk/lakebase/connection.py +0 -0
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/src/onehouse_python_sdk/resources/__init__.py +0 -0
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/src/onehouse_python_sdk/resources/exceptions.py +0 -0
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/src/onehouse_python_sdk/resources/sql/__init__.py +0 -0
- {onehouse_python_sdk-0.2.0 → onehouse_python_sdk-0.4.0}/src/onehouse_python_sdk/resources/status.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: onehouse-python-sdk
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Onehouse Python SDK — modular data-plane clients for LakeBase and beyond
|
|
5
5
|
Project-URL: Homepage, https://github.com/onehouseinc/onehouse-python-sdk
|
|
6
6
|
Project-URL: Repository, https://github.com/onehouseinc/onehouse-python-sdk
|
|
@@ -174,7 +174,7 @@ print(result.api_response) # raw API payload
|
|
|
174
174
|
Credentials resolve from three sources, in order (highest wins):
|
|
175
175
|
|
|
176
176
|
1. **Explicit constructor arguments** (shown above).
|
|
177
|
-
2. **Environment variables** — `ONEHOUSE_ACCOUNT_UID`, `ONEHOUSE_PROJECT_UID`, `ONEHOUSE_API_KEY`, `ONEHOUSE_API_SECRET`, `ONEHOUSE_LINK_UID`, `ONEHOUSE_REGION`, `ONEHOUSE_USER_UID`. Optional: `ONEHOUSE_BASE_URL`, `ONEHOUSE_PROFILE`, `ONEHOUSE_CREDENTIALS_FILE`.
|
|
177
|
+
2. **Environment variables** — `ONEHOUSE_ACCOUNT_UID`, `ONEHOUSE_PROJECT_UID`, `ONEHOUSE_API_KEY`, `ONEHOUSE_API_SECRET`, `ONEHOUSE_LINK_UID`, `ONEHOUSE_REGION`, `ONEHOUSE_USER_UID`. Optional: `ONEHOUSE_BASE_URL`, `ONEHOUSE_PROFILE`, `ONEHOUSE_CREDENTIALS_FILE`. `ONEHOUSE_BASE_URL` is the API host (default `https://api.onehouse.ai`); a trailing `/v1` is dropped, since every request adds it.
|
|
178
178
|
3. **INI credentials file** at `~/.onehouse/credentials` (override with `ONEHOUSE_CREDENTIALS_FILE`).
|
|
179
179
|
|
|
180
180
|
```ini
|
|
@@ -202,6 +202,54 @@ client = OnehouseResources(profile="staging")
|
|
|
202
202
|
|
|
203
203
|
Missing fields produce an `AuthError` listing which fields are unset and how to supply them. A world-readable credentials file triggers a warning — `chmod 600 ~/.onehouse/credentials`.
|
|
204
204
|
|
|
205
|
+
### OAuth sign-in instead of an API key
|
|
206
|
+
|
|
207
|
+
A person can sign in through the browser instead of holding an API key. external-api runs an OAuth 2.1 authorization server (authorization code + PKCE). Its access token is a Firebase ID token for the signed-in user and lasts an hour. Its refresh token rotates on every use and lasts 30 days. The org must have `enable_external_api_bearer_auth` on.
|
|
208
|
+
|
|
209
|
+
```python
|
|
210
|
+
from pathlib import Path
|
|
211
|
+
from onehouse_python_sdk._session import FileSessionStore
|
|
212
|
+
from onehouse_python_sdk.resources import oauth
|
|
213
|
+
|
|
214
|
+
store = FileSessionStore(Path.home() / ".onehouse" / "session.json")
|
|
215
|
+
oauth.login("https://api.onehouse.ai", store) # opens the browser, catches the code on 127.0.0.1
|
|
216
|
+
|
|
217
|
+
client = OnehouseResources(
|
|
218
|
+
account_uid="...", project_uid="...", link_uid="...", region="us-west-2",
|
|
219
|
+
token_source=oauth.FirebaseRefreshTokenSource("https://api.onehouse.ai", store),
|
|
220
|
+
)
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
- With a `token_source`, `api_key`, `api_secret` and `user_uid` are not required. Calls send `Authorization: Bearer` and never the key pair or user id: external-api takes the user from the token. `link_uid` and `region` are still required, because statements that reach a data plane need them.
|
|
224
|
+
- The token source refreshes 60 s before expiry and once after a 401. Refresh runs under the store's cross-process lock and re-reads the store first. Concurrent processes therefore share one rotation; presenting a rotated refresh token twice revokes the whole session.
|
|
225
|
+
- `oauth.logout(base_url, store)` revokes the refresh token and deletes the session. An access token already issued stays valid until it expires, at most an hour.
|
|
226
|
+
- `FileSessionStore` writes one `0600` file. The `oh` CLI keeps the refresh token in the OS keychain instead and falls back to this file.
|
|
227
|
+
- The client is the static public client `oh-cli`, registered for `http://127.0.0.1/oauth/callback` on any port.
|
|
228
|
+
|
|
229
|
+
## AI Gateway tokens
|
|
230
|
+
|
|
231
|
+
`onehouse_python_sdk.ai_gateway` gets tokens for a data plane's AI Gateway. It uses the standard library only, and `resources` never imports it. Every call needs the VPN or the customer network.
|
|
232
|
+
|
|
233
|
+
```python
|
|
234
|
+
from onehouse_python_sdk import ai_gateway
|
|
235
|
+
|
|
236
|
+
gateway = "https://ai-gateway-<id>.dp.onehouse.ai"
|
|
237
|
+
|
|
238
|
+
# A service principal: client_credentials with its API key pair. 300 s tokens, re-minted.
|
|
239
|
+
source = ai_gateway.ServicePrincipalTokenSource(gateway, api_key, api_secret)
|
|
240
|
+
|
|
241
|
+
# A person: sign in once on Keycloak's `onehouse-cli` client, then refresh.
|
|
242
|
+
ai_gateway.login(gateway, store)
|
|
243
|
+
source = ai_gateway.KeycloakSessionTokenSource(gateway, store)
|
|
244
|
+
|
|
245
|
+
token = source.get() # send as X-Onehouse-Authorization (or Authorization on /mcp)
|
|
246
|
+
token = source.get(force=True) # after the gateway answers 401
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
- `ai_gateway.discover(gateway)` follows the gateway's own discovery chain: RFC 9728 protected-resource metadata, then RFC 8414 or OIDC metadata for the issuer it names. It checks that the issuer matches and that the audience is on the gateway's host. Nothing is derived from the hostname or the link id. A gateway with auth off raises `AuthDisabledError`.
|
|
250
|
+
- The human session stores only the refresh token. The access token (7 days on that client) stays in memory and is never written to disk.
|
|
251
|
+
- `ai_gateway.logout(gateway, store)` ends the Keycloak session best effort and always deletes the stored token.
|
|
252
|
+
|
|
205
253
|
### Three ways to run a command
|
|
206
254
|
|
|
207
255
|
```python
|
|
@@ -256,7 +304,91 @@ client.create_flow(
|
|
|
256
304
|
)
|
|
257
305
|
```
|
|
258
306
|
|
|
259
|
-
|
|
307
|
+
#### `CREATE FLOW` — `WITH` options reference
|
|
308
|
+
|
|
309
|
+
Source-specific options and advanced configs are passed via the `options` dict. All keys and values are strings.
|
|
310
|
+
|
|
311
|
+
**Kafka source**
|
|
312
|
+
|
|
313
|
+
| Key | Description |
|
|
314
|
+
|-----|-------------|
|
|
315
|
+
| `kafka.topic.name` | Name of the source Kafka topic. |
|
|
316
|
+
| `kafka.startingOffsets` | `'earliest'` (default) or `'latest'`. |
|
|
317
|
+
|
|
318
|
+
**S3 source**
|
|
319
|
+
|
|
320
|
+
| Key | Description |
|
|
321
|
+
|-----|-------------|
|
|
322
|
+
| `s3.folder.uri` | `s3://bucket/prefix/` folder to ingest from. |
|
|
323
|
+
| `s3.file.format` | `AVRO`, `JSON`, `CSV`, `ORC`, `PARQUET`, or `XML`. |
|
|
324
|
+
| `s3.file.extension` | File extension, e.g. `'.parquet'`, `'.jsonl'`. |
|
|
325
|
+
| `s3.source.bootstrap` | `'TRUE'` or `'FALSE'`. |
|
|
326
|
+
| `s3.source.if_infer_fields_from_source_path` | `'TRUE'` to extract field values from path segments like `field=value`. |
|
|
327
|
+
| `s3.source.fields_to_infer` | Comma-separated field names to extract from the path, e.g. `'schema,table,date'`. |
|
|
328
|
+
| `s3.source.file_path_pattern` | Optional glob applied under `s3.folder.uri`. |
|
|
329
|
+
| `s3.file.csv.header` | `'TRUE'` if the first row is a header. |
|
|
330
|
+
| `s3.file.json.multiline` | `'TRUE'` for single-record-per-file (pretty-printed) JSON. |
|
|
331
|
+
| `s3.file.xml.row.tag` | XML element that delimits one record, e.g. `'Product'`. |
|
|
332
|
+
| `s3.file.xml.value.tag` | Synthetic field name for element text content with attributes (default `'_VALUE'`). |
|
|
333
|
+
| `s3.file.xml.null.value` | String token treated as `null`. |
|
|
334
|
+
| `s3.file.xml.ignore.namespace` | `'TRUE'` to strip namespace prefixes from element names. |
|
|
335
|
+
|
|
336
|
+
**GCS source** — same as S3 with `gcs.` prefix (e.g. `gcs.folder.uri`, `gcs.file.format`, etc.).
|
|
337
|
+
|
|
338
|
+
**Onehouse table source**
|
|
339
|
+
|
|
340
|
+
| Key | Description |
|
|
341
|
+
|-----|-------------|
|
|
342
|
+
| `source.table.name` | Name of the source Onehouse table. |
|
|
343
|
+
| `source.table.database` | Database containing the source table. |
|
|
344
|
+
| `source.table.lake` | Lake containing the source table. |
|
|
345
|
+
|
|
346
|
+
**Postgres source**
|
|
347
|
+
|
|
348
|
+
| Key | Description |
|
|
349
|
+
|-----|-------------|
|
|
350
|
+
| `postgres.table.name` | Postgres table to ingest. |
|
|
351
|
+
| `postgres.schema.name` | Postgres schema name. |
|
|
352
|
+
|
|
353
|
+
**MySQL source**
|
|
354
|
+
|
|
355
|
+
| Key | Description |
|
|
356
|
+
|-----|-------------|
|
|
357
|
+
| `mysql.table.name` | MySQL table to ingest. |
|
|
358
|
+
|
|
359
|
+
**Oracle source**
|
|
360
|
+
|
|
361
|
+
| Key | Description |
|
|
362
|
+
|-----|-------------|
|
|
363
|
+
| `oracle.table.name` | Oracle table to ingest. |
|
|
364
|
+
| `oracle.schema.name` | Oracle schema (owner). |
|
|
365
|
+
|
|
366
|
+
**Schema registry** (Kafka sources)
|
|
367
|
+
|
|
368
|
+
| Key | Description |
|
|
369
|
+
|-----|-------------|
|
|
370
|
+
| `schema.registry.type` | `'glue'`, `'confluent'`, `'jar'`, or `'file'`. |
|
|
371
|
+
| `schema.registry.glue.name` | Glue registry name. |
|
|
372
|
+
| `schema.registry.confluent.servers` | Confluent Schema Registry URL. |
|
|
373
|
+
| `schema.registry.confluent.key` | Confluent API key. |
|
|
374
|
+
| `schema.registry.confluent.secret` | Confluent API secret. |
|
|
375
|
+
| `schema.registry.proto.jar.location` | S3 path to the proto schema JAR. |
|
|
376
|
+
| `schema.registry.file.base.path` | Base path for file-based schema registry. |
|
|
377
|
+
| `schema.registry.file.full.path` | Full path to a specific schema file. |
|
|
378
|
+
|
|
379
|
+
**Cross-source advanced configs**
|
|
380
|
+
|
|
381
|
+
| Key | Description |
|
|
382
|
+
|-----|-------------|
|
|
383
|
+
| `table.configured.base.path` | Custom storage location for the table (disables Clean & Restart). |
|
|
384
|
+
| `table.partition.style` | `'default'` or `'hive'`. |
|
|
385
|
+
| `flow.delayThreshold.numSyncIntervals` | Multiplied by sync frequency to set the delay threshold. `'0'` disables the Delayed state. |
|
|
386
|
+
| `flow.deduplicationPolicy` | `'none'` (default) or `'drop'` — deduplicates append-only flows by record key. |
|
|
387
|
+
|
|
388
|
+
```python
|
|
389
|
+
```
|
|
390
|
+
|
|
391
|
+
Groups, grants, roles and access-control policies have typed helpers too, for example `client.create_group("data_eng")`, `client.grant_privilege("SELECT", on="TABLE", name="lake.db.table", principal_type="GROUP", principal="data_eng")`, `client.grant_member(on="GROUP", name="data_eng", principal_type="USER", principal="a@acme.com")` and `client.show_policies("my_catalog")`. Anything without a helper still works through `client.execute(...)`.
|
|
260
392
|
|
|
261
393
|
#### `unsafe_raw` escape hatch
|
|
262
394
|
|
|
@@ -142,7 +142,7 @@ print(result.api_response) # raw API payload
|
|
|
142
142
|
Credentials resolve from three sources, in order (highest wins):
|
|
143
143
|
|
|
144
144
|
1. **Explicit constructor arguments** (shown above).
|
|
145
|
-
2. **Environment variables** — `ONEHOUSE_ACCOUNT_UID`, `ONEHOUSE_PROJECT_UID`, `ONEHOUSE_API_KEY`, `ONEHOUSE_API_SECRET`, `ONEHOUSE_LINK_UID`, `ONEHOUSE_REGION`, `ONEHOUSE_USER_UID`. Optional: `ONEHOUSE_BASE_URL`, `ONEHOUSE_PROFILE`, `ONEHOUSE_CREDENTIALS_FILE`.
|
|
145
|
+
2. **Environment variables** — `ONEHOUSE_ACCOUNT_UID`, `ONEHOUSE_PROJECT_UID`, `ONEHOUSE_API_KEY`, `ONEHOUSE_API_SECRET`, `ONEHOUSE_LINK_UID`, `ONEHOUSE_REGION`, `ONEHOUSE_USER_UID`. Optional: `ONEHOUSE_BASE_URL`, `ONEHOUSE_PROFILE`, `ONEHOUSE_CREDENTIALS_FILE`. `ONEHOUSE_BASE_URL` is the API host (default `https://api.onehouse.ai`); a trailing `/v1` is dropped, since every request adds it.
|
|
146
146
|
3. **INI credentials file** at `~/.onehouse/credentials` (override with `ONEHOUSE_CREDENTIALS_FILE`).
|
|
147
147
|
|
|
148
148
|
```ini
|
|
@@ -170,6 +170,54 @@ client = OnehouseResources(profile="staging")
|
|
|
170
170
|
|
|
171
171
|
Missing fields produce an `AuthError` listing which fields are unset and how to supply them. A world-readable credentials file triggers a warning — `chmod 600 ~/.onehouse/credentials`.
|
|
172
172
|
|
|
173
|
+
### OAuth sign-in instead of an API key
|
|
174
|
+
|
|
175
|
+
A person can sign in through the browser instead of holding an API key. external-api runs an OAuth 2.1 authorization server (authorization code + PKCE). Its access token is a Firebase ID token for the signed-in user and lasts an hour. Its refresh token rotates on every use and lasts 30 days. The org must have `enable_external_api_bearer_auth` on.
|
|
176
|
+
|
|
177
|
+
```python
|
|
178
|
+
from pathlib import Path
|
|
179
|
+
from onehouse_python_sdk._session import FileSessionStore
|
|
180
|
+
from onehouse_python_sdk.resources import oauth
|
|
181
|
+
|
|
182
|
+
store = FileSessionStore(Path.home() / ".onehouse" / "session.json")
|
|
183
|
+
oauth.login("https://api.onehouse.ai", store) # opens the browser, catches the code on 127.0.0.1
|
|
184
|
+
|
|
185
|
+
client = OnehouseResources(
|
|
186
|
+
account_uid="...", project_uid="...", link_uid="...", region="us-west-2",
|
|
187
|
+
token_source=oauth.FirebaseRefreshTokenSource("https://api.onehouse.ai", store),
|
|
188
|
+
)
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
- With a `token_source`, `api_key`, `api_secret` and `user_uid` are not required. Calls send `Authorization: Bearer` and never the key pair or user id: external-api takes the user from the token. `link_uid` and `region` are still required, because statements that reach a data plane need them.
|
|
192
|
+
- The token source refreshes 60 s before expiry and once after a 401. Refresh runs under the store's cross-process lock and re-reads the store first. Concurrent processes therefore share one rotation; presenting a rotated refresh token twice revokes the whole session.
|
|
193
|
+
- `oauth.logout(base_url, store)` revokes the refresh token and deletes the session. An access token already issued stays valid until it expires, at most an hour.
|
|
194
|
+
- `FileSessionStore` writes one `0600` file. The `oh` CLI keeps the refresh token in the OS keychain instead and falls back to this file.
|
|
195
|
+
- The client is the static public client `oh-cli`, registered for `http://127.0.0.1/oauth/callback` on any port.
|
|
196
|
+
|
|
197
|
+
## AI Gateway tokens
|
|
198
|
+
|
|
199
|
+
`onehouse_python_sdk.ai_gateway` gets tokens for a data plane's AI Gateway. It uses the standard library only, and `resources` never imports it. Every call needs the VPN or the customer network.
|
|
200
|
+
|
|
201
|
+
```python
|
|
202
|
+
from onehouse_python_sdk import ai_gateway
|
|
203
|
+
|
|
204
|
+
gateway = "https://ai-gateway-<id>.dp.onehouse.ai"
|
|
205
|
+
|
|
206
|
+
# A service principal: client_credentials with its API key pair. 300 s tokens, re-minted.
|
|
207
|
+
source = ai_gateway.ServicePrincipalTokenSource(gateway, api_key, api_secret)
|
|
208
|
+
|
|
209
|
+
# A person: sign in once on Keycloak's `onehouse-cli` client, then refresh.
|
|
210
|
+
ai_gateway.login(gateway, store)
|
|
211
|
+
source = ai_gateway.KeycloakSessionTokenSource(gateway, store)
|
|
212
|
+
|
|
213
|
+
token = source.get() # send as X-Onehouse-Authorization (or Authorization on /mcp)
|
|
214
|
+
token = source.get(force=True) # after the gateway answers 401
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
- `ai_gateway.discover(gateway)` follows the gateway's own discovery chain: RFC 9728 protected-resource metadata, then RFC 8414 or OIDC metadata for the issuer it names. It checks that the issuer matches and that the audience is on the gateway's host. Nothing is derived from the hostname or the link id. A gateway with auth off raises `AuthDisabledError`.
|
|
218
|
+
- The human session stores only the refresh token. The access token (7 days on that client) stays in memory and is never written to disk.
|
|
219
|
+
- `ai_gateway.logout(gateway, store)` ends the Keycloak session best effort and always deletes the stored token.
|
|
220
|
+
|
|
173
221
|
### Three ways to run a command
|
|
174
222
|
|
|
175
223
|
```python
|
|
@@ -224,7 +272,91 @@ client.create_flow(
|
|
|
224
272
|
)
|
|
225
273
|
```
|
|
226
274
|
|
|
227
|
-
|
|
275
|
+
#### `CREATE FLOW` — `WITH` options reference
|
|
276
|
+
|
|
277
|
+
Source-specific options and advanced configs are passed via the `options` dict. All keys and values are strings.
|
|
278
|
+
|
|
279
|
+
**Kafka source**
|
|
280
|
+
|
|
281
|
+
| Key | Description |
|
|
282
|
+
|-----|-------------|
|
|
283
|
+
| `kafka.topic.name` | Name of the source Kafka topic. |
|
|
284
|
+
| `kafka.startingOffsets` | `'earliest'` (default) or `'latest'`. |
|
|
285
|
+
|
|
286
|
+
**S3 source**
|
|
287
|
+
|
|
288
|
+
| Key | Description |
|
|
289
|
+
|-----|-------------|
|
|
290
|
+
| `s3.folder.uri` | `s3://bucket/prefix/` folder to ingest from. |
|
|
291
|
+
| `s3.file.format` | `AVRO`, `JSON`, `CSV`, `ORC`, `PARQUET`, or `XML`. |
|
|
292
|
+
| `s3.file.extension` | File extension, e.g. `'.parquet'`, `'.jsonl'`. |
|
|
293
|
+
| `s3.source.bootstrap` | `'TRUE'` or `'FALSE'`. |
|
|
294
|
+
| `s3.source.if_infer_fields_from_source_path` | `'TRUE'` to extract field values from path segments like `field=value`. |
|
|
295
|
+
| `s3.source.fields_to_infer` | Comma-separated field names to extract from the path, e.g. `'schema,table,date'`. |
|
|
296
|
+
| `s3.source.file_path_pattern` | Optional glob applied under `s3.folder.uri`. |
|
|
297
|
+
| `s3.file.csv.header` | `'TRUE'` if the first row is a header. |
|
|
298
|
+
| `s3.file.json.multiline` | `'TRUE'` for single-record-per-file (pretty-printed) JSON. |
|
|
299
|
+
| `s3.file.xml.row.tag` | XML element that delimits one record, e.g. `'Product'`. |
|
|
300
|
+
| `s3.file.xml.value.tag` | Synthetic field name for element text content with attributes (default `'_VALUE'`). |
|
|
301
|
+
| `s3.file.xml.null.value` | String token treated as `null`. |
|
|
302
|
+
| `s3.file.xml.ignore.namespace` | `'TRUE'` to strip namespace prefixes from element names. |
|
|
303
|
+
|
|
304
|
+
**GCS source** — same as S3 with `gcs.` prefix (e.g. `gcs.folder.uri`, `gcs.file.format`, etc.).
|
|
305
|
+
|
|
306
|
+
**Onehouse table source**
|
|
307
|
+
|
|
308
|
+
| Key | Description |
|
|
309
|
+
|-----|-------------|
|
|
310
|
+
| `source.table.name` | Name of the source Onehouse table. |
|
|
311
|
+
| `source.table.database` | Database containing the source table. |
|
|
312
|
+
| `source.table.lake` | Lake containing the source table. |
|
|
313
|
+
|
|
314
|
+
**Postgres source**
|
|
315
|
+
|
|
316
|
+
| Key | Description |
|
|
317
|
+
|-----|-------------|
|
|
318
|
+
| `postgres.table.name` | Postgres table to ingest. |
|
|
319
|
+
| `postgres.schema.name` | Postgres schema name. |
|
|
320
|
+
|
|
321
|
+
**MySQL source**
|
|
322
|
+
|
|
323
|
+
| Key | Description |
|
|
324
|
+
|-----|-------------|
|
|
325
|
+
| `mysql.table.name` | MySQL table to ingest. |
|
|
326
|
+
|
|
327
|
+
**Oracle source**
|
|
328
|
+
|
|
329
|
+
| Key | Description |
|
|
330
|
+
|-----|-------------|
|
|
331
|
+
| `oracle.table.name` | Oracle table to ingest. |
|
|
332
|
+
| `oracle.schema.name` | Oracle schema (owner). |
|
|
333
|
+
|
|
334
|
+
**Schema registry** (Kafka sources)
|
|
335
|
+
|
|
336
|
+
| Key | Description |
|
|
337
|
+
|-----|-------------|
|
|
338
|
+
| `schema.registry.type` | `'glue'`, `'confluent'`, `'jar'`, or `'file'`. |
|
|
339
|
+
| `schema.registry.glue.name` | Glue registry name. |
|
|
340
|
+
| `schema.registry.confluent.servers` | Confluent Schema Registry URL. |
|
|
341
|
+
| `schema.registry.confluent.key` | Confluent API key. |
|
|
342
|
+
| `schema.registry.confluent.secret` | Confluent API secret. |
|
|
343
|
+
| `schema.registry.proto.jar.location` | S3 path to the proto schema JAR. |
|
|
344
|
+
| `schema.registry.file.base.path` | Base path for file-based schema registry. |
|
|
345
|
+
| `schema.registry.file.full.path` | Full path to a specific schema file. |
|
|
346
|
+
|
|
347
|
+
**Cross-source advanced configs**
|
|
348
|
+
|
|
349
|
+
| Key | Description |
|
|
350
|
+
|-----|-------------|
|
|
351
|
+
| `table.configured.base.path` | Custom storage location for the table (disables Clean & Restart). |
|
|
352
|
+
| `table.partition.style` | `'default'` or `'hive'`. |
|
|
353
|
+
| `flow.delayThreshold.numSyncIntervals` | Multiplied by sync frequency to set the delay threshold. `'0'` disables the Delayed state. |
|
|
354
|
+
| `flow.deduplicationPolicy` | `'none'` (default) or `'drop'` — deduplicates append-only flows by record key. |
|
|
355
|
+
|
|
356
|
+
```python
|
|
357
|
+
```
|
|
358
|
+
|
|
359
|
+
Groups, grants, roles and access-control policies have typed helpers too, for example `client.create_group("data_eng")`, `client.grant_privilege("SELECT", on="TABLE", name="lake.db.table", principal_type="GROUP", principal="data_eng")`, `client.grant_member(on="GROUP", name="data_eng", principal_type="USER", principal="a@acme.com")` and `client.show_policies("my_catalog")`. Anything without a helper still works through `client.execute(...)`.
|
|
228
360
|
|
|
229
361
|
#### `unsafe_raw` escape hatch
|
|
230
362
|
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "onehouse-python-sdk"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.4.0"
|
|
8
8
|
description = "Onehouse Python SDK — modular data-plane clients for LakeBase and beyond"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = { text = "Proprietary" }
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
"""Loopback PKCE sign-in and token-endpoint calls, shared by every OAuth login.
|
|
2
|
+
|
|
3
|
+
Standard library only, and outside every subsystem, so both the control-plane
|
|
4
|
+
login (``resources.oauth``, external-api's OAuth server) and the data-plane
|
|
5
|
+
AI-Gateway login (``ai_gateway``, Keycloak) use it without importing each other.
|
|
6
|
+
|
|
7
|
+
The browser only ever carries an authorization code, which is worthless
|
|
8
|
+
without the PKCE verifier that never leaves this process (RFC 7636, RFC 8252).
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import base64
|
|
14
|
+
import hashlib
|
|
15
|
+
import http.server
|
|
16
|
+
import json
|
|
17
|
+
import secrets
|
|
18
|
+
import ssl
|
|
19
|
+
import time
|
|
20
|
+
import urllib.error
|
|
21
|
+
import urllib.parse
|
|
22
|
+
import urllib.request
|
|
23
|
+
import webbrowser
|
|
24
|
+
from dataclasses import dataclass
|
|
25
|
+
from typing import Callable, Mapping, Optional
|
|
26
|
+
|
|
27
|
+
from .exceptions import OnehouseSdkError
|
|
28
|
+
|
|
29
|
+
LOGIN_TIMEOUT_SECONDS = 300
|
|
30
|
+
_HTTP_TIMEOUT_SECONDS = 20
|
|
31
|
+
|
|
32
|
+
_DONE_PAGE = b"""<!doctype html><meta charset="utf-8"><title>Onehouse</title>
|
|
33
|
+
<body style="font:16px system-ui;padding:3rem;max-width:32rem">
|
|
34
|
+
<h2>Signed in</h2><p>You can close this tab and return to your terminal.</p></body>"""
|
|
35
|
+
|
|
36
|
+
_FAILED_PAGE = b"""<!doctype html><meta charset="utf-8"><title>Onehouse</title>
|
|
37
|
+
<body style="font:16px system-ui;padding:3rem;max-width:32rem">
|
|
38
|
+
<h2>Sign-in did not complete</h2><p>Return to your terminal for the reason.</p></body>"""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class OAuthFlowError(OnehouseSdkError):
|
|
42
|
+
"""A sign-in or token-endpoint call did not produce a usable token."""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass(frozen=True)
|
|
46
|
+
class AuthorizationResult:
|
|
47
|
+
"""What the browser step returns: redeem ``code`` with ``verifier`` at the same
|
|
48
|
+
``redirect_uri``, byte for byte."""
|
|
49
|
+
|
|
50
|
+
code: str
|
|
51
|
+
redirect_uri: str
|
|
52
|
+
verifier: str
|
|
53
|
+
iss: Optional[str] = None
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _b64url(raw: bytes) -> str:
|
|
57
|
+
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def new_verifier() -> str:
|
|
61
|
+
"""43 characters of base64url (RFC 7636 4.1)."""
|
|
62
|
+
return _b64url(secrets.token_bytes(32))
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def challenge_for(verifier: str) -> str:
|
|
66
|
+
"""S256 only: ``plain`` would put the verifier in the browser's history."""
|
|
67
|
+
return _b64url(hashlib.sha256(verifier.encode("ascii")).digest())
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def default_ssl_context() -> ssl.SSLContext:
|
|
71
|
+
"""The OS trust store when ``truststore`` is installed, so a TLS-intercepting
|
|
72
|
+
corporate proxy's CA is trusted; then certifi; then Python's defaults.
|
|
73
|
+
The same order as ``lakebase.auth._ssl``, which this module may not import."""
|
|
74
|
+
try:
|
|
75
|
+
import truststore
|
|
76
|
+
|
|
77
|
+
return truststore.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
|
|
78
|
+
except ImportError:
|
|
79
|
+
pass
|
|
80
|
+
try:
|
|
81
|
+
import certifi
|
|
82
|
+
|
|
83
|
+
return ssl.create_default_context(cafile=certifi.where())
|
|
84
|
+
except ImportError:
|
|
85
|
+
return ssl.create_default_context()
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class _Catcher(http.server.BaseHTTPRequestHandler):
|
|
89
|
+
"""Serves one useful request: the authorization server's redirect back."""
|
|
90
|
+
|
|
91
|
+
callback_path = ""
|
|
92
|
+
expected_state = ""
|
|
93
|
+
result: dict = {}
|
|
94
|
+
# A connection that sends nothing (a browser preconnect, a port scan) would
|
|
95
|
+
# otherwise block handle_request() past the login deadline.
|
|
96
|
+
timeout = 5
|
|
97
|
+
|
|
98
|
+
def do_GET(self) -> None: # noqa: N802 - name fixed by BaseHTTPRequestHandler
|
|
99
|
+
parsed = urllib.parse.urlparse(self.path)
|
|
100
|
+
if parsed.path != type(self).callback_path:
|
|
101
|
+
self.send_response(404)
|
|
102
|
+
self.end_headers()
|
|
103
|
+
return
|
|
104
|
+
params = urllib.parse.parse_qs(parsed.query)
|
|
105
|
+
# State before anything else: a callback this process did not start is not
|
|
106
|
+
# ours to act on, and must not end the wait.
|
|
107
|
+
got_state = (params.get("state") or [""])[0]
|
|
108
|
+
if not secrets.compare_digest(got_state, type(self).expected_state):
|
|
109
|
+
self._respond(400, _FAILED_PAGE)
|
|
110
|
+
return
|
|
111
|
+
error = (params.get("error") or [""])[0]
|
|
112
|
+
if error:
|
|
113
|
+
description = (params.get("error_description") or [""])[0]
|
|
114
|
+
type(self).result = {"error": f"{error}: {description}" if description else error}
|
|
115
|
+
self._respond(400, _FAILED_PAGE)
|
|
116
|
+
return
|
|
117
|
+
code = (params.get("code") or [""])[0]
|
|
118
|
+
if not code:
|
|
119
|
+
type(self).result = {"error": "the authorization server returned no code"}
|
|
120
|
+
self._respond(400, _FAILED_PAGE)
|
|
121
|
+
return
|
|
122
|
+
type(self).result = {"code": code, "iss": (params.get("iss") or [None])[0]}
|
|
123
|
+
self._respond(200, _DONE_PAGE)
|
|
124
|
+
|
|
125
|
+
def _respond(self, status: int, body: bytes) -> None:
|
|
126
|
+
self.send_response(status)
|
|
127
|
+
self.send_header("Content-Type", "text/html; charset=utf-8")
|
|
128
|
+
self.send_header("Content-Length", str(len(body)))
|
|
129
|
+
self.end_headers()
|
|
130
|
+
self.wfile.write(body)
|
|
131
|
+
|
|
132
|
+
def log_message(self, *_args) -> None:
|
|
133
|
+
"""The default access log would print the callback URL, code included."""
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
class LoopbackServer:
|
|
137
|
+
"""A one-shot HTTP server on 127.0.0.1, on an OS-chosen port unless one is given.
|
|
138
|
+
|
|
139
|
+
127.0.0.1, never ``localhost``: redirect matching is byte for byte, and
|
|
140
|
+
``localhost`` resolves through DNS (RFC 8252 7.3, 8.3).
|
|
141
|
+
"""
|
|
142
|
+
|
|
143
|
+
def __init__(self, callback_path: str, state: str, port: int = 0) -> None:
|
|
144
|
+
self._callback_path = callback_path
|
|
145
|
+
self._handler = type(
|
|
146
|
+
"_Bound",
|
|
147
|
+
(_Catcher,),
|
|
148
|
+
{"callback_path": callback_path, "expected_state": state, "result": {}},
|
|
149
|
+
)
|
|
150
|
+
self._httpd = http.server.HTTPServer(("127.0.0.1", port), self._handler)
|
|
151
|
+
self._httpd.timeout = 0.5
|
|
152
|
+
|
|
153
|
+
@property
|
|
154
|
+
def port(self) -> int:
|
|
155
|
+
return self._httpd.server_address[1]
|
|
156
|
+
|
|
157
|
+
@property
|
|
158
|
+
def redirect_uri(self) -> str:
|
|
159
|
+
return f"http://127.0.0.1:{self.port}{self._callback_path}"
|
|
160
|
+
|
|
161
|
+
def wait(self, timeout: float) -> dict:
|
|
162
|
+
deadline = time.monotonic() + timeout
|
|
163
|
+
while time.monotonic() < deadline:
|
|
164
|
+
try:
|
|
165
|
+
self._httpd.handle_request()
|
|
166
|
+
except OSError:
|
|
167
|
+
# A connection that timed out or reset mid-read is not an outcome.
|
|
168
|
+
continue
|
|
169
|
+
if self._handler.result:
|
|
170
|
+
return self._handler.result
|
|
171
|
+
return {"error": f"timed out after {int(timeout)}s waiting for the browser"}
|
|
172
|
+
|
|
173
|
+
def close(self) -> None:
|
|
174
|
+
self._httpd.server_close()
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def authorize(
|
|
178
|
+
authorization_endpoint: str,
|
|
179
|
+
*,
|
|
180
|
+
client_id: str,
|
|
181
|
+
callback_path: str,
|
|
182
|
+
scope: Optional[str] = None,
|
|
183
|
+
extra_params: Optional[Mapping[str, str]] = None,
|
|
184
|
+
open_browser: Callable[[str], object] = webbrowser.open,
|
|
185
|
+
announce: Callable[[str], None] = print,
|
|
186
|
+
timeout: float = LOGIN_TIMEOUT_SECONDS,
|
|
187
|
+
port: int = 0,
|
|
188
|
+
) -> AuthorizationResult:
|
|
189
|
+
"""Run the browser step of an authorization-code + PKCE sign-in.
|
|
190
|
+
|
|
191
|
+
``port`` pins the loopback port, which only an SSH forward needs: the tunnel
|
|
192
|
+
must name a port before the sign-in starts.
|
|
193
|
+
"""
|
|
194
|
+
verifier = new_verifier()
|
|
195
|
+
state = _b64url(secrets.token_bytes(16))
|
|
196
|
+
try:
|
|
197
|
+
server = LoopbackServer(callback_path, state, port)
|
|
198
|
+
except OSError as err:
|
|
199
|
+
where = f"port {port}" if port else "a local port"
|
|
200
|
+
raise OAuthFlowError(f"could not bind {where} for the sign-in ({err})") from err
|
|
201
|
+
try:
|
|
202
|
+
params = {
|
|
203
|
+
"response_type": "code",
|
|
204
|
+
"client_id": client_id,
|
|
205
|
+
"redirect_uri": server.redirect_uri,
|
|
206
|
+
"state": state,
|
|
207
|
+
"code_challenge": challenge_for(verifier),
|
|
208
|
+
"code_challenge_method": "S256",
|
|
209
|
+
}
|
|
210
|
+
if scope:
|
|
211
|
+
params["scope"] = scope
|
|
212
|
+
params.update(extra_params or {})
|
|
213
|
+
separator = "&" if "?" in authorization_endpoint else "?"
|
|
214
|
+
url = f"{authorization_endpoint}{separator}{urllib.parse.urlencode(params)}"
|
|
215
|
+
announce(f"Opening your browser to sign in.\nIf it does not open, visit:\n {url}")
|
|
216
|
+
open_browser(url)
|
|
217
|
+
outcome = server.wait(timeout)
|
|
218
|
+
redirect_uri = server.redirect_uri
|
|
219
|
+
finally:
|
|
220
|
+
server.close()
|
|
221
|
+
if "error" in outcome:
|
|
222
|
+
raise OAuthFlowError(outcome["error"])
|
|
223
|
+
return AuthorizationResult(
|
|
224
|
+
code=outcome["code"], redirect_uri=redirect_uri, verifier=verifier, iss=outcome.get("iss")
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
@dataclass(frozen=True)
|
|
229
|
+
class HttpResult:
|
|
230
|
+
status: int
|
|
231
|
+
body: dict
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _read_json(raw: bytes) -> dict:
|
|
235
|
+
try:
|
|
236
|
+
parsed = json.loads(raw.decode("utf-8")) if raw else {}
|
|
237
|
+
except (ValueError, UnicodeDecodeError):
|
|
238
|
+
return {}
|
|
239
|
+
return parsed if isinstance(parsed, dict) else {}
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _send(request: urllib.request.Request, timeout: float) -> HttpResult:
|
|
243
|
+
try:
|
|
244
|
+
with urllib.request.urlopen(
|
|
245
|
+
request, timeout=timeout, context=default_ssl_context()
|
|
246
|
+
) as resp:
|
|
247
|
+
return HttpResult(resp.status, _read_json(resp.read()))
|
|
248
|
+
except urllib.error.HTTPError as err:
|
|
249
|
+
# An OAuth error response is a 4xx with a JSON body worth reading.
|
|
250
|
+
return HttpResult(err.code, _read_json(err.read()))
|
|
251
|
+
except (urllib.error.URLError, OSError) as err:
|
|
252
|
+
raise OAuthFlowError(f"could not reach {request.full_url} ({err})") from err
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def post_form(url: str, data: Mapping[str, str], timeout: float = _HTTP_TIMEOUT_SECONDS) -> HttpResult:
|
|
256
|
+
"""POST ``application/x-www-form-urlencoded``, as every token endpoint takes it."""
|
|
257
|
+
request = urllib.request.Request(
|
|
258
|
+
url,
|
|
259
|
+
data=urllib.parse.urlencode(dict(data)).encode("ascii"),
|
|
260
|
+
headers={
|
|
261
|
+
"Content-Type": "application/x-www-form-urlencoded",
|
|
262
|
+
"Accept": "application/json",
|
|
263
|
+
},
|
|
264
|
+
method="POST",
|
|
265
|
+
)
|
|
266
|
+
return _send(request, timeout)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def get_json(url: str, timeout: float = _HTTP_TIMEOUT_SECONDS) -> HttpResult:
|
|
270
|
+
request = urllib.request.Request(url, headers={"Accept": "application/json"}, method="GET")
|
|
271
|
+
return _send(request, timeout)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def oauth_error(result: HttpResult) -> str:
|
|
275
|
+
"""The RFC 6749 5.2 ``error`` code of a failed call, or its HTTP status."""
|
|
276
|
+
error = result.body.get("error")
|
|
277
|
+
return str(error) if error else f"HTTP {result.status}"
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
__all__ = [
|
|
281
|
+
"LOGIN_TIMEOUT_SECONDS",
|
|
282
|
+
"AuthorizationResult",
|
|
283
|
+
"HttpResult",
|
|
284
|
+
"LoopbackServer",
|
|
285
|
+
"OAuthFlowError",
|
|
286
|
+
"authorize",
|
|
287
|
+
"challenge_for",
|
|
288
|
+
"default_ssl_context",
|
|
289
|
+
"get_json",
|
|
290
|
+
"new_verifier",
|
|
291
|
+
"oauth_error",
|
|
292
|
+
"post_form",
|
|
293
|
+
]
|