periplus-python-sdk 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- periplus_python_sdk-0.11.0.dist-info/METADATA +150 -0
- periplus_python_sdk-0.11.0.dist-info/RECORD +23 -0
- periplus_sdk/__init__.py +5 -3
- periplus_sdk/async_resources.py +298 -0
- periplus_sdk/async_stream.py +104 -0
- periplus_sdk/client.py +118 -9
- periplus_sdk/errors.py +3 -1
- periplus_sdk/resources.py +297 -0
- periplus_sdk/resources_types.py +264 -0
- periplus_sdk/selection.py +29 -0
- periplus_sdk/sqlalchemy.py +25 -25
- periplus_sdk/stream.py +8 -2
- periplus_sdk/types.py +91 -3
- periplus_python_sdk-0.9.0.dist-info/METADATA +0 -83
- periplus_python_sdk-0.9.0.dist-info/RECORD +0 -18
- {periplus_python_sdk-0.9.0.dist-info → periplus_python_sdk-0.11.0.dist-info}/WHEEL +0 -0
- {periplus_python_sdk-0.9.0.dist-info → periplus_python_sdk-0.11.0.dist-info}/entry_points.txt +0 -0
- {periplus_python_sdk-0.9.0.dist-info → periplus_python_sdk-0.11.0.dist-info}/licenses/licensing/LICENSE +0 -0
- {periplus_python_sdk-0.9.0.dist-info → periplus_python_sdk-0.11.0.dist-info}/licenses/licensing/NOTICE +0 -0
- {periplus_python_sdk-0.9.0.dist-info → periplus_python_sdk-0.11.0.dist-info}/licenses/licensing/README.md +0 -0
- {periplus_python_sdk-0.9.0.dist-info → periplus_python_sdk-0.11.0.dist-info}/licenses/licensing/THIRD_PARTY_NOTICES.md +0 -0
- {periplus_python_sdk-0.9.0.dist-info → periplus_python_sdk-0.11.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: periplus-python-sdk
|
|
3
|
+
Version: 0.11.0
|
|
4
|
+
Summary: Typed Periplus platform client with SQL and notebook integration
|
|
5
|
+
License-Expression: AGPL-3.0-only
|
|
6
|
+
Project-URL: Repository, https://github.com/elei-io/periplus
|
|
7
|
+
Project-URL: Issues, https://github.com/elei-io/periplus/issues
|
|
8
|
+
Requires-Python: >=3.11
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: licensing/LICENSE
|
|
11
|
+
License-File: licensing/NOTICE
|
|
12
|
+
License-File: licensing/README.md
|
|
13
|
+
License-File: licensing/THIRD_PARTY_NOTICES.md
|
|
14
|
+
Requires-Dist: httpx>=0.28
|
|
15
|
+
Requires-Dist: pydantic<3,>=2.12
|
|
16
|
+
Requires-Dist: sqlalchemy<3,>=2.0
|
|
17
|
+
Provides-Extra: notebook
|
|
18
|
+
Requires-Dist: marimo[sql]>=0.24.1; extra == "notebook"
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# Periplus Python SDK
|
|
22
|
+
|
|
23
|
+
Typed organization operations and SQL access through the Periplus HTTP API.
|
|
24
|
+
The 0.11.0 platform interface requires the matching API release. It includes
|
|
25
|
+
the source-snapshot metadata introduced in 0.10.0 alongside customer operations.
|
|
26
|
+
Install from PyPI:
|
|
27
|
+
|
|
28
|
+
```sh
|
|
29
|
+
python -m pip install --upgrade periplus-python-sdk
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
from periplus_sdk import Client
|
|
34
|
+
|
|
35
|
+
with Client("https://api.periplus.dev", api_key="ppl_…") as client:
|
|
36
|
+
result = client.execute(
|
|
37
|
+
"SELECT capture_id, url FROM public_v1.captures LIMIT ?", [10]
|
|
38
|
+
)
|
|
39
|
+
print(result.columns, result.rows)
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
The default origin is `https://api.periplus.dev`. Use `PERIPLUS_API_URL` to override it and `PERIPLUS_API_KEY` to
|
|
43
|
+
supply your personal organization API key. A key with `organization:sql:exec`
|
|
44
|
+
is required. Database usernames/passwords and anonymous access are not supported.
|
|
45
|
+
Use HTTPS outside loopback development. Connect directly to the API origin,
|
|
46
|
+
not the public marketing site.
|
|
47
|
+
|
|
48
|
+
Version 0.9.0 requires personal API keys instead of database credentials. Create
|
|
49
|
+
a key in the app's Settings → My API keys. It inherits your current access in
|
|
50
|
+
that organization; SQL requires your membership to have `organization:sql:exec`.
|
|
51
|
+
For local development, install `./clients/periplus-python-sdk` from the repository
|
|
52
|
+
root and connect to `http://localhost:8000`.
|
|
53
|
+
|
|
54
|
+
`AsyncClient` accepts the same options. `prepare` explains a SELECT; `execute`
|
|
55
|
+
returns typed columns/rows for read-only queries. `schema()` returns
|
|
56
|
+
visible tables, column types/descriptions and helper documentation. All SQL uses
|
|
57
|
+
`POST /api/v1/sql`; schema discovery uses `GET /api/v1/schema`. ClickHouse enforces
|
|
58
|
+
permissions. The SDK never retries automatically, including failed queries.
|
|
59
|
+
|
|
60
|
+
Public HTML joins use `parse_id` and `node_index`; `document_id` identifies exact
|
|
61
|
+
raw bytes. Public shorthand uses the `public_v1` schema.
|
|
62
|
+
|
|
63
|
+
In 0.10.0, `result.source_snapshot` is a typed `SourceSnapshot` containing
|
|
64
|
+
`layout_id` (UUID) and `publication_epoch` (integer), or `None` when no build-bound
|
|
65
|
+
public corpus was read. Buffered, asynchronous, streamed and DB-API results share
|
|
66
|
+
this contract. Compare both fields, not just the epoch. The identity describes
|
|
67
|
+
the public corpus inputs, not any native staff/external inputs. It does not retain
|
|
68
|
+
the data or request historical reads. This replaces the old nullable integer field;
|
|
69
|
+
use this SDK version with the corresponding API release.
|
|
70
|
+
|
|
71
|
+
For notebook/SQLAlchemy integration:
|
|
72
|
+
|
|
73
|
+
```python
|
|
74
|
+
from periplus_sdk import sql_api
|
|
75
|
+
from sqlalchemy import text
|
|
76
|
+
|
|
77
|
+
engine = sql_api.create_engine(base_url="http://localhost:8000", api_key="ppl_…")
|
|
78
|
+
with engine.connect() as connection:
|
|
79
|
+
print(connection.execute(text("SELECT url FROM public_v1.captures LIMIT 5")).all())
|
|
80
|
+
engine.dispose()
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Marimo discovers accessible tables, views, typed columns and comments through the
|
|
84
|
+
schema endpoint. Reflection does not execute SQL. One SQLAlchemy Inspector caches
|
|
85
|
+
its metadata; call `inspector.clear_cache()` to refresh it. Missing metadata raises
|
|
86
|
+
an error rather than presenting an apparently complete empty schema.
|
|
87
|
+
|
|
88
|
+
The DB-API connection advertises the ClickHouse dialect and converts native
|
|
89
|
+
nullable integer, decimal, date and datetime types. Nested types retain JSON wire
|
|
90
|
+
values. Writes are not exposed through the query API. There are no client
|
|
91
|
+
transactions; each statement is independent.
|
|
92
|
+
Streaming cursors expose incomplete/truncated results explicitly; configure
|
|
93
|
+
`allow_partial` only when partial results suit the application.
|
|
94
|
+
|
|
95
|
+
See [the public schema](../../docs/SCHEMA.md) and [query boundary](../../docs/QUERY.md).
|
|
96
|
+
|
|
97
|
+
## Organization resources
|
|
98
|
+
|
|
99
|
+
The same key selects your organization and inherits your live membership access.
|
|
100
|
+
`Client` and `AsyncClient` expose the same namespaces:
|
|
101
|
+
|
|
102
|
+
| Namespace | Operations |
|
|
103
|
+
| --- | --- |
|
|
104
|
+
| `identity`, `availability` | `get` |
|
|
105
|
+
| `discovery` | `create`, `get`, `list`, `iter`, `arrivals`, `cancel` |
|
|
106
|
+
| `retention`, `monitoring` | `create`, `preview`, `activate`, `get`, `list`, `iter`, `members`, `update`, `pause`, `resume`, `delete` |
|
|
107
|
+
| `saved_queries` | `create`, `get`, `list`, `iter`, `rename`, `delete` |
|
|
108
|
+
| `members` | `list`, `update_role`, `remove` |
|
|
109
|
+
| `invitations` | `list`, `create`, `cancel` |
|
|
110
|
+
| `api_keys` | `create`, `list`, `iter`, `revoke` |
|
|
111
|
+
| `usage` | `get` |
|
|
112
|
+
| `query_history` | `list`, `iter`, `get`, `summary` |
|
|
113
|
+
| `audit` | `list`, `iter` |
|
|
114
|
+
|
|
115
|
+
```python
|
|
116
|
+
from periplus_sdk import Client
|
|
117
|
+
|
|
118
|
+
with Client() as client:
|
|
119
|
+
preview = client.retention.preview(
|
|
120
|
+
name="Research sources", days=90,
|
|
121
|
+
sql="SELECT capture_id FROM captures WHERE domain(url) = ?",
|
|
122
|
+
parameters=["example.com"],
|
|
123
|
+
)
|
|
124
|
+
print(preview.id, preview.capture_count, preview.sample)
|
|
125
|
+
# Review the selection before calling:
|
|
126
|
+
# active = client.retention.activate(preview)
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
SQL previews persist inactive, frozen selections; activation never reruns SQL.
|
|
130
|
+
Explicit IDs/URLs passed to `create` activate immediately. Monitoring URLs must
|
|
131
|
+
already be known in the corpus. Use `members` and its `next_cursor` for complete
|
|
132
|
+
membership: policy detail contains a sample. Updates/deletion accept a fetched
|
|
133
|
+
`Policy` or an ID with `expected_version`; conflicts are never retried.
|
|
134
|
+
|
|
135
|
+
Paginated lists return `Page[T]` (`items`, `next_cursor`). Pass cursors unchanged
|
|
136
|
+
or use `iter()`; membership/invitation lists are bounded snapshots instead.
|
|
137
|
+
Offset-based lists can shift during concurrent changes. Query history and arrivals
|
|
138
|
+
use keyset cursors. UTC usage ranges have an exclusive end date, at most 93 days.
|
|
139
|
+
Query history is best-effort and expires after 30 days; it is not a billing ledger.
|
|
140
|
+
|
|
141
|
+
`ApiError` includes HTTP status, code, optional request ID, validation fields and
|
|
142
|
+
Retry-After seconds. `TransportError` means the outcome of a write can be unknown.
|
|
143
|
+
Keep creation IDs to reconcile; never blindly retry. Key creation is one-time:
|
|
144
|
+
`created.secret.get_secret_value()` reveals the secret and must only be used for
|
|
145
|
+
secure storage. Its ordinary representation is masked. Lost secrets cannot be
|
|
146
|
+
recovered.
|
|
147
|
+
|
|
148
|
+
For async streaming, use `async with await client.stream(sql) as stream` followed
|
|
149
|
+
by `async for batch in stream`. Streams validate completion and close on early
|
|
150
|
+
exit, errors, and cancellation. No threads or background polling are introduced.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
periplus_python_sdk-0.11.0.dist-info/licenses/licensing/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
|
|
2
|
+
periplus_python_sdk-0.11.0.dist-info/licenses/licensing/NOTICE,sha256=W4W3pzZe-a0ebw4MxgWVmFGj7NSGL1qj8hFx4YG9iF0,105
|
|
3
|
+
periplus_python_sdk-0.11.0.dist-info/licenses/licensing/README.md,sha256=NABTC2wm2Y9iRNMnlMeyVMA2dsAw7XRmCrsV1B2TQdk,1996
|
|
4
|
+
periplus_python_sdk-0.11.0.dist-info/licenses/licensing/THIRD_PARTY_NOTICES.md,sha256=cuA1RaKPfcq5bOMFHTLiTaxjaySKlOqoU5WhHqV8h4U,9643
|
|
5
|
+
periplus_sdk/__init__.py,sha256=awbiP1tubzF5wIt8ffEb82KuuAs8Y2-5kydvZOYncjs,821
|
|
6
|
+
periplus_sdk/async_resources.py,sha256=ao6X3UhBwfxdBo4NRQYxTws4uFSQyjHb_qJ7e2lcURs,15375
|
|
7
|
+
periplus_sdk/async_stream.py,sha256=O5hO_w_-ooQuDV1fOZRX4Qei1-w7Wri8eIlsn3yCpGQ,4592
|
|
8
|
+
periplus_sdk/client.py,sha256=OMIeZnw241I23timweqMYvvv-vGfMeMje3ksiKvWftU,13637
|
|
9
|
+
periplus_sdk/dbapi.py,sha256=dkJA-DkrMtcgMGjLQ5Vv8-pb6riRrAaG4Yqrn22uRZY,12064
|
|
10
|
+
periplus_sdk/errors.py,sha256=nnbmRzEu1GEFGeKNsLqEo6pHCFu0oFy__5t-yBLO2hw,965
|
|
11
|
+
periplus_sdk/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
12
|
+
periplus_sdk/resources.py,sha256=i4JCemsLFUdaH_54JRIl6HVMrW8UIVA7q-AaBFXe-uA,14820
|
|
13
|
+
periplus_sdk/resources_types.py,sha256=Wy6TbNSv5pEG6_gbaE4paZTJyQB-IA3Zbgq8Ns9ZV6s,6135
|
|
14
|
+
periplus_sdk/selection.py,sha256=9MdqJF817wuMXSNmeFKoxCzL_tkh7ZmNI2Aie5AKFRo,1343
|
|
15
|
+
periplus_sdk/sql_api.py,sha256=uUWdwMRSZFMNXXy3-yLozcuT6WluvT6BTjYDB0lNrAU,1917
|
|
16
|
+
periplus_sdk/sqlalchemy.py,sha256=XE_OzdDh9OZ8goWWRoKlBzp7Q0FM-eS9-hjH7Ofzy-E,6292
|
|
17
|
+
periplus_sdk/stream.py,sha256=6Ytb6FkP3Y5C4sDA_SDdU0MJHBQdgHde1lNM6YxHdIQ,5560
|
|
18
|
+
periplus_sdk/types.py,sha256=TSaQRjN5vng_yV4hjZV03s5sU_RHL874qJe40-Hb1KQ,3644
|
|
19
|
+
periplus_python_sdk-0.11.0.dist-info/METADATA,sha256=V4EGzlbbSWPNz-8NKujkkxfnS_iPbFkNIY6LXoG-P8g,6957
|
|
20
|
+
periplus_python_sdk-0.11.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
21
|
+
periplus_python_sdk-0.11.0.dist-info/entry_points.txt,sha256=Pr14L_7AhLinq-4qDxB4awFVubrvR1BEfkaFVRpdCbU,73
|
|
22
|
+
periplus_python_sdk-0.11.0.dist-info/top_level.txt,sha256=o41t5TzwgoxzSmbKoP6olWW1FyEAGWVYTjeoKadBK40,13
|
|
23
|
+
periplus_python_sdk-0.11.0.dist-info/RECORD,,
|
periplus_sdk/__init__.py
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""Typed organization operations and read-only SQL through the Periplus API."""
|
|
2
2
|
from .dbapi import connect
|
|
3
3
|
from .client import AsyncClient, Client
|
|
4
4
|
from .errors import ApiError, ConfigurationError, PeriplusError, ResponseError, TransportError
|
|
5
|
-
from .types import Diagnostic, PreparedQuery, QueryHelper, QueryHelpers, QueryResult
|
|
5
|
+
from .types import Diagnostic, PreparedQuery, QueryHelper, QueryHelpers, QueryResult, SourceSnapshot
|
|
6
|
+
from .resources_types import Page, Policy, Discovery, Member, Invitation, ApiKey, CreatedApiKey, Identity, Usage
|
|
6
7
|
|
|
7
8
|
__all__ = ["connect", "AsyncClient", "Client", "ApiError", "ConfigurationError", "PeriplusError",
|
|
8
9
|
"ResponseError", "TransportError", "Diagnostic", "PreparedQuery", "QueryHelper",
|
|
9
|
-
"QueryHelpers", "QueryResult"
|
|
10
|
+
"QueryHelpers", "QueryResult", "Page", "Policy", "Discovery", "Member", "Invitation",
|
|
11
|
+
"ApiKey", "CreatedApiKey", "Identity", "Usage", "SourceSnapshot"]
|
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
"""Async customer operations; the same contract as resources.py."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
from collections.abc import AsyncIterator, Sequence
|
|
4
|
+
from datetime import date
|
|
5
|
+
from typing import Generic, Literal, TypeVar
|
|
6
|
+
from urllib.parse import quote
|
|
7
|
+
from uuid import UUID, uuid4
|
|
8
|
+
|
|
9
|
+
from pydantic import JsonValue
|
|
10
|
+
|
|
11
|
+
from .resources_types import (
|
|
12
|
+
ApiKey, ArrivalsPage, AuditEvent, Availability, CreatedApiKey, Discovery,
|
|
13
|
+
Identity, Invitation, Member, Members, Page, Policy, PolicyMember,
|
|
14
|
+
QueryExecution, QueryUsage, Role, SavedQuery, Usage,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def path_id(value: str | UUID) -> str:
|
|
19
|
+
return quote(str(value), safe="")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def page_offset(cursor: str | None) -> int:
|
|
23
|
+
if cursor is None:
|
|
24
|
+
return 0
|
|
25
|
+
if not cursor.isascii() or not cursor.isdecimal():
|
|
26
|
+
raise ValueError("Invalid page cursor")
|
|
27
|
+
return int(cursor)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def page_size(limit: int) -> int:
|
|
31
|
+
if not 1 <= limit <= 100:
|
|
32
|
+
raise ValueError("Page size must be between 1 and 100")
|
|
33
|
+
return limit
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def policy_ref(policy: Policy | str | UUID, expected_version: int | None, kind: str):
|
|
37
|
+
if isinstance(policy, Policy):
|
|
38
|
+
if policy.kind != kind:
|
|
39
|
+
raise ValueError("Policy belongs to a different resource")
|
|
40
|
+
return policy.id, policy.version if expected_version is None else expected_version
|
|
41
|
+
if expected_version is None:
|
|
42
|
+
raise ValueError("An ID requires expected_version; fetch the policy before changing it")
|
|
43
|
+
return policy, expected_version
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def member_ref(member: Member | str, expected_role: Role | None):
|
|
47
|
+
if isinstance(member, Member):
|
|
48
|
+
return member.id, expected_role if expected_role is not None else member.role
|
|
49
|
+
if expected_role is None:
|
|
50
|
+
raise ValueError("An ID requires expected_role; fetch membership before changing it")
|
|
51
|
+
return member, expected_role
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class Resource:
|
|
55
|
+
def __init__(self, client):
|
|
56
|
+
self._client = client
|
|
57
|
+
|
|
58
|
+
T = TypeVar("T")
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class PagedResource(Resource, Generic[T]):
|
|
62
|
+
async def iter(self, **filters) -> AsyncIterator[T]:
|
|
63
|
+
"""Lazily iterate list pages. No snapshot is implied while resources change."""
|
|
64
|
+
cursor = filters.pop("cursor", None)
|
|
65
|
+
while True:
|
|
66
|
+
page = await self.list(cursor=cursor, **filters)
|
|
67
|
+
for item in page.items:
|
|
68
|
+
yield item
|
|
69
|
+
if page.next_cursor is None:
|
|
70
|
+
return
|
|
71
|
+
if page.next_cursor == cursor:
|
|
72
|
+
raise ValueError("Server returned a non-advancing cursor")
|
|
73
|
+
cursor = page.next_cursor
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class IdentityResource(Resource):
|
|
77
|
+
async def get(self) -> Identity:
|
|
78
|
+
return await self._client._resource_request("GET", "identity", Identity)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class AvailabilityResource(Resource):
|
|
82
|
+
async def get(self) -> Availability:
|
|
83
|
+
return await self._client._resource_request("GET", "access", Availability)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class DiscoveryResource(PagedResource[Discovery]):
|
|
87
|
+
async def select_urls(self, sql: str, parameters: Sequence[JsonValue] = ()) -> list[str]:
|
|
88
|
+
"""Resolve a complete starting-page selection without creating discovery."""
|
|
89
|
+
from .selection import starting_urls
|
|
90
|
+
return starting_urls(await self._client.execute(sql, parameters))
|
|
91
|
+
|
|
92
|
+
async def create(self, *, urls: Sequence[str], link_depth: int = 2, page_limit: int = 10_000,
|
|
93
|
+
follow_external_links: bool = False, allowed_hosts: Sequence[str] = (),
|
|
94
|
+
excluded_hosts: Sequence[str] = (), allowed_paths: Sequence[str] = (),
|
|
95
|
+
excluded_paths: Sequence[str] = (), retain_days: int | None = None,
|
|
96
|
+
id: UUID | None = None) -> Discovery:
|
|
97
|
+
return await self._client._resource_request("POST", "discoveries", Discovery, json={
|
|
98
|
+
"id": str(id or uuid4()), "specification": {
|
|
99
|
+
"seed_urls": list(urls), "max_depth": link_depth, "page_limit": page_limit,
|
|
100
|
+
"follow_scope": "linked_sites" if follow_external_links else "starting_sites",
|
|
101
|
+
"allowed_hosts": list(allowed_hosts), "excluded_hosts": list(excluded_hosts),
|
|
102
|
+
"allowed_paths": list(allowed_paths), "excluded_paths": list(excluded_paths),
|
|
103
|
+
"retain_days": retain_days}})
|
|
104
|
+
|
|
105
|
+
async def list(self, *, cursor: str | None = None, limit: int = 20,
|
|
106
|
+
status: Literal["active", "paused", "settled"] | None = None) -> Page[Discovery]:
|
|
107
|
+
return await self._client._resource_request("GET", "discoveries", Page[Discovery],
|
|
108
|
+
params={"offset": page_offset(cursor), "limit": page_size(limit), "status": status})
|
|
109
|
+
|
|
110
|
+
async def get(self, id: str | UUID) -> Discovery:
|
|
111
|
+
return await self._client._resource_request("GET", f"discoveries/{path_id(id)}", Discovery)
|
|
112
|
+
|
|
113
|
+
async def cancel(self, id: str | UUID) -> Discovery:
|
|
114
|
+
return await self._client._resource_request("POST", f"discoveries/{path_id(id)}/actions",
|
|
115
|
+
Discovery, json={"action": "cancel"})
|
|
116
|
+
|
|
117
|
+
async def arrivals(self, id: str | UUID, *, cursor: str | None = None, limit: int = 20) -> ArrivalsPage:
|
|
118
|
+
return await self._client._resource_request("GET", f"discoveries/{path_id(id)}/arrivals",
|
|
119
|
+
ArrivalsPage, params={"cursor": cursor, "limit": page_size(limit)})
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class PolicyResource(PagedResource[Policy]):
|
|
123
|
+
kind: Literal["retention", "refresh"]
|
|
124
|
+
|
|
125
|
+
async def list(self, *, cursor: str | None = None, limit: int = 20,
|
|
126
|
+
include_drafts: bool = True) -> Page[Policy]:
|
|
127
|
+
return await self._client._resource_request("GET", "page-policies", Page[Policy],
|
|
128
|
+
params={"kind": self.kind, "offset": page_offset(cursor), "limit": page_size(limit),
|
|
129
|
+
"include_drafts": include_drafts})
|
|
130
|
+
|
|
131
|
+
async def get(self, id: str | UUID) -> Policy:
|
|
132
|
+
return await self._client._resource_request("GET", f"page-policies/{path_id(id)}", Policy,
|
|
133
|
+
params={"kind": self.kind, "members_limit": 20})
|
|
134
|
+
|
|
135
|
+
async def members(self, id: str | UUID, *, cursor: str | None = None, limit: int = 100) -> Page[PolicyMember]:
|
|
136
|
+
return await self._client._resource_request("GET", f"page-policies/{path_id(id)}/members",
|
|
137
|
+
Page[PolicyMember], params={"kind": self.kind, "offset": page_offset(cursor), "limit": page_size(limit)})
|
|
138
|
+
|
|
139
|
+
async def _create(self, name, days, selection, retain_days, id) -> Policy:
|
|
140
|
+
return await self._client._resource_request("POST", "page-policies", Policy, json={
|
|
141
|
+
"id": str(id or uuid4()), "kind": self.kind, "name": name, "days": days,
|
|
142
|
+
"selection": selection, "retain_days": retain_days})
|
|
143
|
+
|
|
144
|
+
async def _action(self, policy, action, expected_version) -> Policy:
|
|
145
|
+
id, version = policy_ref(policy, expected_version, self.kind)
|
|
146
|
+
return await self._client._resource_request("POST", f"page-policies/{path_id(id)}/actions", Policy,
|
|
147
|
+
params={"kind": self.kind}, json={"action": action, "expected_version": version})
|
|
148
|
+
|
|
149
|
+
async def activate(self, policy: Policy | str | UUID, *, expected_version: int | None = None) -> Policy:
|
|
150
|
+
"""Activate the persisted SQL selection without executing SQL again."""
|
|
151
|
+
return await self._action(policy, "activate", expected_version)
|
|
152
|
+
|
|
153
|
+
async def pause(self, policy: Policy | str | UUID, *, expected_version: int | None = None) -> Policy:
|
|
154
|
+
return await self._action(policy, "pause", expected_version)
|
|
155
|
+
|
|
156
|
+
async def resume(self, policy: Policy | str | UUID, *, expected_version: int | None = None) -> Policy:
|
|
157
|
+
"""Resume without restarting the original retention period."""
|
|
158
|
+
return await self._action(policy, "resume", expected_version)
|
|
159
|
+
|
|
160
|
+
async def delete(self, policy: Policy | str | UUID, *, expected_version: int | None = None) -> None:
|
|
161
|
+
id, version = policy_ref(policy, expected_version, self.kind)
|
|
162
|
+
return await self._client._resource_request("DELETE", f"page-policies/{path_id(id)}", None,
|
|
163
|
+
params={"kind": self.kind, "expected_version": version})
|
|
164
|
+
|
|
165
|
+
async def _update(self, policy, expected_version, fields) -> Policy:
|
|
166
|
+
id, version = policy_ref(policy, expected_version, self.kind)
|
|
167
|
+
return await self._client._resource_request("PATCH", f"page-policies/{path_id(id)}", Policy,
|
|
168
|
+
params={"kind": self.kind}, json={"expected_version": version, **fields})
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
class RetentionResource(PolicyResource):
|
|
172
|
+
kind = "retention"
|
|
173
|
+
|
|
174
|
+
async def create(self, *, name: str, capture_ids: Sequence[str | UUID], days: int,
|
|
175
|
+
id: UUID | None = None) -> Policy:
|
|
176
|
+
"""Protect explicit capture IDs immediately."""
|
|
177
|
+
return await self._create(name, days, {"capture_ids": [str(id) for id in capture_ids]}, None, id)
|
|
178
|
+
|
|
179
|
+
async def preview(self, *, name: str, sql: str, days: int,
|
|
180
|
+
parameters: Sequence[JsonValue] = (), id: UUID | None = None) -> Policy:
|
|
181
|
+
"""Save an inactive, frozen SQL selection. Call activate separately."""
|
|
182
|
+
return await self._create(name, days, {"sql": sql, "parameters": list(parameters)}, None, id)
|
|
183
|
+
|
|
184
|
+
async def update(self, policy: Policy | str | UUID, *, name: str | None = None,
|
|
185
|
+
days: int | None = None, expected_version: int | None = None) -> Policy:
|
|
186
|
+
return await self._update(policy, expected_version,
|
|
187
|
+
{key: value for key, value in {"name": name, "days": days}.items() if value is not None})
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
class MonitoringResource(PolicyResource):
|
|
191
|
+
kind = "refresh"
|
|
192
|
+
|
|
193
|
+
async def create(self, *, name: str, urls: Sequence[str], interval_days: int,
|
|
194
|
+
retain_days: int | None = None, id: UUID | None = None) -> Policy:
|
|
195
|
+
"""Monitor exact known corpus URLs immediately; does not follow links."""
|
|
196
|
+
return await self._create(name, interval_days, {"urls": list(urls)}, retain_days, id)
|
|
197
|
+
|
|
198
|
+
async def preview(self, *, name: str, sql: str, interval_days: int,
|
|
199
|
+
parameters: Sequence[JsonValue] = (), retain_days: int | None = None,
|
|
200
|
+
id: UUID | None = None) -> Policy:
|
|
201
|
+
return await self._create(name, interval_days, {"sql": sql, "parameters": list(parameters)}, retain_days, id)
|
|
202
|
+
|
|
203
|
+
async def update(self, policy: Policy | str | UUID, *, name: str | None = None,
|
|
204
|
+
interval_days: int | None = None, retain_days: int | None = None,
|
|
205
|
+
clear_retention: bool = False, expected_version: int | None = None) -> Policy:
|
|
206
|
+
if clear_retention and retain_days is not None:
|
|
207
|
+
raise ValueError("Choose retain_days or clear_retention, not both")
|
|
208
|
+
fields = {key: value for key, value in {"name": name, "days": interval_days,
|
|
209
|
+
"retain_days": retain_days}.items() if value is not None}
|
|
210
|
+
if clear_retention:
|
|
211
|
+
fields["retain_days"] = None
|
|
212
|
+
return await self._update(policy, expected_version, fields)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
class SavedQueriesResource(PagedResource[SavedQuery]):
|
|
216
|
+
async def list(self, *, search: str = "", cursor: str | None = None) -> Page[SavedQuery]:
|
|
217
|
+
return await self._client._resource_request("GET", "saved-queries", Page[SavedQuery],
|
|
218
|
+
params={"search": search, "offset": page_offset(cursor)})
|
|
219
|
+
|
|
220
|
+
async def get(self, id: str | UUID) -> SavedQuery:
|
|
221
|
+
return await self._client._resource_request("GET", f"saved-queries/{path_id(id)}", SavedQuery)
|
|
222
|
+
|
|
223
|
+
async def create(self, *, name: str, sql: str, id: UUID | None = None) -> SavedQuery:
|
|
224
|
+
return await self._client._resource_request("POST", "saved-queries", SavedQuery,
|
|
225
|
+
json={"id": str(id or uuid4()), "name": name, "sql": sql})
|
|
226
|
+
|
|
227
|
+
async def rename(self, id: str | UUID, *, name: str) -> SavedQuery:
|
|
228
|
+
return await self._client._resource_request("PATCH", f"saved-queries/{path_id(id)}", SavedQuery, json={"name": name})
|
|
229
|
+
|
|
230
|
+
async def delete(self, id: str | UUID) -> None:
|
|
231
|
+
return await self._client._resource_request("DELETE", f"saved-queries/{path_id(id)}", None)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
class MembersResource(Resource):
|
|
235
|
+
async def list(self) -> Members:
|
|
236
|
+
"""Organization membership and invitation snapshot, including owner guards."""
|
|
237
|
+
return await self._client._resource_request("GET", "members", Members)
|
|
238
|
+
|
|
239
|
+
async def update_role(self, member: Member | str, *, role: Role, expected_role: Role | None = None) -> None:
|
|
240
|
+
member_id, current_role = member_ref(member, expected_role)
|
|
241
|
+
return await self._client._resource_request("PUT", f"members/{path_id(member_id)}", None,
|
|
242
|
+
json={"action": role, "expected_role": current_role})
|
|
243
|
+
|
|
244
|
+
async def remove(self, member: Member | str, *, expected_role: Role | None = None) -> None:
|
|
245
|
+
member_id, current_role = member_ref(member, expected_role)
|
|
246
|
+
return await self._client._resource_request("PUT", f"members/{path_id(member_id)}", None,
|
|
247
|
+
json={"action": "remove", "expected_role": current_role})
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
class InvitationsResource(Resource):
|
|
251
|
+
async def list(self) -> list[Invitation]:
|
|
252
|
+
return (await self._client._resource_request("GET", "members", Members)).invitations
|
|
253
|
+
|
|
254
|
+
async def create(self, *, email: str, role: Role = "member") -> None:
|
|
255
|
+
return await self._client._resource_request("POST", "members/invitations", None,
|
|
256
|
+
json={"email": email, "role": role})
|
|
257
|
+
|
|
258
|
+
async def cancel(self, id: str) -> None:
|
|
259
|
+
return await self._client._resource_request("DELETE", f"members/invitations/{path_id(id)}", None)
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
class ApiKeysResource(PagedResource[ApiKey]):
|
|
263
|
+
async def list(self, *, cursor: str | None = None, limit: int = 20) -> Page[ApiKey]:
|
|
264
|
+
return await self._client._resource_request("GET", "api-keys", Page[ApiKey],
|
|
265
|
+
params={"offset": page_offset(cursor), "limit": page_size(limit)})
|
|
266
|
+
|
|
267
|
+
async def create(self, *, name: str, id: UUID | None = None) -> CreatedApiKey:
|
|
268
|
+
"""Secret is returned once; access via secret.get_secret_value()."""
|
|
269
|
+
return await self._client._resource_request("POST", "api-keys", CreatedApiKey,
|
|
270
|
+
json={"id": str(id or uuid4()), "name": name})
|
|
271
|
+
|
|
272
|
+
async def revoke(self, id: str | UUID) -> None:
|
|
273
|
+
return await self._client._resource_request("DELETE", f"api-keys/{path_id(id)}", None)
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
class UsageResource(Resource):
|
|
277
|
+
async def get(self, *, start: date | None = None, end: date | None = None) -> Usage:
|
|
278
|
+
"""UTC dates; end is exclusive. Usage is consumption, not an invoice."""
|
|
279
|
+
return await self._client._resource_request("GET", "usage", Usage, params={"start": start, "end": end})
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
class QueryHistoryResource(PagedResource[QueryExecution]):
|
|
283
|
+
async def summary(self, *, start: date | None = None, end: date | None = None) -> QueryUsage:
|
|
284
|
+
return await self._client._resource_request("GET", "usage/queries", QueryUsage, params={"start": start, "end": end})
|
|
285
|
+
|
|
286
|
+
async def list(self, *, cursor: str | None = None, limit: int = 50) -> Page[QueryExecution]:
|
|
287
|
+
return await self._client._resource_request("GET", "usage/query-history", Page[QueryExecution],
|
|
288
|
+
params={"cursor": cursor, "limit": page_size(limit)})
|
|
289
|
+
|
|
290
|
+
async def get(self, id: str | UUID) -> QueryExecution:
|
|
291
|
+
return await self._client._resource_request("GET", f"usage/queries/{path_id(id)}", QueryExecution)
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
class AuditResource(PagedResource[AuditEvent]):
|
|
295
|
+
async def list(self, *, cursor: str | None = None, action: str | None = None,
|
|
296
|
+
outcome: str | None = None) -> Page[AuditEvent]:
|
|
297
|
+
return await self._client._resource_request("GET", "audit", Page[AuditEvent],
|
|
298
|
+
params={"offset": page_offset(cursor), "action": action, "outcome": outcome})
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""Asynchronous incremental query frames with explicit completion checks."""
|
|
2
|
+
import json
|
|
3
|
+
import httpx
|
|
4
|
+
from pydantic import JsonValue, ValidationError
|
|
5
|
+
from .errors import ApiError, ResponseError, TransportError
|
|
6
|
+
from .stream import MEDIA_TYPE, StreamResult, Rows, Completion
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class AsyncQueryStream:
|
|
10
|
+
"""One response, consumed a batch at a time. Close early to cancel delivery."""
|
|
11
|
+
|
|
12
|
+
@classmethod
|
|
13
|
+
async def open(cls, response: httpx.Response, *, allow_partial: bool = False):
|
|
14
|
+
self = cls()
|
|
15
|
+
self.response = response
|
|
16
|
+
self.allow_partial = allow_partial
|
|
17
|
+
self.closed = False
|
|
18
|
+
self._count = 0
|
|
19
|
+
self._lines = response.aiter_lines()
|
|
20
|
+
try:
|
|
21
|
+
if response.headers.get("content-type", "").split(";")[0] != MEDIA_TYPE:
|
|
22
|
+
raise ResponseError("Expected a streaming query response.")
|
|
23
|
+
frame = await self._frame()
|
|
24
|
+
if frame.get("type") != "metadata":
|
|
25
|
+
raise ResponseError("Query stream is missing metadata.")
|
|
26
|
+
self.result = StreamResult.model_validate(frame)
|
|
27
|
+
if len(self.result.columns) != len(self.result.types):
|
|
28
|
+
raise ResponseError("Query columns and types have inconsistent widths.")
|
|
29
|
+
except ValidationError:
|
|
30
|
+
await self.aclose()
|
|
31
|
+
raise ResponseError("Invalid query stream metadata.") from None
|
|
32
|
+
except BaseException:
|
|
33
|
+
await self.aclose()
|
|
34
|
+
raise
|
|
35
|
+
|
|
36
|
+
return self
|
|
37
|
+
|
|
38
|
+
async def _frame(self):
|
|
39
|
+
try:
|
|
40
|
+
async for line in self._lines:
|
|
41
|
+
if not line.strip():
|
|
42
|
+
continue
|
|
43
|
+
frame = json.loads(line)
|
|
44
|
+
if not isinstance(frame, dict):
|
|
45
|
+
raise ResponseError("Invalid query stream frame.")
|
|
46
|
+
if frame.get("type") == "error":
|
|
47
|
+
raise ApiError(frame.get("detail", "Query stream failed."),
|
|
48
|
+
status_code=frame.get("status", 500), code=frame.get("code"))
|
|
49
|
+
return frame
|
|
50
|
+
except httpx.RequestError:
|
|
51
|
+
raise TransportError("Query stream interrupted; the result is incomplete.") from None
|
|
52
|
+
except ValueError:
|
|
53
|
+
raise ResponseError("Invalid query stream frame.") from None
|
|
54
|
+
raise ResponseError("Query stream ended without completion; the result is incomplete.")
|
|
55
|
+
|
|
56
|
+
def __aiter__(self):
|
|
57
|
+
return self
|
|
58
|
+
|
|
59
|
+
async def __anext__(self) -> list[list[JsonValue]]:
|
|
60
|
+
if self.closed:
|
|
61
|
+
if not self.result.complete:
|
|
62
|
+
raise ResponseError("Query stream closed before completion; the result is incomplete.")
|
|
63
|
+
if self.result.truncated and not self.allow_partial:
|
|
64
|
+
raise ResponseError("Query result exceeded its budget and is incomplete.")
|
|
65
|
+
raise StopAsyncIteration
|
|
66
|
+
try:
|
|
67
|
+
frame = await self._frame()
|
|
68
|
+
if frame.get("type") == "rows":
|
|
69
|
+
rows = Rows.model_validate(frame).rows
|
|
70
|
+
if any(len(row) != len(self.result.columns) for row in rows):
|
|
71
|
+
raise ResponseError("Query row has an inconsistent width.")
|
|
72
|
+
self._count += len(rows)
|
|
73
|
+
return rows
|
|
74
|
+
completion = Completion.model_validate(frame)
|
|
75
|
+
if completion.row_count != self._count:
|
|
76
|
+
raise ResponseError("Query completion row count does not match delivered rows.")
|
|
77
|
+
for name, value in completion.model_dump(exclude={"type"}).items():
|
|
78
|
+
setattr(self.result, name, value)
|
|
79
|
+
self.result.complete = True
|
|
80
|
+
await self.aclose()
|
|
81
|
+
if completion.truncated and not self.allow_partial:
|
|
82
|
+
budget = completion.truncation_reason
|
|
83
|
+
limit = self.result.limits.get(budget, "unknown")
|
|
84
|
+
raise ApiError(f"Query result is incomplete: {budget} ({limit}) reached after {self._count} rows. "
|
|
85
|
+
"Narrow the query or ask the administrator to increase this budget.",
|
|
86
|
+
status_code=422, code="result_limit")
|
|
87
|
+
raise StopAsyncIteration
|
|
88
|
+
except ValidationError:
|
|
89
|
+
await self.aclose()
|
|
90
|
+
raise ResponseError("Invalid query stream frame.") from None
|
|
91
|
+
except BaseException:
|
|
92
|
+
await self.aclose()
|
|
93
|
+
raise
|
|
94
|
+
|
|
95
|
+
async def aclose(self):
|
|
96
|
+
if not self.closed:
|
|
97
|
+
self.closed = True
|
|
98
|
+
await self.response.aclose()
|
|
99
|
+
|
|
100
|
+
async def __aenter__(self):
|
|
101
|
+
return self
|
|
102
|
+
|
|
103
|
+
async def __aexit__(self, *args):
|
|
104
|
+
await self.aclose()
|