tilebox-core 0.68.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _tilebox/__init__.py +1 -0
- _tilebox/aio/__init__.py +1 -0
- _tilebox/aio/concurrency.py +65 -0
- _tilebox/aio/pagination.py +29 -0
- _tilebox/client.py +211 -0
- _tilebox/client_metadata.py +210 -0
- _tilebox/concurrency.py +45 -0
- _tilebox/errors.py +65 -0
- _tilebox/pagination.py +42 -0
- _tilebox/proto/tilebox/v1/id_pb.py +63 -0
- _tilebox/proto/tilebox/v1/query_pb.py +234 -0
- _tilebox/replay.py +147 -0
- _tilebox/time.py +62 -0
- _tilebox/uuid.py +44 -0
- tilebox/errors.py +47 -0
- tilebox/query/__init__.py +7 -0
- tilebox/query/id_interval.py +70 -0
- tilebox/query/pagination.py +21 -0
- tilebox/query/time_interval.py +212 -0
- tilebox_core-0.68.0.dist-info/METADATA +31 -0
- tilebox_core-0.68.0.dist-info/RECORD +22 -0
- tilebox_core-0.68.0.dist-info/WHEEL +4 -0
_tilebox/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Shared native Connect transport support for Tilebox SDKs."""
|
_tilebox/aio/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Asynchronous pagination and concurrency helpers."""
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import inspect
|
|
3
|
+
from collections.abc import AsyncIterator, Awaitable, Callable
|
|
4
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
5
|
+
from typing import Any, TypeVar
|
|
6
|
+
|
|
7
|
+
from anyio import EndOfStream, create_memory_object_stream, create_task_group
|
|
8
|
+
from anyio.streams.memory import MemoryObjectReceiveStream, MemoryObjectSendStream
|
|
9
|
+
|
|
10
|
+
T = TypeVar("T")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
async def async_producer_consumer(
|
|
14
|
+
producer: AsyncIterator[T],
|
|
15
|
+
consumer: Callable[[T], None] | Callable[[T], Awaitable[Any]],
|
|
16
|
+
buffer_size: int = 10,
|
|
17
|
+
) -> None:
|
|
18
|
+
"""
|
|
19
|
+
Consume messages from a async producer (e.g. a service endpoint) as they come in. This ensures that the consumer
|
|
20
|
+
is already computing each produced message while the next message is being produced.
|
|
21
|
+
|
|
22
|
+
If the consumer is actually slower than the producer producing new message, the messages are buffered up to the
|
|
23
|
+
given buffer size. If the buffer is full, the producer will block until the consumer has consumed a message.
|
|
24
|
+
|
|
25
|
+
Args:
|
|
26
|
+
producer: The async generator to consume from
|
|
27
|
+
consumer: The consumer function to call for each message
|
|
28
|
+
buffer_size: The maximum number of messages to buffer before blocking the producer
|
|
29
|
+
"""
|
|
30
|
+
streams: Any = create_memory_object_stream(max_buffer_size=buffer_size)
|
|
31
|
+
send: MemoryObjectSendStream[T] = streams[0]
|
|
32
|
+
receive: MemoryObjectReceiveStream[T] = streams[1]
|
|
33
|
+
async with create_task_group() as task_group:
|
|
34
|
+
task_group.start_soon(_producer, producer, send)
|
|
35
|
+
task_group.start_soon(_consumer, consumer, receive)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
async def _producer(producer: AsyncIterator[T], send_stream: MemoryObjectSendStream[T]) -> None:
|
|
39
|
+
"""
|
|
40
|
+
Forward all messages from a producer (async iterator) and send them to a buffered send stream.
|
|
41
|
+
"""
|
|
42
|
+
async with send_stream:
|
|
43
|
+
async for message in producer:
|
|
44
|
+
await send_stream.send(message)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
async def _consumer(
|
|
48
|
+
consume: Callable[[T], None] | Callable[[T], Awaitable[Any]], receive_stream: MemoryObjectReceiveStream[T]
|
|
49
|
+
) -> None:
|
|
50
|
+
"""
|
|
51
|
+
Receive messages from a buffered receive stream and forward them to a consumer until no more messages are available.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
loop = asyncio.get_running_loop()
|
|
55
|
+
|
|
56
|
+
with ThreadPoolExecutor(max_workers=1) as executor, receive_stream:
|
|
57
|
+
try:
|
|
58
|
+
while True:
|
|
59
|
+
message = await receive_stream.receive()
|
|
60
|
+
# result = consume(message), but in a background thread so that async loop can continue in the meantime:
|
|
61
|
+
result = await loop.run_in_executor(executor, consume, message)
|
|
62
|
+
if inspect.isawaitable(result):
|
|
63
|
+
await result
|
|
64
|
+
except EndOfStream:
|
|
65
|
+
pass
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from collections.abc import AsyncIterator, Awaitable, Callable
|
|
2
|
+
|
|
3
|
+
from _tilebox.pagination import AnyResultPage, Pagination
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
async def paginated_request(
|
|
7
|
+
paging_request: Callable[[Pagination], Awaitable[AnyResultPage]],
|
|
8
|
+
initial_page: Pagination,
|
|
9
|
+
) -> AsyncIterator[AnyResultPage]:
|
|
10
|
+
"""Make a paginated request to a service endpoint.
|
|
11
|
+
|
|
12
|
+
The endpoint is expected to return a next_page field, which is used for subsequent requests. Once no such
|
|
13
|
+
next_page field is returned, the request is completed.
|
|
14
|
+
|
|
15
|
+
Args:
|
|
16
|
+
paging_request: A function that takes a page as input and returns a Datapoints object
|
|
17
|
+
Often this will be a functools.partial object that wraps a gRPC service endpoint
|
|
18
|
+
and only leaves the page argument remaining
|
|
19
|
+
initial_page: The initial page to request
|
|
20
|
+
|
|
21
|
+
Yields:
|
|
22
|
+
Datapoints: The individual pages of the response
|
|
23
|
+
"""
|
|
24
|
+
response = await paging_request(initial_page)
|
|
25
|
+
yield response
|
|
26
|
+
|
|
27
|
+
while response.next_page.starting_after is not None:
|
|
28
|
+
response = await paging_request(response.next_page)
|
|
29
|
+
yield response
|
_tilebox/client.py
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
"""Owned HTTP transports for native Connect clients."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import time
|
|
5
|
+
from collections.abc import Awaitable, Callable, Iterable
|
|
6
|
+
from typing import Literal, Self, TypeAlias, TypeVar
|
|
7
|
+
from urllib.parse import urlsplit, urlunsplit
|
|
8
|
+
|
|
9
|
+
from connectrpc.client import ConnectClient, ConnectClientSync
|
|
10
|
+
from connectrpc.code import Code
|
|
11
|
+
from connectrpc.errors import ConnectError
|
|
12
|
+
from connectrpc.interceptor import Interceptor, InterceptorSync
|
|
13
|
+
from connectrpc.request import RequestContext
|
|
14
|
+
from pyqwest import Client, HTTPTransport, HTTPVersion, SyncClient, SyncHTTPTransport
|
|
15
|
+
|
|
16
|
+
from _tilebox.client_metadata import client_metadata
|
|
17
|
+
from _tilebox.errors import translate_connect_error
|
|
18
|
+
|
|
19
|
+
Transport: TypeAlias = Literal["http2", "http1", "grpc"]
|
|
20
|
+
|
|
21
|
+
Sync = TypeVar("Sync", bound=ConnectClientSync)
|
|
22
|
+
Async = TypeVar("Async", bound=ConnectClient)
|
|
23
|
+
Req = TypeVar("Req")
|
|
24
|
+
Res = TypeVar("Res")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _http_version(transport: Transport) -> HTTPVersion:
|
|
28
|
+
match transport:
|
|
29
|
+
case "http2":
|
|
30
|
+
return HTTPVersion.HTTP2
|
|
31
|
+
case "http1":
|
|
32
|
+
return HTTPVersion.HTTP1
|
|
33
|
+
case _:
|
|
34
|
+
raise ValueError(f"Unsupported transport {transport!r}; use 'http2' or 'http1' (both use Connect)")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _retry_delay(ctx: RequestContext, error: ConnectError, attempt: int) -> float | None:
|
|
38
|
+
delay = min(0.02 * 3**attempt, 1.0)
|
|
39
|
+
if (
|
|
40
|
+
attempt >= 4
|
|
41
|
+
or error.code not in {Code.UNAVAILABLE, Code.RESOURCE_EXHAUSTED}
|
|
42
|
+
or ctx.timeout_ms is None
|
|
43
|
+
or ctx.timeout_ms <= delay * 1000
|
|
44
|
+
):
|
|
45
|
+
return None
|
|
46
|
+
return delay
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def connect_address(url: str, rpc_method_prefix: str | None = None) -> str:
|
|
50
|
+
"""Normalize endpoints, defaulting bare hosts to TLS and development ports to HTTP."""
|
|
51
|
+
if url.startswith("unix:"):
|
|
52
|
+
raise ValueError("Connect HTTP transport does not support unix socket URLs")
|
|
53
|
+
if "://" not in url:
|
|
54
|
+
parsed = urlsplit(f"//{url}")
|
|
55
|
+
scheme = "http" if parsed.port not in (None, 443) else "https"
|
|
56
|
+
url = f"{scheme}://{url}"
|
|
57
|
+
parsed = urlsplit(url)
|
|
58
|
+
if (
|
|
59
|
+
parsed.scheme not in {"http", "https"}
|
|
60
|
+
or not parsed.hostname
|
|
61
|
+
or parsed.username
|
|
62
|
+
or parsed.password
|
|
63
|
+
or parsed.query
|
|
64
|
+
or parsed.fragment
|
|
65
|
+
or any(character.isspace() for character in url)
|
|
66
|
+
):
|
|
67
|
+
raise ValueError(f"Invalid URL: {url}")
|
|
68
|
+
if parsed.scheme == "http" and parsed.port is None:
|
|
69
|
+
raise ValueError("Explicit port required for insecure HTTP connection")
|
|
70
|
+
if parsed.scheme == "https" and parsed.port == 443:
|
|
71
|
+
url = urlunsplit((parsed.scheme, parsed.netloc.removesuffix(":443"), parsed.path, "", ""))
|
|
72
|
+
prefix = (rpc_method_prefix or "").strip("/")
|
|
73
|
+
return url.rstrip("/") + (f"/{prefix}" if prefix else "")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class _MetadataErrors:
|
|
77
|
+
def __init__(self, token: str | None, address: str) -> None:
|
|
78
|
+
self.headers = client_metadata()
|
|
79
|
+
self.token = token
|
|
80
|
+
self.prefix = urlsplit(address).path.rstrip("/")
|
|
81
|
+
|
|
82
|
+
def _prepare(self, ctx: RequestContext) -> str:
|
|
83
|
+
for name, value in self.headers.items():
|
|
84
|
+
ctx.request_headers[name] = value
|
|
85
|
+
if self.token is not None:
|
|
86
|
+
ctx.request_headers["authorization"] = f"Bearer {self.token}"
|
|
87
|
+
return f"{self.prefix}/{ctx.method.service_name}/{ctx.method.name}"
|
|
88
|
+
|
|
89
|
+
def intercept_unary_sync(
|
|
90
|
+
self,
|
|
91
|
+
call_next: Callable[[Req, RequestContext], Res],
|
|
92
|
+
request: Req,
|
|
93
|
+
ctx: RequestContext,
|
|
94
|
+
) -> Res:
|
|
95
|
+
method = self._prepare(ctx)
|
|
96
|
+
attempt = 0
|
|
97
|
+
while True:
|
|
98
|
+
try:
|
|
99
|
+
return call_next(request, ctx)
|
|
100
|
+
except ConnectError as error:
|
|
101
|
+
delay = _retry_delay(ctx, error, attempt)
|
|
102
|
+
if delay is None:
|
|
103
|
+
raise translate_connect_error(error, method) from None
|
|
104
|
+
time.sleep(delay)
|
|
105
|
+
attempt += 1
|
|
106
|
+
|
|
107
|
+
async def intercept_unary(
|
|
108
|
+
self,
|
|
109
|
+
call_next: Callable[[Req, RequestContext], Awaitable[Res]],
|
|
110
|
+
request: Req,
|
|
111
|
+
ctx: RequestContext,
|
|
112
|
+
) -> Res:
|
|
113
|
+
method = self._prepare(ctx)
|
|
114
|
+
attempt = 0
|
|
115
|
+
while True:
|
|
116
|
+
try:
|
|
117
|
+
return await call_next(request, ctx)
|
|
118
|
+
except ConnectError as error:
|
|
119
|
+
delay = _retry_delay(ctx, error, attempt)
|
|
120
|
+
if delay is None:
|
|
121
|
+
raise translate_connect_error(error, method) from None
|
|
122
|
+
await asyncio.sleep(delay)
|
|
123
|
+
attempt += 1
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
class Connection:
|
|
127
|
+
"""Share one explicitly owned HTTP transport across generated synchronous clients."""
|
|
128
|
+
|
|
129
|
+
def __init__(
|
|
130
|
+
self,
|
|
131
|
+
url: str,
|
|
132
|
+
token: str | None = None,
|
|
133
|
+
rpc_method_prefix: str | None = None,
|
|
134
|
+
*,
|
|
135
|
+
transport: Transport = "http2",
|
|
136
|
+
interceptors: Iterable[InterceptorSync] = (),
|
|
137
|
+
) -> None:
|
|
138
|
+
self.address = connect_address(url, rpc_method_prefix)
|
|
139
|
+
self.transport = "http2" if transport == "grpc" else transport
|
|
140
|
+
self._transport = SyncHTTPTransport(http_version=_http_version(self.transport), tls_include_system_certs=True)
|
|
141
|
+
self._http = SyncClient(self._transport)
|
|
142
|
+
self._interceptors = (_MetadataErrors(token, self.address), *interceptors)
|
|
143
|
+
self._closed = False
|
|
144
|
+
|
|
145
|
+
def client(self, client_class: type[Sync]) -> Sync:
|
|
146
|
+
"""Construct a native generated client using this connection's HTTP client."""
|
|
147
|
+
if self._closed:
|
|
148
|
+
raise RuntimeError("Connection is closed")
|
|
149
|
+
return client_class(
|
|
150
|
+
self.address,
|
|
151
|
+
http_client=self._http,
|
|
152
|
+
interceptors=self._interceptors,
|
|
153
|
+
read_max_bytes=512 * 1024 * 1024,
|
|
154
|
+
timeout_ms=60_000,
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
def close(self) -> None:
|
|
158
|
+
"""Release this connection's transport; safe to call more than once."""
|
|
159
|
+
if not self._closed:
|
|
160
|
+
self._closed = True
|
|
161
|
+
self._transport.close()
|
|
162
|
+
|
|
163
|
+
def __enter__(self) -> Self:
|
|
164
|
+
return self
|
|
165
|
+
|
|
166
|
+
def __exit__(self, *exc: object) -> None:
|
|
167
|
+
self.close()
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
class AsyncConnection:
|
|
171
|
+
"""Share one explicitly owned HTTP transport across generated asynchronous clients."""
|
|
172
|
+
|
|
173
|
+
def __init__(
|
|
174
|
+
self,
|
|
175
|
+
url: str,
|
|
176
|
+
token: str | None = None,
|
|
177
|
+
rpc_method_prefix: str | None = None,
|
|
178
|
+
*,
|
|
179
|
+
transport: Transport = "http2",
|
|
180
|
+
interceptors: Iterable[Interceptor] = (),
|
|
181
|
+
) -> None:
|
|
182
|
+
self.address = connect_address(url, rpc_method_prefix)
|
|
183
|
+
self.transport = "http2" if transport == "grpc" else transport
|
|
184
|
+
self._transport = HTTPTransport(http_version=_http_version(self.transport), tls_include_system_certs=True)
|
|
185
|
+
self._http = Client(self._transport)
|
|
186
|
+
self._interceptors = (_MetadataErrors(token, self.address), *interceptors)
|
|
187
|
+
self._closed = False
|
|
188
|
+
|
|
189
|
+
def client(self, client_class: type[Async]) -> Async:
|
|
190
|
+
"""Construct a native generated client without opening an event loop."""
|
|
191
|
+
if self._closed:
|
|
192
|
+
raise RuntimeError("Connection is closed")
|
|
193
|
+
return client_class(
|
|
194
|
+
self.address,
|
|
195
|
+
http_client=self._http,
|
|
196
|
+
interceptors=self._interceptors,
|
|
197
|
+
read_max_bytes=512 * 1024 * 1024,
|
|
198
|
+
timeout_ms=60_000,
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
async def close(self) -> None:
|
|
202
|
+
"""Release this connection's transport; safe to call more than once."""
|
|
203
|
+
if not self._closed:
|
|
204
|
+
self._closed = True
|
|
205
|
+
await self._transport.aclose()
|
|
206
|
+
|
|
207
|
+
async def __aenter__(self) -> Self:
|
|
208
|
+
return self
|
|
209
|
+
|
|
210
|
+
async def __aexit__(self, *exc: object) -> None:
|
|
211
|
+
await self.close()
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
import builtins
|
|
2
|
+
import os
|
|
3
|
+
import platform
|
|
4
|
+
import sys
|
|
5
|
+
from collections.abc import Mapping
|
|
6
|
+
from functools import lru_cache
|
|
7
|
+
from importlib.metadata import PackageNotFoundError
|
|
8
|
+
from importlib.metadata import version as package_version
|
|
9
|
+
|
|
10
|
+
CLIENT_HEADER = "Tilebox-Client"
|
|
11
|
+
|
|
12
|
+
_MAX_FIELD_BYTES = 256
|
|
13
|
+
_MAX_HEADER_BYTES = 2 * 1024
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@lru_cache(maxsize=1)
|
|
17
|
+
def client_metadata() -> dict[str, str]:
|
|
18
|
+
"""Return cached metadata identifying this SDK and its execution context."""
|
|
19
|
+
return {CLIENT_HEADER: _client_header()}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _client_header() -> str:
|
|
23
|
+
fields = {
|
|
24
|
+
"name": "python",
|
|
25
|
+
"version": _package_version(),
|
|
26
|
+
"runtime": "python",
|
|
27
|
+
"runtime-version": platform.python_version(),
|
|
28
|
+
"os": _os_name(),
|
|
29
|
+
"os-version": platform.release(),
|
|
30
|
+
"arch": _architecture(),
|
|
31
|
+
"execution-environment": _execution_environment(os.environ),
|
|
32
|
+
**_invoker(os.environ),
|
|
33
|
+
**_cloud_environment(os.environ),
|
|
34
|
+
}
|
|
35
|
+
return _serialize_dictionary(fields)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _package_version() -> str | None:
|
|
39
|
+
try:
|
|
40
|
+
return package_version("tilebox-python")
|
|
41
|
+
except PackageNotFoundError:
|
|
42
|
+
try:
|
|
43
|
+
return package_version("tilebox-core")
|
|
44
|
+
except PackageNotFoundError:
|
|
45
|
+
return None
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _os_name() -> str | None:
|
|
49
|
+
return platform.system().lower() or None
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _architecture() -> str | None:
|
|
53
|
+
machine = platform.machine().lower()
|
|
54
|
+
architectures = {
|
|
55
|
+
"aarch64": "arm64",
|
|
56
|
+
"amd64": "amd64",
|
|
57
|
+
"arm64": "arm64",
|
|
58
|
+
"armv6l": "arm",
|
|
59
|
+
"armv7l": "arm",
|
|
60
|
+
"i386": "386",
|
|
61
|
+
"i686": "386",
|
|
62
|
+
"x86": "386",
|
|
63
|
+
"x86_64": "amd64",
|
|
64
|
+
}
|
|
65
|
+
return architectures.get(machine, machine or None)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _execution_environment(environ: Mapping[str, str]) -> str | None:
|
|
69
|
+
signals = (
|
|
70
|
+
(environ.get("GITHUB_ACTIONS") == "true", "github-actions"),
|
|
71
|
+
(environ.get("GITLAB_CI") == "true", "gitlab-ci"),
|
|
72
|
+
(_env_set(environ, "BUILDKITE"), "buildkite"),
|
|
73
|
+
(_env_set(environ, "CIRCLECI"), "circleci"),
|
|
74
|
+
(_env_set(environ, "JENKINS_URL"), "jenkins"),
|
|
75
|
+
(_env_set(environ, "TEAMCITY_VERSION"), "teamcity"),
|
|
76
|
+
(_env_set(environ, "TF_BUILD"), "azure-pipelines"),
|
|
77
|
+
(_env_set(environ, "K_SERVICE", "CLOUD_RUN_JOB"), "google-cloud-run"),
|
|
78
|
+
(environ.get("AWS_EXECUTION_ENV", "").startswith("AWS_Lambda_"), "aws-lambda"),
|
|
79
|
+
(_env_set(environ, "FUNCTIONS_WORKER_RUNTIME"), "azure-functions"),
|
|
80
|
+
(_env_set(environ, "KUBERNETES_SERVICE_HOST"), "kubernetes"),
|
|
81
|
+
)
|
|
82
|
+
environment = next((name for detected, name in signals if detected), None)
|
|
83
|
+
|
|
84
|
+
if environment is not None:
|
|
85
|
+
return environment
|
|
86
|
+
|
|
87
|
+
shell = _interactive_shell()
|
|
88
|
+
if _env_set(environ, "COLAB_RELEASE_TAG") or shell == "google-colab":
|
|
89
|
+
environment = "google-colab"
|
|
90
|
+
elif shell == "jupyter" or _env_set(environ, "JPY_PARENT_PID"):
|
|
91
|
+
jupyterlab_variables = ("JUPYTERLAB_DIR", "JUPYTERLAB_SETTINGS_DIR", "JUPYTERLAB_WORKSPACES_DIR")
|
|
92
|
+
environment = "jupyterlab" if _env_set(environ, *jupyterlab_variables) else "jupyter"
|
|
93
|
+
elif shell == "terminal" or hasattr(sys, "ps1") or (sys.stdin is not None and sys.stdin.isatty()):
|
|
94
|
+
environment = "terminal"
|
|
95
|
+
return environment
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _interactive_shell() -> str | None:
|
|
99
|
+
get_ipython = getattr(builtins, "get_ipython", None)
|
|
100
|
+
if not callable(get_ipython):
|
|
101
|
+
return None
|
|
102
|
+
shell = get_ipython()
|
|
103
|
+
shell_type = type(shell)
|
|
104
|
+
qualified_name = f"{shell_type.__module__}.{shell_type.__name__}".lower()
|
|
105
|
+
if "google.colab" in qualified_name:
|
|
106
|
+
return "google-colab"
|
|
107
|
+
if "zmqinteractiveshell" in qualified_name:
|
|
108
|
+
return "jupyter"
|
|
109
|
+
if "terminalinteractiveshell" in qualified_name:
|
|
110
|
+
return "terminal"
|
|
111
|
+
return None
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _invoker(environ: Mapping[str, str]) -> dict[str, str]:
|
|
115
|
+
invoker = None
|
|
116
|
+
version = None
|
|
117
|
+
if environ.get("AGENT") == "amp":
|
|
118
|
+
invoker = "amp"
|
|
119
|
+
elif _env_set(environ, "COPILOT_AGENT_SESSION_ID"):
|
|
120
|
+
invoker = "github-copilot"
|
|
121
|
+
elif environ.get("OPENCODE") == "1":
|
|
122
|
+
invoker = "opencode"
|
|
123
|
+
elif _env_set(environ, "CLAUDECODE"):
|
|
124
|
+
invoker = "claude-code"
|
|
125
|
+
elif _env_set(environ, "CURSOR_AGENT"):
|
|
126
|
+
invoker = "cursor"
|
|
127
|
+
elif _env_set(environ, "CODEX_SESSION_ID", "CODEX_THREAD_ID"):
|
|
128
|
+
invoker = "codex"
|
|
129
|
+
version = environ.get("CODEX_VERSION")
|
|
130
|
+
elif _env_set(environ, "GEMINI_CLI"):
|
|
131
|
+
invoker = "gemini-cli"
|
|
132
|
+
return _optional_version(invoker, version) if invoker else {}
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _optional_version(invoker: str, version: str | None) -> dict[str, str]:
|
|
136
|
+
metadata = {"invoker": invoker}
|
|
137
|
+
if version:
|
|
138
|
+
metadata["invoker-version"] = version
|
|
139
|
+
return metadata
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _cloud_environment(environ: Mapping[str, str]) -> dict[str, str]:
|
|
143
|
+
provider, platform_name, region = _detect_cloud_environment(environ)
|
|
144
|
+
return {
|
|
145
|
+
key: value
|
|
146
|
+
for key, value in (
|
|
147
|
+
("cloud-provider", provider),
|
|
148
|
+
("cloud-platform", platform_name),
|
|
149
|
+
("cloud-region", region),
|
|
150
|
+
)
|
|
151
|
+
if value
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _detect_cloud_environment(environ: Mapping[str, str]) -> tuple[str | None, str | None, str | None]:
|
|
156
|
+
if _env_set(environ, "AWS_EXECUTION_ENV", "AWS_REGION", "AWS_DEFAULT_REGION"):
|
|
157
|
+
execution_environment = environ.get("AWS_EXECUTION_ENV", "")
|
|
158
|
+
platform_name = None
|
|
159
|
+
if execution_environment.startswith("AWS_Lambda_"):
|
|
160
|
+
platform_name = "aws_lambda"
|
|
161
|
+
elif execution_environment.startswith("AWS_ECS_"):
|
|
162
|
+
platform_name = "aws_ecs"
|
|
163
|
+
return "aws", platform_name, _first_environment_value(environ, "AWS_REGION", "AWS_DEFAULT_REGION")
|
|
164
|
+
|
|
165
|
+
if _env_set(environ, "K_SERVICE", "CLOUD_RUN_JOB", "GAE_ENV"):
|
|
166
|
+
platform_name = "gcp_cloud_run" if _env_set(environ, "K_SERVICE", "CLOUD_RUN_JOB") else "gcp_app_engine"
|
|
167
|
+
region = _first_environment_value(environ, "GOOGLE_CLOUD_REGION", "CLOUD_RUN_REGION", "FUNCTION_REGION")
|
|
168
|
+
return "gcp", platform_name, region
|
|
169
|
+
|
|
170
|
+
if _env_set(environ, "WEBSITE_INSTANCE_ID", "FUNCTIONS_WORKER_RUNTIME", "REGION_NAME"):
|
|
171
|
+
platform_name = None
|
|
172
|
+
if _env_set(environ, "FUNCTIONS_WORKER_RUNTIME"):
|
|
173
|
+
platform_name = "azure_functions"
|
|
174
|
+
elif _env_set(environ, "WEBSITE_INSTANCE_ID"):
|
|
175
|
+
platform_name = "azure_app_service"
|
|
176
|
+
return "azure", platform_name, environ.get("REGION_NAME")
|
|
177
|
+
|
|
178
|
+
return None, None, None
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _env_set(environ: Mapping[str, str], *names: str) -> bool:
|
|
182
|
+
return any(environ.get(name) for name in names)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _first_environment_value(environ: Mapping[str, str], *names: str) -> str | None:
|
|
186
|
+
return next((value for name in names if (value := environ.get(name))), None)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _serialize_dictionary(fields: Mapping[str, str | None]) -> str:
|
|
190
|
+
members: list[str] = []
|
|
191
|
+
header_size = 0
|
|
192
|
+
for key, value in fields.items():
|
|
193
|
+
if not value or not _valid_string(value):
|
|
194
|
+
continue
|
|
195
|
+
escaped = value.replace("\\", "\\\\").replace('"', '\\"')
|
|
196
|
+
member = f'{key}="{escaped}"'
|
|
197
|
+
member_size = len(member.encode("ascii")) + (2 if members else 0)
|
|
198
|
+
if header_size + member_size > _MAX_HEADER_BYTES:
|
|
199
|
+
continue
|
|
200
|
+
members.append(member)
|
|
201
|
+
header_size += member_size
|
|
202
|
+
return ", ".join(members)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _valid_string(value: str) -> bool:
|
|
206
|
+
try:
|
|
207
|
+
encoded = value.encode("ascii")
|
|
208
|
+
except UnicodeEncodeError:
|
|
209
|
+
return False
|
|
210
|
+
return len(encoded) <= _MAX_FIELD_BYTES and all(0x20 <= byte <= 0x7E for byte in encoded)
|
_tilebox/concurrency.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""Bounded producer/consumer concurrency with exception propagation."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Callable, Iterator
|
|
4
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
5
|
+
from queue import Empty, Full, Queue
|
|
6
|
+
from threading import Event
|
|
7
|
+
from typing import TypeVar
|
|
8
|
+
|
|
9
|
+
T = TypeVar("T")
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def concurrent_producer_consumer(
|
|
13
|
+
producer: Iterator[T],
|
|
14
|
+
consumer: Callable[[T], object],
|
|
15
|
+
buffer_size: int = 10,
|
|
16
|
+
) -> None:
|
|
17
|
+
"""Produce on the calling thread and consume on one worker, propagating failures."""
|
|
18
|
+
if buffer_size < 1:
|
|
19
|
+
raise ValueError("buffer_size must be positive")
|
|
20
|
+
queue: Queue[T] = Queue(buffer_size)
|
|
21
|
+
done = Event()
|
|
22
|
+
|
|
23
|
+
def consume() -> None:
|
|
24
|
+
while not done.is_set() or not queue.empty():
|
|
25
|
+
try:
|
|
26
|
+
item = queue.get(timeout=0.05)
|
|
27
|
+
except Empty:
|
|
28
|
+
continue
|
|
29
|
+
consumer(item)
|
|
30
|
+
|
|
31
|
+
with ThreadPoolExecutor(max_workers=1) as executor:
|
|
32
|
+
future = executor.submit(consume)
|
|
33
|
+
try:
|
|
34
|
+
for item in producer:
|
|
35
|
+
while True:
|
|
36
|
+
if future.done():
|
|
37
|
+
future.result()
|
|
38
|
+
try:
|
|
39
|
+
queue.put(item, timeout=0.05)
|
|
40
|
+
break
|
|
41
|
+
except Full:
|
|
42
|
+
continue
|
|
43
|
+
finally:
|
|
44
|
+
done.set()
|
|
45
|
+
future.result()
|
_tilebox/errors.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Translate Connect failures into public Tilebox exceptions."""
|
|
2
|
+
|
|
3
|
+
from connectrpc.code import Code
|
|
4
|
+
from connectrpc.errors import ConnectError
|
|
5
|
+
|
|
6
|
+
from tilebox.errors import (
|
|
7
|
+
ArgumentError,
|
|
8
|
+
AuthenticationError,
|
|
9
|
+
InternalServerError,
|
|
10
|
+
InvalidRequestError,
|
|
11
|
+
NetworkError,
|
|
12
|
+
NetworkTimeoutError,
|
|
13
|
+
NotFoundError,
|
|
14
|
+
RequestCanceledError,
|
|
15
|
+
SubscriptionLimitExceededError,
|
|
16
|
+
TileboxError,
|
|
17
|
+
TooManyRequestsError,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def translate_connect_error(error: ConnectError, method_name: str = "") -> TileboxError: # noqa: C901, PLR0911, PLR0912
|
|
22
|
+
"""Map native Connect status codes to the SDK's public exception types."""
|
|
23
|
+
message = error.message
|
|
24
|
+
public = method_name.startswith("/public/")
|
|
25
|
+
match error.code:
|
|
26
|
+
case Code.NOT_FOUND:
|
|
27
|
+
return NotFoundError(message)
|
|
28
|
+
case Code.INVALID_ARGUMENT:
|
|
29
|
+
return ArgumentError(message)
|
|
30
|
+
case Code.UNAUTHENTICATED:
|
|
31
|
+
return AuthenticationError(f"Unauthenticated: {message}")
|
|
32
|
+
case Code.PERMISSION_DENIED:
|
|
33
|
+
return AuthenticationError(f"Unauthorized: {message}")
|
|
34
|
+
case Code.RESOURCE_EXHAUSTED | Code.UNAVAILABLE:
|
|
35
|
+
if "429" in message or "too many requests" in message.lower():
|
|
36
|
+
text = "Too many requests, try again later."
|
|
37
|
+
if public:
|
|
38
|
+
text += (
|
|
39
|
+
" To increase your request quota create a free account at "
|
|
40
|
+
"https://console.tilebox.com and authenticate with an API Key."
|
|
41
|
+
)
|
|
42
|
+
return TooManyRequestsError(text)
|
|
43
|
+
if error.code == Code.RESOURCE_EXHAUSTED:
|
|
44
|
+
return SubscriptionLimitExceededError(message)
|
|
45
|
+
return NetworkError(f"network error when requesting {method_name}: {message}" if method_name else message)
|
|
46
|
+
case Code.DEADLINE_EXCEEDED:
|
|
47
|
+
return NetworkTimeoutError(f"Request timed out: {message}")
|
|
48
|
+
case Code.ABORTED:
|
|
49
|
+
return NetworkError(f"Request aborted: {message}")
|
|
50
|
+
case Code.CANCELED:
|
|
51
|
+
return RequestCanceledError(f"Request canceled: {message}")
|
|
52
|
+
case Code.UNIMPLEMENTED:
|
|
53
|
+
if public:
|
|
54
|
+
action = method_name.rsplit("/", maxsplit=1)[-1]
|
|
55
|
+
return AuthenticationError(
|
|
56
|
+
f"{action} is only available for authenticated users. Please provide a valid API Key "
|
|
57
|
+
"token in your Client. Create a free account at https://console.tilebox.com"
|
|
58
|
+
)
|
|
59
|
+
return InvalidRequestError(
|
|
60
|
+
f"Requested URL {method_name} not found. Update your Tilebox Python Clients to the latest version."
|
|
61
|
+
if method_name
|
|
62
|
+
else message
|
|
63
|
+
)
|
|
64
|
+
case _:
|
|
65
|
+
return InternalServerError(f"Oops, something went wrong: {message}")
|
_tilebox/pagination.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
from collections.abc import Callable, Iterator
|
|
2
|
+
from typing import Protocol, TypeVar
|
|
3
|
+
from uuid import UUID
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Pagination(Protocol):
|
|
7
|
+
limit: int | None
|
|
8
|
+
starting_after: UUID | None
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class ResultPage(Protocol):
|
|
12
|
+
@property
|
|
13
|
+
def next_page(self) -> Pagination: ...
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
AnyResultPage = TypeVar("AnyResultPage", bound=ResultPage)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def paginated_request(
|
|
20
|
+
paging_request: Callable[[Pagination], AnyResultPage],
|
|
21
|
+
initial_page: Pagination,
|
|
22
|
+
) -> Iterator[AnyResultPage]:
|
|
23
|
+
"""Make a paginated request to a service endpoint.
|
|
24
|
+
|
|
25
|
+
The endpoint is expected to return a next_page field, which is used for subsequent requests. Once no such
|
|
26
|
+
next_page field is returned, the request is completed.
|
|
27
|
+
|
|
28
|
+
Args:
|
|
29
|
+
paging_request: A function that takes a page as input and returns a Datapoints object
|
|
30
|
+
Often this will be a functools.partial object that wraps a gRPC service endpoint
|
|
31
|
+
and only leaves the page argument remaining
|
|
32
|
+
initial_page: The initial page to request
|
|
33
|
+
|
|
34
|
+
Yields:
|
|
35
|
+
Datapoints: The individual pages of the response
|
|
36
|
+
"""
|
|
37
|
+
response = paging_request(initial_page)
|
|
38
|
+
yield response
|
|
39
|
+
|
|
40
|
+
while response.next_page.starting_after is not None:
|
|
41
|
+
response = paging_request(response.next_page)
|
|
42
|
+
yield response
|