tilebox-workflows 0.63.2__tar.gz → 0.65.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/PKG-INFO +6 -6
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/pyproject.toml +5 -5
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/__init__.py +12 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/automations/client.py +0 -8
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/automations/service.py +1 -16
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/automations/storage_event.py +1 -1
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/client.py +7 -1
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/data.py +149 -49
- tilebox_workflows-0.65.0/tilebox/workflows/observability/_log_stream.py +172 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/observability/logging.py +76 -50
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/observability/tracing.py +8 -13
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/runner/__main__.py +10 -5
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/runner/executor.py +1 -1
- tilebox_workflows-0.65.0/tilebox/workflows/runner/worker_server.py +128 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/runner/worker_service.py +73 -7
- tilebox_workflows-0.65.0/tilebox/workflows/storage_locations/__init__.py +35 -0
- tilebox_workflows-0.65.0/tilebox/workflows/storage_locations/client.py +38 -0
- tilebox_workflows-0.65.0/tilebox/workflows/storage_locations/data.py +60 -0
- tilebox_workflows-0.65.0/tilebox/workflows/storage_locations/service.py +46 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/automation_pb2.py +2 -2
- tilebox_workflows-0.65.0/tilebox/workflows/workflows/v1/storage_location_pb2.py +114 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/storage_location_pb2.pyi +17 -13
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/worker_connect.py +66 -0
- tilebox_workflows-0.65.0/tilebox/workflows/workflows/v1/worker_pb2.py +51 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/worker_pb2.pyi +1 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/worker_pb2_grpc.py +57 -1
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/workflows_connect.py +65 -0
- tilebox_workflows-0.65.0/tilebox/workflows/workflows/v1/workflows_pb2.py +115 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/workflows_pb2.pyi +23 -4
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/workflows_pb2_grpc.py +43 -0
- tilebox_workflows-0.63.2/tilebox/workflows/observability/_log_pipe.py +0 -124
- tilebox_workflows-0.63.2/tilebox/workflows/runner/worker_server.py +0 -67
- tilebox_workflows-0.63.2/tilebox/workflows/workflows/v1/storage_location_pb2.py +0 -114
- tilebox_workflows-0.63.2/tilebox/workflows/workflows/v1/worker_pb2.py +0 -50
- tilebox_workflows-0.63.2/tilebox/workflows/workflows/v1/workflows_pb2.py +0 -102
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/.gitignore +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/README.md +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/_codec.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/_codecs/__init__.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/_codecs/affine.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/_codecs/async_geotiff.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/_codecs/odc.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/_codecs/pyproj.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/_codecs/rasterio.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/_codecs/shapely.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/_codecs/tilebox.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/_serialization.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/automations/__init__.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/automations/cron.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/cache.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/clusters/__init__.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/clusters/client.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/clusters/service.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/formatting/__init__.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/formatting/job.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/jobs/__init__.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/jobs/client.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/jobs/service.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/jobs/telemetry_service.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/observability/__init__.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/observability/_logging.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/runner/__init__.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/runner/runner.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/runner/task_runner.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/runner/task_service.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/task.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/__init__.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/client.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/service.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/automation_connect.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/automation_pb2.pyi +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/automation_pb2_grpc.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/core_pb2.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/core_pb2.pyi +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/core_pb2_grpc.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/diagram_connect.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/diagram_pb2.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/diagram_pb2.pyi +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/diagram_pb2_grpc.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/job_connect.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/job_pb2.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/job_pb2.pyi +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/job_pb2_grpc.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/storage_location_connect.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/storage_location_pb2_grpc.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/task_connect.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/task_pb2.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/task_pb2.pyi +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/task_pb2_grpc.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/task_release_pb2.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/task_release_pb2.pyi +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/task_release_pb2_grpc.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/telemetry_connect.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/telemetry_pb2.py +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/telemetry_pb2.pyi +0 -0
- {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/telemetry_pb2_grpc.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: tilebox-workflows
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.65.0
|
|
4
4
|
Summary: Workflow client and task runner for Tilebox
|
|
5
5
|
Project-URL: Homepage, https://tilebox.com
|
|
6
6
|
Project-URL: Documentation, https://docs.tilebox.com/workflows/introduction
|
|
@@ -25,11 +25,11 @@ Requires-Dist: google-auth[requests]>=2.29
|
|
|
25
25
|
Requires-Dist: grpcio>=1.84.0
|
|
26
26
|
Requires-Dist: msgspec>=0.19
|
|
27
27
|
Requires-Dist: obstore>=0.8.2
|
|
28
|
-
Requires-Dist: opentelemetry-api>=1.
|
|
29
|
-
Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.
|
|
30
|
-
Requires-Dist: opentelemetry-instrumentation-logging>=0.
|
|
31
|
-
Requires-Dist: opentelemetry-proto>=1.
|
|
32
|
-
Requires-Dist: opentelemetry-sdk>=1.
|
|
28
|
+
Requires-Dist: opentelemetry-api>=1.44.0
|
|
29
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.44.0
|
|
30
|
+
Requires-Dist: opentelemetry-instrumentation-logging>=0.65b0
|
|
31
|
+
Requires-Dist: opentelemetry-proto>=1.44.0
|
|
32
|
+
Requires-Dist: opentelemetry-sdk>=1.44.0
|
|
33
33
|
Requires-Dist: tenacity>=8
|
|
34
34
|
Requires-Dist: tilebox-datasets>=0.56.0
|
|
35
35
|
Requires-Dist: tilebox-grpc>=0.28.0
|
|
@@ -27,14 +27,14 @@ dependencies = [
|
|
|
27
27
|
"grpcio>=1.84.0",
|
|
28
28
|
"google-auth[requests]>=2.29",
|
|
29
29
|
"azure-identity>=1.23",
|
|
30
|
-
"opentelemetry-api>=1.
|
|
31
|
-
"opentelemetry-exporter-otlp-proto-http>=1.
|
|
32
|
-
"opentelemetry-sdk>=1.
|
|
30
|
+
"opentelemetry-api>=1.44.0",
|
|
31
|
+
"opentelemetry-exporter-otlp-proto-http>=1.44.0",
|
|
32
|
+
"opentelemetry-sdk>=1.44.0",
|
|
33
33
|
"tenacity>=8",
|
|
34
34
|
"boto3>=1.40.2",
|
|
35
35
|
"obstore>=0.8.2",
|
|
36
|
-
"opentelemetry-proto>=1.
|
|
37
|
-
"opentelemetry-instrumentation-logging>=0.
|
|
36
|
+
"opentelemetry-proto>=1.44.0",
|
|
37
|
+
"opentelemetry-instrumentation-logging>=0.65b0",
|
|
38
38
|
"msgspec>=0.19",
|
|
39
39
|
]
|
|
40
40
|
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import os
|
|
2
2
|
from typing import TYPE_CHECKING, Any
|
|
3
3
|
|
|
4
|
+
# Apply native logging defaults before any submodule can initialize gRPC.
|
|
5
|
+
import _tilebox.grpc # noqa: F401
|
|
6
|
+
|
|
4
7
|
if TYPE_CHECKING:
|
|
5
8
|
from tilebox.workflows.client import Client
|
|
6
9
|
from tilebox.workflows.data import Job
|
|
@@ -48,8 +51,17 @@ def __dir__() -> list[str]:
|
|
|
48
51
|
def _initialize_logging_from_environment() -> None:
|
|
49
52
|
# Stage 1: current CLIs provide credentials before importing workflow code, so
|
|
50
53
|
# even import-time logs reach the API. Ordinary SDK imports stay lightweight.
|
|
54
|
+
runtime_dir = os.environ.get("TILEBOX_RUNTIME_DIR")
|
|
55
|
+
runtime_token = os.environ.get("TILEBOX_RUNTIME_TOKEN")
|
|
56
|
+
if bool(runtime_dir) != bool(runtime_token):
|
|
57
|
+
raise RuntimeError("TILEBOX_RUNTIME_DIR and TILEBOX_RUNTIME_TOKEN must be set together")
|
|
58
|
+
|
|
51
59
|
url = os.environ.get("TILEBOX_API_URL")
|
|
52
60
|
token = os.environ.get("TILEBOX_API_KEY")
|
|
61
|
+
if runtime_dir and runtime_token:
|
|
62
|
+
# Importing installs the managed local stream before workflow modules can log.
|
|
63
|
+
from tilebox.workflows.observability import logging as _runtime_logging # noqa: F401, PLC0415
|
|
64
|
+
|
|
53
65
|
if not url or not token:
|
|
54
66
|
return
|
|
55
67
|
|
{tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/automations/client.py
RENAMED
|
@@ -18,14 +18,6 @@ class AutomationClient:
|
|
|
18
18
|
def __init__(self, service: AutomationService) -> None:
|
|
19
19
|
self._service = service
|
|
20
20
|
|
|
21
|
-
def storage_locations(self) -> list[StorageLocation]:
|
|
22
|
-
"""List all available storage locations that can potentially be used as storage event triggers.
|
|
23
|
-
|
|
24
|
-
Returns:
|
|
25
|
-
A list of all available storage locations.
|
|
26
|
-
"""
|
|
27
|
-
return self._service.list_storage_locations()
|
|
28
|
-
|
|
29
21
|
def all(self) -> list[AutomationPrototype]:
|
|
30
22
|
"""List all registered automations.
|
|
31
23
|
|
{tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/automations/service.py
RENAMED
|
@@ -9,39 +9,24 @@ from _tilebox.grpc.error import with_pythonic_errors
|
|
|
9
9
|
from tilebox.datasets.uuid import uuid_to_uuid_message
|
|
10
10
|
from tilebox.workflows.data import (
|
|
11
11
|
AutomationPrototype,
|
|
12
|
-
StorageLocation,
|
|
13
12
|
)
|
|
14
13
|
from tilebox.workflows.workflows.v1.automation_pb2 import AutomationPrototype as AutomationPrototypeMessage
|
|
15
14
|
from tilebox.workflows.workflows.v1.automation_pb2 import Automations, DeleteAutomationRequest
|
|
16
15
|
from tilebox.workflows.workflows.v1.automation_pb2_grpc import AutomationServiceStub
|
|
17
|
-
from tilebox.workflows.workflows.v1.storage_location_pb2 import StorageLocations
|
|
18
|
-
from tilebox.workflows.workflows.v1.storage_location_pb2_grpc import StorageLocationServiceStub
|
|
19
16
|
|
|
20
17
|
|
|
21
18
|
class AutomationService:
|
|
22
|
-
def __init__(self, channel: Channel | Any
|
|
19
|
+
def __init__(self, channel: Channel | Any) -> None:
|
|
23
20
|
"""
|
|
24
21
|
A wrapper around the AutomationServiceStub that provides a more pythonic interface and converts the protobuf
|
|
25
22
|
messages to and from the data classes used in the rest of the tilebox-workflows codebase.
|
|
26
23
|
|
|
27
24
|
Args:
|
|
28
25
|
channel: The gRPC channel to use for the service.
|
|
29
|
-
storage_channel: Optional separate storage-location client for the Connect transport.
|
|
30
26
|
"""
|
|
31
27
|
self.service = (
|
|
32
28
|
with_pythonic_errors(AutomationServiceStub(channel)) if hasattr(channel, "unary_unary") else channel
|
|
33
29
|
)
|
|
34
|
-
if storage_channel is None:
|
|
35
|
-
storage_channel = channel
|
|
36
|
-
self.storage_service = (
|
|
37
|
-
with_pythonic_errors(StorageLocationServiceStub(storage_channel))
|
|
38
|
-
if hasattr(storage_channel, "unary_unary")
|
|
39
|
-
else storage_channel
|
|
40
|
-
)
|
|
41
|
-
|
|
42
|
-
def list_storage_locations(self) -> builtins.list[StorageLocation]:
|
|
43
|
-
response: StorageLocations = self.storage_service.ListStorageLocations(Empty())
|
|
44
|
-
return [StorageLocation.from_message(sl) for sl in response.locations]
|
|
45
30
|
|
|
46
31
|
def list(self) -> builtins.list[AutomationPrototype]:
|
|
47
32
|
response: Automations = self.service.ListAutomations(Empty())
|
{tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/automations/storage_event.py
RENAMED
|
@@ -13,7 +13,7 @@ from tilebox.workflows.workflows.v1.automation_pb2 import Automation as Automati
|
|
|
13
13
|
from tilebox.workflows.workflows.v1.storage_location_pb2 import TriggeredStorageEvent as TriggeredStorageEventMessage
|
|
14
14
|
|
|
15
15
|
_NOT_TRIGGERED = TriggeredStorageEvent(
|
|
16
|
-
StorageLocation(UUID(int=0), "", StorageType.
|
|
16
|
+
StorageLocation(UUID(int=0), "", StorageType.LOCAL),
|
|
17
17
|
StorageEventType.CREATED,
|
|
18
18
|
"",
|
|
19
19
|
)
|
|
@@ -23,6 +23,8 @@ from tilebox.workflows.observability.tracing import WorkflowTracer
|
|
|
23
23
|
from tilebox.workflows.runner.executor import LazyStorageLocations
|
|
24
24
|
from tilebox.workflows.runner.task_runner import TaskRunner, _LeaseRenewer
|
|
25
25
|
from tilebox.workflows.runner.task_service import TaskService
|
|
26
|
+
from tilebox.workflows.storage_locations.client import StorageLocationClient
|
|
27
|
+
from tilebox.workflows.storage_locations.service import StorageLocationService
|
|
26
28
|
from tilebox.workflows.task import Task
|
|
27
29
|
from tilebox.workflows.workflows.client import WorkflowClient
|
|
28
30
|
from tilebox.workflows.workflows.service import WorkflowService
|
|
@@ -207,7 +209,11 @@ class Client:
|
|
|
207
209
|
Returns:
|
|
208
210
|
A client for the automations service.
|
|
209
211
|
"""
|
|
210
|
-
return AutomationClient(AutomationService(self._automation_service
|
|
212
|
+
return AutomationClient(AutomationService(self._automation_service))
|
|
213
|
+
|
|
214
|
+
def storage_locations(self) -> StorageLocationClient:
|
|
215
|
+
"""Get a client for storage locations."""
|
|
216
|
+
return StorageLocationClient(StorageLocationService(self._storage_location_service))
|
|
211
217
|
|
|
212
218
|
|
|
213
219
|
def _token_from_env(url: str, token: str | None) -> str | None:
|
|
@@ -969,13 +969,67 @@ class QueryJobSpansResponse:
|
|
|
969
969
|
|
|
970
970
|
|
|
971
971
|
class StorageType(Enum):
|
|
972
|
-
GCS = storage_pb.
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
972
|
+
GCS = storage_pb.STORAGE_TYPE_GCS # Google Cloud Storage
|
|
973
|
+
AWS_S3 = storage_pb.STORAGE_TYPE_AWS_S3 # Amazon Web Services S3
|
|
974
|
+
LOCAL = storage_pb.STORAGE_TYPE_LOCAL # Local Filesystem
|
|
975
|
+
AZURE_BLOB = storage_pb.STORAGE_TYPE_AZURE_BLOB # Azure Blob Storage
|
|
976
976
|
|
|
977
977
|
|
|
978
|
-
|
|
978
|
+
@dataclass(frozen=True)
|
|
979
|
+
class AWSS3BucketReference:
|
|
980
|
+
bucket: str
|
|
981
|
+
region: str = ""
|
|
982
|
+
|
|
983
|
+
def to_message(self) -> storage_pb.StorageLocationReference:
|
|
984
|
+
return storage_pb.StorageLocationReference(
|
|
985
|
+
type=storage_pb.STORAGE_TYPE_AWS_S3,
|
|
986
|
+
aws_s3_bucket=storage_pb.AWSS3BucketReference(bucket=self.bucket, region=self.region),
|
|
987
|
+
)
|
|
988
|
+
|
|
989
|
+
|
|
990
|
+
@dataclass(frozen=True)
|
|
991
|
+
class GCSBucketReference:
|
|
992
|
+
bucket: str
|
|
993
|
+
project_id: str = ""
|
|
994
|
+
location: str = ""
|
|
995
|
+
|
|
996
|
+
def to_message(self) -> storage_pb.StorageLocationReference:
|
|
997
|
+
return storage_pb.StorageLocationReference(
|
|
998
|
+
type=storage_pb.STORAGE_TYPE_GCS,
|
|
999
|
+
gcs_bucket=storage_pb.GCSBucketReference(
|
|
1000
|
+
bucket=self.bucket, project_id=self.project_id, location=self.location
|
|
1001
|
+
),
|
|
1002
|
+
)
|
|
1003
|
+
|
|
1004
|
+
|
|
1005
|
+
@dataclass(frozen=True)
|
|
1006
|
+
class AzureBlobReference:
|
|
1007
|
+
storage_account_resource_id: str
|
|
1008
|
+
container: str
|
|
1009
|
+
region: str = ""
|
|
1010
|
+
|
|
1011
|
+
def to_message(self) -> storage_pb.StorageLocationReference:
|
|
1012
|
+
return storage_pb.StorageLocationReference(
|
|
1013
|
+
type=storage_pb.STORAGE_TYPE_AZURE_BLOB,
|
|
1014
|
+
azure_blob=storage_pb.AzureBlobReference(
|
|
1015
|
+
storage_account_resource_id=self.storage_account_resource_id,
|
|
1016
|
+
container=self.container,
|
|
1017
|
+
region=self.region,
|
|
1018
|
+
),
|
|
1019
|
+
)
|
|
1020
|
+
|
|
1021
|
+
|
|
1022
|
+
@dataclass(frozen=True)
|
|
1023
|
+
class LocalReference:
|
|
1024
|
+
path: str
|
|
1025
|
+
|
|
1026
|
+
def to_message(self) -> storage_pb.StorageLocationReference:
|
|
1027
|
+
return storage_pb.StorageLocationReference(
|
|
1028
|
+
type=storage_pb.STORAGE_TYPE_LOCAL, local=storage_pb.LocalReference(path=self.path)
|
|
1029
|
+
)
|
|
1030
|
+
|
|
1031
|
+
|
|
1032
|
+
StorageLocationReference: TypeAlias = AWSS3BucketReference | GCSBucketReference | AzureBlobReference | LocalReference
|
|
979
1033
|
|
|
980
1034
|
|
|
981
1035
|
@total_ordering
|
|
@@ -985,53 +1039,81 @@ class StorageLocation:
|
|
|
985
1039
|
location: str
|
|
986
1040
|
type: StorageType
|
|
987
1041
|
runner_context: "RunnerContext | None" = None
|
|
1042
|
+
name: str = field(default="", kw_only=True)
|
|
988
1043
|
|
|
989
1044
|
def __lt__(self, other: object) -> bool:
|
|
990
1045
|
if not isinstance(other, StorageLocation):
|
|
991
1046
|
return NotImplemented
|
|
992
1047
|
return (self.id, self.location, self.type.value) < (other.id, other.location, other.type.value)
|
|
993
1048
|
|
|
1049
|
+
def __str__(self) -> str:
|
|
1050
|
+
match self.type:
|
|
1051
|
+
case StorageType.AWS_S3:
|
|
1052
|
+
return f"s3://{self.location}"
|
|
1053
|
+
case StorageType.GCS:
|
|
1054
|
+
return f"gs://{self.location}"
|
|
1055
|
+
case StorageType.LOCAL:
|
|
1056
|
+
return self.location
|
|
1057
|
+
case _:
|
|
1058
|
+
return self.type.name.lower().replace("_", "-")
|
|
1059
|
+
|
|
994
1060
|
@classmethod
|
|
995
1061
|
def from_message(cls, storage_location: storage_pb.StorageLocation) -> "StorageLocation":
|
|
996
1062
|
"""Convert a StorageLocation protobuf message to a StorageLocation object."""
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1063
|
+
reference = storage_location.reference
|
|
1064
|
+
match reference.type:
|
|
1065
|
+
case storage_pb.STORAGE_TYPE_AZURE_BLOB:
|
|
1066
|
+
return AzureBlobStorageLocation(
|
|
1067
|
+
id=uuid_message_to_uuid(storage_location.id),
|
|
1068
|
+
name=storage_location.name,
|
|
1069
|
+
location=reference.azure_blob.container,
|
|
1070
|
+
type=StorageType.AZURE_BLOB,
|
|
1071
|
+
storage_account_resource_id=reference.azure_blob.storage_account_resource_id,
|
|
1072
|
+
region=reference.azure_blob.region,
|
|
1073
|
+
)
|
|
1074
|
+
case storage_pb.STORAGE_TYPE_GCS:
|
|
1075
|
+
return GCSStorageLocation(
|
|
1076
|
+
id=uuid_message_to_uuid(storage_location.id),
|
|
1077
|
+
name=storage_location.name,
|
|
1078
|
+
location=reference.gcs_bucket.bucket,
|
|
1079
|
+
type=StorageType.GCS,
|
|
1080
|
+
project_id=reference.gcs_bucket.project_id,
|
|
1081
|
+
bucket_location=reference.gcs_bucket.location,
|
|
1082
|
+
)
|
|
1083
|
+
case storage_pb.STORAGE_TYPE_AWS_S3:
|
|
1084
|
+
return AWSS3StorageLocation(
|
|
1085
|
+
id=uuid_message_to_uuid(storage_location.id),
|
|
1086
|
+
name=storage_location.name,
|
|
1087
|
+
location=reference.aws_s3_bucket.bucket,
|
|
1088
|
+
type=StorageType.AWS_S3,
|
|
1089
|
+
region=reference.aws_s3_bucket.region or None,
|
|
1090
|
+
)
|
|
1091
|
+
case storage_pb.STORAGE_TYPE_LOCAL:
|
|
1092
|
+
return cls(
|
|
1093
|
+
id=uuid_message_to_uuid(storage_location.id),
|
|
1094
|
+
name=storage_location.name,
|
|
1095
|
+
location=reference.local.path,
|
|
1096
|
+
type=StorageType.LOCAL,
|
|
1097
|
+
)
|
|
1098
|
+
case _:
|
|
1099
|
+
raise ValueError(f"Unsupported storage reference type: {reference.type}")
|
|
1026
1100
|
|
|
1027
1101
|
def _with_runner_context(self, runner_context: "RunnerContext") -> "StorageLocation":
|
|
1028
1102
|
return replace(self, runner_context=runner_context)
|
|
1029
1103
|
|
|
1030
1104
|
def to_message(self) -> storage_pb.StorageLocation:
|
|
1031
1105
|
"""Convert a StorageLocation object to a StorageLocation protobuf message."""
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1106
|
+
reference = storage_pb.StorageLocationReference(type=self.type.value)
|
|
1107
|
+
match self.type:
|
|
1108
|
+
case StorageType.AWS_S3:
|
|
1109
|
+
reference.aws_s3_bucket.bucket = self.location
|
|
1110
|
+
case StorageType.GCS:
|
|
1111
|
+
reference.gcs_bucket.bucket = self.location
|
|
1112
|
+
case StorageType.AZURE_BLOB:
|
|
1113
|
+
reference.azure_blob.container = self.location
|
|
1114
|
+
case StorageType.LOCAL:
|
|
1115
|
+
reference.local.path = self.location
|
|
1116
|
+
return storage_pb.StorageLocation(id=uuid_to_uuid_message(self.id), name=self.name, reference=reference)
|
|
1035
1117
|
|
|
1036
1118
|
def read(self, path: str) -> bytes:
|
|
1037
1119
|
runner_context = self.runner_context or RunnerContext()
|
|
@@ -1042,21 +1124,26 @@ class StorageLocation:
|
|
|
1042
1124
|
case StorageType.GCS:
|
|
1043
1125
|
span.set_attribute("bucket", self.location)
|
|
1044
1126
|
store = runner_context.gcs_client(self.location)
|
|
1045
|
-
case StorageType.
|
|
1127
|
+
case StorageType.AWS_S3:
|
|
1046
1128
|
span.set_attribute("bucket", self.location)
|
|
1047
|
-
if isinstance(self,
|
|
1129
|
+
if isinstance(self, AWSS3StorageLocation) and self.region:
|
|
1048
1130
|
store = runner_context.s3_client(self.location, region=self.region)
|
|
1049
1131
|
else:
|
|
1050
1132
|
store = runner_context.s3_client(self.location)
|
|
1051
|
-
case StorageType.
|
|
1052
|
-
if not isinstance(self,
|
|
1133
|
+
case StorageType.AZURE_BLOB:
|
|
1134
|
+
if not isinstance(self, AzureBlobStorageLocation) or not self.storage_account_resource_id:
|
|
1053
1135
|
raise ValueError("Azure storage location is missing its storage account resource ID")
|
|
1054
1136
|
span.set_attribute("storage_account_resource_id", self.storage_account_resource_id)
|
|
1055
1137
|
span.set_attribute("container", self.location)
|
|
1056
1138
|
store = runner_context.azure_client(self.storage_account_resource_id, self.location)
|
|
1057
|
-
case StorageType.
|
|
1139
|
+
case StorageType.LOCAL:
|
|
1140
|
+
from obstore.store import LocalStore # noqa: PLC0415
|
|
1141
|
+
|
|
1058
1142
|
span.set_attribute("root_directory", self.location)
|
|
1059
|
-
|
|
1143
|
+
root = runner_context.local_path(self.location)
|
|
1144
|
+
if not self.location or not root.is_dir():
|
|
1145
|
+
raise ValueError(f"Local storage location root is not available: {self.location!r}")
|
|
1146
|
+
store = LocalStore(root)
|
|
1060
1147
|
|
|
1061
1148
|
import obstore # noqa: PLC0415
|
|
1062
1149
|
|
|
@@ -1064,28 +1151,41 @@ class StorageLocation:
|
|
|
1064
1151
|
|
|
1065
1152
|
|
|
1066
1153
|
@dataclass(frozen=True)
|
|
1067
|
-
class
|
|
1154
|
+
class AWSS3StorageLocation(StorageLocation):
|
|
1068
1155
|
region: str | None = field(default=None, kw_only=True)
|
|
1069
1156
|
|
|
1070
1157
|
def to_message(self) -> storage_pb.StorageLocation:
|
|
1071
1158
|
"""Include the S3 bucket and its region in the storage reference."""
|
|
1072
1159
|
message = super().to_message()
|
|
1073
|
-
message.reference.type = storage_pb.STORAGE_TYPE_AWS_S3_BUCKET
|
|
1074
|
-
message.reference.aws_s3_bucket.bucket = self.location
|
|
1075
1160
|
message.reference.aws_s3_bucket.region = self.region or ""
|
|
1076
1161
|
return message
|
|
1077
1162
|
|
|
1078
1163
|
|
|
1079
1164
|
@dataclass(frozen=True)
|
|
1080
|
-
class
|
|
1165
|
+
class GCSStorageLocation(StorageLocation):
|
|
1166
|
+
project_id: str = field(default="", kw_only=True)
|
|
1167
|
+
bucket_location: str = field(default="", kw_only=True)
|
|
1168
|
+
|
|
1169
|
+
def to_message(self) -> storage_pb.StorageLocation:
|
|
1170
|
+
message = super().to_message()
|
|
1171
|
+
message.reference.gcs_bucket.project_id = self.project_id
|
|
1172
|
+
message.reference.gcs_bucket.location = self.bucket_location
|
|
1173
|
+
return message
|
|
1174
|
+
|
|
1175
|
+
|
|
1176
|
+
@dataclass(frozen=True)
|
|
1177
|
+
class AzureBlobStorageLocation(StorageLocation):
|
|
1081
1178
|
storage_account_resource_id: str = field(kw_only=True)
|
|
1179
|
+
region: str = field(default="", kw_only=True)
|
|
1180
|
+
|
|
1181
|
+
def __str__(self) -> str:
|
|
1182
|
+
return f"{self.storage_account_resource_id}/containers/{self.location}"
|
|
1082
1183
|
|
|
1083
1184
|
def to_message(self) -> storage_pb.StorageLocation:
|
|
1084
1185
|
"""Include the Azure account and container in the storage reference."""
|
|
1085
1186
|
message = super().to_message()
|
|
1086
|
-
message.reference.type = storage_pb.STORAGE_TYPE_AZURE_BLOB
|
|
1087
1187
|
message.reference.azure_blob.storage_account_resource_id = self.storage_account_resource_id
|
|
1088
|
-
message.reference.azure_blob.
|
|
1188
|
+
message.reference.azure_blob.region = self.region
|
|
1089
1189
|
return message
|
|
1090
1190
|
|
|
1091
1191
|
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
"""Bounded OTLP log streaming for CLI-managed workflow runtimes."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
import threading
|
|
6
|
+
import time
|
|
7
|
+
import traceback
|
|
8
|
+
from collections import deque
|
|
9
|
+
from collections.abc import Callable, Iterator
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from opentelemetry.instrumentation.log_utils import std_to_otel
|
|
13
|
+
from opentelemetry.proto.common.v1.common_pb2 import AnyValue, ArrayValue, KeyValue, KeyValueList
|
|
14
|
+
from opentelemetry.proto.logs.v1.logs_pb2 import LogRecord, SeverityNumber
|
|
15
|
+
from opentelemetry.semconv.attributes import exception_attributes
|
|
16
|
+
|
|
17
|
+
from tilebox.workflows.observability._logging import _record_attributes
|
|
18
|
+
|
|
19
|
+
_MAX_RECORD_SIZE = 1024 * 1024
|
|
20
|
+
_MANAGED_RUNTIME = bool(os.environ.get("TILEBOX_RUNTIME_DIR") and os.environ.get("TILEBOX_RUNTIME_TOKEN"))
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _any_value(value: Any) -> AnyValue: # noqa: PLR0911 -- each protobuf union arm is clearest explicitly
|
|
24
|
+
"""Convert an already-normalized JSON value to OTLP."""
|
|
25
|
+
if value is None:
|
|
26
|
+
return AnyValue()
|
|
27
|
+
if isinstance(value, bool):
|
|
28
|
+
return AnyValue(bool_value=value)
|
|
29
|
+
if isinstance(value, int):
|
|
30
|
+
if abs(value) > 2**63 - 1:
|
|
31
|
+
return AnyValue(string_value=str(value))
|
|
32
|
+
return AnyValue(int_value=value)
|
|
33
|
+
if isinstance(value, float):
|
|
34
|
+
return AnyValue(double_value=value)
|
|
35
|
+
if isinstance(value, dict):
|
|
36
|
+
return AnyValue(
|
|
37
|
+
kvlist_value=KeyValueList(values=[KeyValue(key=key, value=_any_value(item)) for key, item in value.items()])
|
|
38
|
+
)
|
|
39
|
+
if isinstance(value, list):
|
|
40
|
+
return AnyValue(array_value=ArrayValue(values=[_any_value(item) for item in value]))
|
|
41
|
+
return AnyValue(string_value=value)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class _LogQueue:
|
|
45
|
+
"""Non-blocking bounded queue with explicit sealed-and-empty termination."""
|
|
46
|
+
|
|
47
|
+
def __init__(self, capacity: int = 256, max_record_size: int = _MAX_RECORD_SIZE) -> None:
|
|
48
|
+
self._capacity = capacity
|
|
49
|
+
self._max_record_size = max_record_size
|
|
50
|
+
self._records: deque[LogRecord] = deque()
|
|
51
|
+
self._condition = threading.Condition()
|
|
52
|
+
self._subscriber = False
|
|
53
|
+
self._sealed = False
|
|
54
|
+
self._dropped = 0
|
|
55
|
+
|
|
56
|
+
def submit(self, record: LogRecord) -> None:
|
|
57
|
+
with self._condition:
|
|
58
|
+
if self._sealed:
|
|
59
|
+
return
|
|
60
|
+
if record.ByteSize() > self._max_record_size or len(self._records) >= self._capacity:
|
|
61
|
+
self._dropped += 1
|
|
62
|
+
self._condition.notify_all()
|
|
63
|
+
return
|
|
64
|
+
if self._dropped:
|
|
65
|
+
self._records.append(_drop_record(self._dropped))
|
|
66
|
+
self._dropped = 0
|
|
67
|
+
if len(self._records) >= self._capacity:
|
|
68
|
+
self._dropped += 1
|
|
69
|
+
self._condition.notify_all()
|
|
70
|
+
return
|
|
71
|
+
self._records.append(record)
|
|
72
|
+
self._condition.notify_all()
|
|
73
|
+
|
|
74
|
+
def subscribe(self, active: Callable[[], bool] = lambda: True) -> Iterator[LogRecord]:
|
|
75
|
+
with self._condition:
|
|
76
|
+
if self._subscriber:
|
|
77
|
+
raise RuntimeError("A log subscriber is already connected")
|
|
78
|
+
self._subscriber = True
|
|
79
|
+
try:
|
|
80
|
+
while active():
|
|
81
|
+
with self._condition:
|
|
82
|
+
if self._records:
|
|
83
|
+
record = self._records.popleft()
|
|
84
|
+
self._condition.notify_all()
|
|
85
|
+
elif self._dropped:
|
|
86
|
+
record = _drop_record(self._dropped)
|
|
87
|
+
self._dropped = 0
|
|
88
|
+
self._condition.notify_all()
|
|
89
|
+
elif self._sealed:
|
|
90
|
+
return
|
|
91
|
+
else:
|
|
92
|
+
self._condition.wait(timeout=0.1)
|
|
93
|
+
continue
|
|
94
|
+
yield record
|
|
95
|
+
finally:
|
|
96
|
+
with self._condition:
|
|
97
|
+
self._subscriber = False
|
|
98
|
+
self._condition.notify_all()
|
|
99
|
+
|
|
100
|
+
def seal(self) -> None:
|
|
101
|
+
with self._condition:
|
|
102
|
+
self._sealed = True
|
|
103
|
+
self._condition.notify_all()
|
|
104
|
+
|
|
105
|
+
def wait_empty(self, timeout: float) -> bool:
|
|
106
|
+
deadline = time.monotonic() + timeout
|
|
107
|
+
with self._condition:
|
|
108
|
+
while self._records or self._dropped:
|
|
109
|
+
remaining = deadline - time.monotonic()
|
|
110
|
+
if remaining <= 0:
|
|
111
|
+
return False
|
|
112
|
+
self._condition.wait(timeout=remaining)
|
|
113
|
+
return True
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _drop_record(count: int) -> LogRecord:
|
|
117
|
+
return LogRecord(
|
|
118
|
+
time_unix_nano=time.time_ns(),
|
|
119
|
+
severity_number=_severity_number(logging.WARNING),
|
|
120
|
+
severity_text="WARN",
|
|
121
|
+
body=AnyValue(string_value=f"Dropped {count} local log records"),
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class _OTLPQueueHandler(logging.Handler):
|
|
126
|
+
def __init__(self, records: _LogQueue) -> None:
|
|
127
|
+
super().__init__(logging.NOTSET)
|
|
128
|
+
self._records = records
|
|
129
|
+
|
|
130
|
+
def emit(self, record: logging.LogRecord) -> None:
|
|
131
|
+
try:
|
|
132
|
+
attributes = dict(_record_attributes(record))
|
|
133
|
+
trace_id = _hex_id(attributes.pop("trace_id", ""), 16)
|
|
134
|
+
span_id = _hex_id(attributes.pop("span_id", ""), 8)
|
|
135
|
+
if record.exc_info:
|
|
136
|
+
exctype, value, tb = record.exc_info
|
|
137
|
+
if exctype is not None:
|
|
138
|
+
attributes[exception_attributes.EXCEPTION_TYPE] = exctype.__name__
|
|
139
|
+
if value is not None:
|
|
140
|
+
attributes[exception_attributes.EXCEPTION_MESSAGE] = str(value)
|
|
141
|
+
if tb is not None:
|
|
142
|
+
attributes[exception_attributes.EXCEPTION_STACKTRACE] = "".join(
|
|
143
|
+
traceback.format_exception(*record.exc_info)
|
|
144
|
+
)
|
|
145
|
+
self._records.submit(
|
|
146
|
+
LogRecord(
|
|
147
|
+
time_unix_nano=int(record.created * 1e9),
|
|
148
|
+
severity_number=_severity_number(record.levelno),
|
|
149
|
+
severity_text={"WARNING": "WARN", "CRITICAL": "FATAL"}.get(record.levelname, record.levelname),
|
|
150
|
+
body=_any_value(record.getMessage()),
|
|
151
|
+
attributes=[KeyValue(key=str(key), value=_any_value(value)) for key, value in attributes.items()],
|
|
152
|
+
trace_id=trace_id,
|
|
153
|
+
span_id=span_id,
|
|
154
|
+
)
|
|
155
|
+
)
|
|
156
|
+
except Exception: # noqa: BLE001 -- logging must never break workflow execution
|
|
157
|
+
self.handleError(record)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _hex_id(value: Any, size: int) -> bytes:
|
|
161
|
+
try:
|
|
162
|
+
result = bytes.fromhex(value) if isinstance(value, str) else b""
|
|
163
|
+
except ValueError:
|
|
164
|
+
return b""
|
|
165
|
+
return result if len(result) == size else b""
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _severity_number(level: int) -> SeverityNumber.ValueType:
|
|
169
|
+
return SeverityNumber.Value(f"SEVERITY_NUMBER_{std_to_otel(level).name}")
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
managed_log_queue = _LogQueue() if _MANAGED_RUNTIME else None
|