tilebox-workflows 0.63.2__tar.gz → 0.65.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/PKG-INFO +6 -6
  2. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/pyproject.toml +5 -5
  3. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/__init__.py +12 -0
  4. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/automations/client.py +0 -8
  5. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/automations/service.py +1 -16
  6. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/automations/storage_event.py +1 -1
  7. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/client.py +7 -1
  8. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/data.py +149 -49
  9. tilebox_workflows-0.65.0/tilebox/workflows/observability/_log_stream.py +172 -0
  10. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/observability/logging.py +76 -50
  11. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/observability/tracing.py +8 -13
  12. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/runner/__main__.py +10 -5
  13. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/runner/executor.py +1 -1
  14. tilebox_workflows-0.65.0/tilebox/workflows/runner/worker_server.py +128 -0
  15. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/runner/worker_service.py +73 -7
  16. tilebox_workflows-0.65.0/tilebox/workflows/storage_locations/__init__.py +35 -0
  17. tilebox_workflows-0.65.0/tilebox/workflows/storage_locations/client.py +38 -0
  18. tilebox_workflows-0.65.0/tilebox/workflows/storage_locations/data.py +60 -0
  19. tilebox_workflows-0.65.0/tilebox/workflows/storage_locations/service.py +46 -0
  20. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/automation_pb2.py +2 -2
  21. tilebox_workflows-0.65.0/tilebox/workflows/workflows/v1/storage_location_pb2.py +114 -0
  22. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/storage_location_pb2.pyi +17 -13
  23. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/worker_connect.py +66 -0
  24. tilebox_workflows-0.65.0/tilebox/workflows/workflows/v1/worker_pb2.py +51 -0
  25. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/worker_pb2.pyi +1 -0
  26. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/worker_pb2_grpc.py +57 -1
  27. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/workflows_connect.py +65 -0
  28. tilebox_workflows-0.65.0/tilebox/workflows/workflows/v1/workflows_pb2.py +115 -0
  29. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/workflows_pb2.pyi +23 -4
  30. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/workflows_pb2_grpc.py +43 -0
  31. tilebox_workflows-0.63.2/tilebox/workflows/observability/_log_pipe.py +0 -124
  32. tilebox_workflows-0.63.2/tilebox/workflows/runner/worker_server.py +0 -67
  33. tilebox_workflows-0.63.2/tilebox/workflows/workflows/v1/storage_location_pb2.py +0 -114
  34. tilebox_workflows-0.63.2/tilebox/workflows/workflows/v1/worker_pb2.py +0 -50
  35. tilebox_workflows-0.63.2/tilebox/workflows/workflows/v1/workflows_pb2.py +0 -102
  36. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/.gitignore +0 -0
  37. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/README.md +0 -0
  38. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/_codec.py +0 -0
  39. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/_codecs/__init__.py +0 -0
  40. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/_codecs/affine.py +0 -0
  41. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/_codecs/async_geotiff.py +0 -0
  42. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/_codecs/odc.py +0 -0
  43. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/_codecs/pyproj.py +0 -0
  44. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/_codecs/rasterio.py +0 -0
  45. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/_codecs/shapely.py +0 -0
  46. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/_codecs/tilebox.py +0 -0
  47. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/_serialization.py +0 -0
  48. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/automations/__init__.py +0 -0
  49. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/automations/cron.py +0 -0
  50. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/cache.py +0 -0
  51. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/clusters/__init__.py +0 -0
  52. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/clusters/client.py +0 -0
  53. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/clusters/service.py +0 -0
  54. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/formatting/__init__.py +0 -0
  55. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/formatting/job.py +0 -0
  56. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/jobs/__init__.py +0 -0
  57. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/jobs/client.py +0 -0
  58. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/jobs/service.py +0 -0
  59. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/jobs/telemetry_service.py +0 -0
  60. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/observability/__init__.py +0 -0
  61. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/observability/_logging.py +0 -0
  62. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/runner/__init__.py +0 -0
  63. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/runner/runner.py +0 -0
  64. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/runner/task_runner.py +0 -0
  65. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/runner/task_service.py +0 -0
  66. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/task.py +0 -0
  67. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/__init__.py +0 -0
  68. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/client.py +0 -0
  69. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/service.py +0 -0
  70. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/automation_connect.py +0 -0
  71. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/automation_pb2.pyi +0 -0
  72. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/automation_pb2_grpc.py +0 -0
  73. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/core_pb2.py +0 -0
  74. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/core_pb2.pyi +0 -0
  75. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/core_pb2_grpc.py +0 -0
  76. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/diagram_connect.py +0 -0
  77. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/diagram_pb2.py +0 -0
  78. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/diagram_pb2.pyi +0 -0
  79. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/diagram_pb2_grpc.py +0 -0
  80. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/job_connect.py +0 -0
  81. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/job_pb2.py +0 -0
  82. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/job_pb2.pyi +0 -0
  83. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/job_pb2_grpc.py +0 -0
  84. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/storage_location_connect.py +0 -0
  85. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/storage_location_pb2_grpc.py +0 -0
  86. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/task_connect.py +0 -0
  87. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/task_pb2.py +0 -0
  88. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/task_pb2.pyi +0 -0
  89. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/task_pb2_grpc.py +0 -0
  90. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/task_release_pb2.py +0 -0
  91. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/task_release_pb2.pyi +0 -0
  92. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/task_release_pb2_grpc.py +0 -0
  93. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/telemetry_connect.py +0 -0
  94. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/telemetry_pb2.py +0 -0
  95. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/telemetry_pb2.pyi +0 -0
  96. {tilebox_workflows-0.63.2 → tilebox_workflows-0.65.0}/tilebox/workflows/workflows/v1/telemetry_pb2_grpc.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: tilebox-workflows
3
- Version: 0.63.2
3
+ Version: 0.65.0
4
4
  Summary: Workflow client and task runner for Tilebox
5
5
  Project-URL: Homepage, https://tilebox.com
6
6
  Project-URL: Documentation, https://docs.tilebox.com/workflows/introduction
@@ -25,11 +25,11 @@ Requires-Dist: google-auth[requests]>=2.29
25
25
  Requires-Dist: grpcio>=1.84.0
26
26
  Requires-Dist: msgspec>=0.19
27
27
  Requires-Dist: obstore>=0.8.2
28
- Requires-Dist: opentelemetry-api>=1.43.0
29
- Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.43.0
30
- Requires-Dist: opentelemetry-instrumentation-logging>=0.64b0
31
- Requires-Dist: opentelemetry-proto>=1.43.0
32
- Requires-Dist: opentelemetry-sdk>=1.43.0
28
+ Requires-Dist: opentelemetry-api>=1.44.0
29
+ Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.44.0
30
+ Requires-Dist: opentelemetry-instrumentation-logging>=0.65b0
31
+ Requires-Dist: opentelemetry-proto>=1.44.0
32
+ Requires-Dist: opentelemetry-sdk>=1.44.0
33
33
  Requires-Dist: tenacity>=8
34
34
  Requires-Dist: tilebox-datasets>=0.56.0
35
35
  Requires-Dist: tilebox-grpc>=0.28.0
@@ -27,14 +27,14 @@ dependencies = [
27
27
  "grpcio>=1.84.0",
28
28
  "google-auth[requests]>=2.29",
29
29
  "azure-identity>=1.23",
30
- "opentelemetry-api>=1.43.0",
31
- "opentelemetry-exporter-otlp-proto-http>=1.43.0",
32
- "opentelemetry-sdk>=1.43.0",
30
+ "opentelemetry-api>=1.44.0",
31
+ "opentelemetry-exporter-otlp-proto-http>=1.44.0",
32
+ "opentelemetry-sdk>=1.44.0",
33
33
  "tenacity>=8",
34
34
  "boto3>=1.40.2",
35
35
  "obstore>=0.8.2",
36
- "opentelemetry-proto>=1.43.0",
37
- "opentelemetry-instrumentation-logging>=0.64b0",
36
+ "opentelemetry-proto>=1.44.0",
37
+ "opentelemetry-instrumentation-logging>=0.65b0",
38
38
  "msgspec>=0.19",
39
39
  ]
40
40
 
@@ -1,6 +1,9 @@
1
1
  import os
2
2
  from typing import TYPE_CHECKING, Any
3
3
 
4
+ # Apply native logging defaults before any submodule can initialize gRPC.
5
+ import _tilebox.grpc # noqa: F401
6
+
4
7
  if TYPE_CHECKING:
5
8
  from tilebox.workflows.client import Client
6
9
  from tilebox.workflows.data import Job
@@ -48,8 +51,17 @@ def __dir__() -> list[str]:
48
51
  def _initialize_logging_from_environment() -> None:
49
52
  # Stage 1: current CLIs provide credentials before importing workflow code, so
50
53
  # even import-time logs reach the API. Ordinary SDK imports stay lightweight.
54
+ runtime_dir = os.environ.get("TILEBOX_RUNTIME_DIR")
55
+ runtime_token = os.environ.get("TILEBOX_RUNTIME_TOKEN")
56
+ if bool(runtime_dir) != bool(runtime_token):
57
+ raise RuntimeError("TILEBOX_RUNTIME_DIR and TILEBOX_RUNTIME_TOKEN must be set together")
58
+
51
59
  url = os.environ.get("TILEBOX_API_URL")
52
60
  token = os.environ.get("TILEBOX_API_KEY")
61
+ if runtime_dir and runtime_token:
62
+ # Importing installs the managed local stream before workflow modules can log.
63
+ from tilebox.workflows.observability import logging as _runtime_logging # noqa: F401, PLC0415
64
+
53
65
  if not url or not token:
54
66
  return
55
67
 
@@ -18,14 +18,6 @@ class AutomationClient:
18
18
  def __init__(self, service: AutomationService) -> None:
19
19
  self._service = service
20
20
 
21
- def storage_locations(self) -> list[StorageLocation]:
22
- """List all available storage locations that can potentially be used as storage event triggers.
23
-
24
- Returns:
25
- A list of all available storage locations.
26
- """
27
- return self._service.list_storage_locations()
28
-
29
21
  def all(self) -> list[AutomationPrototype]:
30
22
  """List all registered automations.
31
23
 
@@ -9,39 +9,24 @@ from _tilebox.grpc.error import with_pythonic_errors
9
9
  from tilebox.datasets.uuid import uuid_to_uuid_message
10
10
  from tilebox.workflows.data import (
11
11
  AutomationPrototype,
12
- StorageLocation,
13
12
  )
14
13
  from tilebox.workflows.workflows.v1.automation_pb2 import AutomationPrototype as AutomationPrototypeMessage
15
14
  from tilebox.workflows.workflows.v1.automation_pb2 import Automations, DeleteAutomationRequest
16
15
  from tilebox.workflows.workflows.v1.automation_pb2_grpc import AutomationServiceStub
17
- from tilebox.workflows.workflows.v1.storage_location_pb2 import StorageLocations
18
- from tilebox.workflows.workflows.v1.storage_location_pb2_grpc import StorageLocationServiceStub
19
16
 
20
17
 
21
18
  class AutomationService:
22
- def __init__(self, channel: Channel | Any, storage_channel: Channel | Any = None) -> None:
19
+ def __init__(self, channel: Channel | Any) -> None:
23
20
  """
24
21
  A wrapper around the AutomationServiceStub that provides a more pythonic interface and converts the protobuf
25
22
  messages to and from the data classes used in the rest of the tilebox-workflows codebase.
26
23
 
27
24
  Args:
28
25
  channel: The gRPC channel to use for the service.
29
- storage_channel: Optional separate storage-location client for the Connect transport.
30
26
  """
31
27
  self.service = (
32
28
  with_pythonic_errors(AutomationServiceStub(channel)) if hasattr(channel, "unary_unary") else channel
33
29
  )
34
- if storage_channel is None:
35
- storage_channel = channel
36
- self.storage_service = (
37
- with_pythonic_errors(StorageLocationServiceStub(storage_channel))
38
- if hasattr(storage_channel, "unary_unary")
39
- else storage_channel
40
- )
41
-
42
- def list_storage_locations(self) -> builtins.list[StorageLocation]:
43
- response: StorageLocations = self.storage_service.ListStorageLocations(Empty())
44
- return [StorageLocation.from_message(sl) for sl in response.locations]
45
30
 
46
31
  def list(self) -> builtins.list[AutomationPrototype]:
47
32
  response: Automations = self.service.ListAutomations(Empty())
@@ -13,7 +13,7 @@ from tilebox.workflows.workflows.v1.automation_pb2 import Automation as Automati
13
13
  from tilebox.workflows.workflows.v1.storage_location_pb2 import TriggeredStorageEvent as TriggeredStorageEventMessage
14
14
 
15
15
  _NOT_TRIGGERED = TriggeredStorageEvent(
16
- StorageLocation(UUID(int=0), "", StorageType.FS),
16
+ StorageLocation(UUID(int=0), "", StorageType.LOCAL),
17
17
  StorageEventType.CREATED,
18
18
  "",
19
19
  )
@@ -23,6 +23,8 @@ from tilebox.workflows.observability.tracing import WorkflowTracer
23
23
  from tilebox.workflows.runner.executor import LazyStorageLocations
24
24
  from tilebox.workflows.runner.task_runner import TaskRunner, _LeaseRenewer
25
25
  from tilebox.workflows.runner.task_service import TaskService
26
+ from tilebox.workflows.storage_locations.client import StorageLocationClient
27
+ from tilebox.workflows.storage_locations.service import StorageLocationService
26
28
  from tilebox.workflows.task import Task
27
29
  from tilebox.workflows.workflows.client import WorkflowClient
28
30
  from tilebox.workflows.workflows.service import WorkflowService
@@ -207,7 +209,11 @@ class Client:
207
209
  Returns:
208
210
  A client for the automations service.
209
211
  """
210
- return AutomationClient(AutomationService(self._automation_service, self._storage_location_service))
212
+ return AutomationClient(AutomationService(self._automation_service))
213
+
214
+ def storage_locations(self) -> StorageLocationClient:
215
+ """Get a client for storage locations."""
216
+ return StorageLocationClient(StorageLocationService(self._storage_location_service))
211
217
 
212
218
 
213
219
  def _token_from_env(url: str, token: str | None) -> str | None:
@@ -969,13 +969,67 @@ class QueryJobSpansResponse:
969
969
 
970
970
 
971
971
  class StorageType(Enum):
972
- GCS = storage_pb.STORAGE_TYPE_GCS_BUCKET # Google Cloud Storage
973
- S3 = storage_pb.STORAGE_TYPE_AWS_S3_BUCKET # Amazon Web Services S3
974
- FS = storage_pb.STORAGE_TYPE_FILESYSTEM # Local Filesystem
975
- AZURE = storage_pb.STORAGE_TYPE_AZURE_BLOB # Azure Blob Storage
972
+ GCS = storage_pb.STORAGE_TYPE_GCS # Google Cloud Storage
973
+ AWS_S3 = storage_pb.STORAGE_TYPE_AWS_S3 # Amazon Web Services S3
974
+ LOCAL = storage_pb.STORAGE_TYPE_LOCAL # Local Filesystem
975
+ AZURE_BLOB = storage_pb.STORAGE_TYPE_AZURE_BLOB # Azure Blob Storage
976
976
 
977
977
 
978
- _STORAGE_TYPE_TO_ENUM = {storage_type.value: storage_type for storage_type in StorageType}
978
+ @dataclass(frozen=True)
979
+ class AWSS3BucketReference:
980
+ bucket: str
981
+ region: str = ""
982
+
983
+ def to_message(self) -> storage_pb.StorageLocationReference:
984
+ return storage_pb.StorageLocationReference(
985
+ type=storage_pb.STORAGE_TYPE_AWS_S3,
986
+ aws_s3_bucket=storage_pb.AWSS3BucketReference(bucket=self.bucket, region=self.region),
987
+ )
988
+
989
+
990
+ @dataclass(frozen=True)
991
+ class GCSBucketReference:
992
+ bucket: str
993
+ project_id: str = ""
994
+ location: str = ""
995
+
996
+ def to_message(self) -> storage_pb.StorageLocationReference:
997
+ return storage_pb.StorageLocationReference(
998
+ type=storage_pb.STORAGE_TYPE_GCS,
999
+ gcs_bucket=storage_pb.GCSBucketReference(
1000
+ bucket=self.bucket, project_id=self.project_id, location=self.location
1001
+ ),
1002
+ )
1003
+
1004
+
1005
+ @dataclass(frozen=True)
1006
+ class AzureBlobReference:
1007
+ storage_account_resource_id: str
1008
+ container: str
1009
+ region: str = ""
1010
+
1011
+ def to_message(self) -> storage_pb.StorageLocationReference:
1012
+ return storage_pb.StorageLocationReference(
1013
+ type=storage_pb.STORAGE_TYPE_AZURE_BLOB,
1014
+ azure_blob=storage_pb.AzureBlobReference(
1015
+ storage_account_resource_id=self.storage_account_resource_id,
1016
+ container=self.container,
1017
+ region=self.region,
1018
+ ),
1019
+ )
1020
+
1021
+
1022
+ @dataclass(frozen=True)
1023
+ class LocalReference:
1024
+ path: str
1025
+
1026
+ def to_message(self) -> storage_pb.StorageLocationReference:
1027
+ return storage_pb.StorageLocationReference(
1028
+ type=storage_pb.STORAGE_TYPE_LOCAL, local=storage_pb.LocalReference(path=self.path)
1029
+ )
1030
+
1031
+
1032
+ StorageLocationReference: TypeAlias = AWSS3BucketReference | GCSBucketReference | AzureBlobReference | LocalReference
979
1033
 
980
1034
 
981
1035
  @total_ordering
@@ -985,53 +1039,81 @@ class StorageLocation:
985
1039
  location: str
986
1040
  type: StorageType
987
1041
  runner_context: "RunnerContext | None" = None
1042
+ name: str = field(default="", kw_only=True)
988
1043
 
989
1044
  def __lt__(self, other: object) -> bool:
990
1045
  if not isinstance(other, StorageLocation):
991
1046
  return NotImplemented
992
1047
  return (self.id, self.location, self.type.value) < (other.id, other.location, other.type.value)
993
1048
 
1049
+ def __str__(self) -> str:
1050
+ match self.type:
1051
+ case StorageType.AWS_S3:
1052
+ return f"s3://{self.location}"
1053
+ case StorageType.GCS:
1054
+ return f"gs://{self.location}"
1055
+ case StorageType.LOCAL:
1056
+ return self.location
1057
+ case _:
1058
+ return self.type.name.lower().replace("_", "-")
1059
+
994
1060
  @classmethod
995
1061
  def from_message(cls, storage_location: storage_pb.StorageLocation) -> "StorageLocation":
996
1062
  """Convert a StorageLocation protobuf message to a StorageLocation object."""
997
- location = storage_location.location
998
- storage_type = storage_location.type
999
- if storage_location.HasField("reference"):
1000
- reference = storage_location.reference
1001
- storage_type = reference.type
1002
- match storage_type:
1003
- case storage_pb.STORAGE_TYPE_AZURE_BLOB:
1004
- return AzureStorageLocation(
1005
- id=uuid_message_to_uuid(storage_location.id),
1006
- location=reference.azure_blob.container,
1007
- type=StorageType.AZURE,
1008
- storage_account_resource_id=reference.azure_blob.storage_account_resource_id,
1009
- )
1010
- case storage_pb.STORAGE_TYPE_GCS_BUCKET:
1011
- location = reference.gcs_bucket.bucket
1012
- case storage_pb.STORAGE_TYPE_AWS_S3_BUCKET:
1013
- return S3StorageLocation(
1014
- id=uuid_message_to_uuid(storage_location.id),
1015
- location=reference.aws_s3_bucket.bucket,
1016
- type=StorageType.S3,
1017
- region=reference.aws_s3_bucket.region or None,
1018
- )
1019
- case storage_pb.STORAGE_TYPE_FILESYSTEM:
1020
- location = reference.filesystem.path
1021
- return cls(
1022
- id=uuid_message_to_uuid(storage_location.id),
1023
- location=location,
1024
- type=_STORAGE_TYPE_TO_ENUM[storage_type],
1025
- )
1063
+ reference = storage_location.reference
1064
+ match reference.type:
1065
+ case storage_pb.STORAGE_TYPE_AZURE_BLOB:
1066
+ return AzureBlobStorageLocation(
1067
+ id=uuid_message_to_uuid(storage_location.id),
1068
+ name=storage_location.name,
1069
+ location=reference.azure_blob.container,
1070
+ type=StorageType.AZURE_BLOB,
1071
+ storage_account_resource_id=reference.azure_blob.storage_account_resource_id,
1072
+ region=reference.azure_blob.region,
1073
+ )
1074
+ case storage_pb.STORAGE_TYPE_GCS:
1075
+ return GCSStorageLocation(
1076
+ id=uuid_message_to_uuid(storage_location.id),
1077
+ name=storage_location.name,
1078
+ location=reference.gcs_bucket.bucket,
1079
+ type=StorageType.GCS,
1080
+ project_id=reference.gcs_bucket.project_id,
1081
+ bucket_location=reference.gcs_bucket.location,
1082
+ )
1083
+ case storage_pb.STORAGE_TYPE_AWS_S3:
1084
+ return AWSS3StorageLocation(
1085
+ id=uuid_message_to_uuid(storage_location.id),
1086
+ name=storage_location.name,
1087
+ location=reference.aws_s3_bucket.bucket,
1088
+ type=StorageType.AWS_S3,
1089
+ region=reference.aws_s3_bucket.region or None,
1090
+ )
1091
+ case storage_pb.STORAGE_TYPE_LOCAL:
1092
+ return cls(
1093
+ id=uuid_message_to_uuid(storage_location.id),
1094
+ name=storage_location.name,
1095
+ location=reference.local.path,
1096
+ type=StorageType.LOCAL,
1097
+ )
1098
+ case _:
1099
+ raise ValueError(f"Unsupported storage reference type: {reference.type}")
1026
1100
 
1027
1101
  def _with_runner_context(self, runner_context: "RunnerContext") -> "StorageLocation":
1028
1102
  return replace(self, runner_context=runner_context)
1029
1103
 
1030
1104
  def to_message(self) -> storage_pb.StorageLocation:
1031
1105
  """Convert a StorageLocation object to a StorageLocation protobuf message."""
1032
- return storage_pb.StorageLocation(
1033
- id=uuid_to_uuid_message(self.id), location=self.location, type=self.type.value
1034
- )
1106
+ reference = storage_pb.StorageLocationReference(type=self.type.value)
1107
+ match self.type:
1108
+ case StorageType.AWS_S3:
1109
+ reference.aws_s3_bucket.bucket = self.location
1110
+ case StorageType.GCS:
1111
+ reference.gcs_bucket.bucket = self.location
1112
+ case StorageType.AZURE_BLOB:
1113
+ reference.azure_blob.container = self.location
1114
+ case StorageType.LOCAL:
1115
+ reference.local.path = self.location
1116
+ return storage_pb.StorageLocation(id=uuid_to_uuid_message(self.id), name=self.name, reference=reference)
1035
1117
 
1036
1118
  def read(self, path: str) -> bytes:
1037
1119
  runner_context = self.runner_context or RunnerContext()
@@ -1042,21 +1124,26 @@ class StorageLocation:
1042
1124
  case StorageType.GCS:
1043
1125
  span.set_attribute("bucket", self.location)
1044
1126
  store = runner_context.gcs_client(self.location)
1045
- case StorageType.S3:
1127
+ case StorageType.AWS_S3:
1046
1128
  span.set_attribute("bucket", self.location)
1047
- if isinstance(self, S3StorageLocation) and self.region:
1129
+ if isinstance(self, AWSS3StorageLocation) and self.region:
1048
1130
  store = runner_context.s3_client(self.location, region=self.region)
1049
1131
  else:
1050
1132
  store = runner_context.s3_client(self.location)
1051
- case StorageType.AZURE:
1052
- if not isinstance(self, AzureStorageLocation) or not self.storage_account_resource_id:
1133
+ case StorageType.AZURE_BLOB:
1134
+ if not isinstance(self, AzureBlobStorageLocation) or not self.storage_account_resource_id:
1053
1135
  raise ValueError("Azure storage location is missing its storage account resource ID")
1054
1136
  span.set_attribute("storage_account_resource_id", self.storage_account_resource_id)
1055
1137
  span.set_attribute("container", self.location)
1056
1138
  store = runner_context.azure_client(self.storage_account_resource_id, self.location)
1057
- case StorageType.FS:
1139
+ case StorageType.LOCAL:
1140
+ from obstore.store import LocalStore # noqa: PLC0415
1141
+
1058
1142
  span.set_attribute("root_directory", self.location)
1059
- return Path(self.location).joinpath(path).read_bytes()
1143
+ root = runner_context.local_path(self.location)
1144
+ if not self.location or not root.is_dir():
1145
+ raise ValueError(f"Local storage location root is not available: {self.location!r}")
1146
+ store = LocalStore(root)
1060
1147
 
1061
1148
  import obstore # noqa: PLC0415
1062
1149
 
@@ -1064,28 +1151,41 @@ class StorageLocation:
1064
1151
 
1065
1152
 
1066
1153
  @dataclass(frozen=True)
1067
- class S3StorageLocation(StorageLocation):
1154
+ class AWSS3StorageLocation(StorageLocation):
1068
1155
  region: str | None = field(default=None, kw_only=True)
1069
1156
 
1070
1157
  def to_message(self) -> storage_pb.StorageLocation:
1071
1158
  """Include the S3 bucket and its region in the storage reference."""
1072
1159
  message = super().to_message()
1073
- message.reference.type = storage_pb.STORAGE_TYPE_AWS_S3_BUCKET
1074
- message.reference.aws_s3_bucket.bucket = self.location
1075
1160
  message.reference.aws_s3_bucket.region = self.region or ""
1076
1161
  return message
1077
1162
 
1078
1163
 
1079
1164
  @dataclass(frozen=True)
1080
- class AzureStorageLocation(StorageLocation):
1165
+ class GCSStorageLocation(StorageLocation):
1166
+ project_id: str = field(default="", kw_only=True)
1167
+ bucket_location: str = field(default="", kw_only=True)
1168
+
1169
+ def to_message(self) -> storage_pb.StorageLocation:
1170
+ message = super().to_message()
1171
+ message.reference.gcs_bucket.project_id = self.project_id
1172
+ message.reference.gcs_bucket.location = self.bucket_location
1173
+ return message
1174
+
1175
+
1176
+ @dataclass(frozen=True)
1177
+ class AzureBlobStorageLocation(StorageLocation):
1081
1178
  storage_account_resource_id: str = field(kw_only=True)
1179
+ region: str = field(default="", kw_only=True)
1180
+
1181
+ def __str__(self) -> str:
1182
+ return f"{self.storage_account_resource_id}/containers/{self.location}"
1082
1183
 
1083
1184
  def to_message(self) -> storage_pb.StorageLocation:
1084
1185
  """Include the Azure account and container in the storage reference."""
1085
1186
  message = super().to_message()
1086
- message.reference.type = storage_pb.STORAGE_TYPE_AZURE_BLOB
1087
1187
  message.reference.azure_blob.storage_account_resource_id = self.storage_account_resource_id
1088
- message.reference.azure_blob.container = self.location
1188
+ message.reference.azure_blob.region = self.region
1089
1189
  return message
1090
1190
 
1091
1191
 
@@ -0,0 +1,172 @@
1
+ """Bounded OTLP log streaming for CLI-managed workflow runtimes."""
2
+
3
+ import logging
4
+ import os
5
+ import threading
6
+ import time
7
+ import traceback
8
+ from collections import deque
9
+ from collections.abc import Callable, Iterator
10
+ from typing import Any
11
+
12
+ from opentelemetry.instrumentation.log_utils import std_to_otel
13
+ from opentelemetry.proto.common.v1.common_pb2 import AnyValue, ArrayValue, KeyValue, KeyValueList
14
+ from opentelemetry.proto.logs.v1.logs_pb2 import LogRecord, SeverityNumber
15
+ from opentelemetry.semconv.attributes import exception_attributes
16
+
17
+ from tilebox.workflows.observability._logging import _record_attributes
18
+
19
+ _MAX_RECORD_SIZE = 1024 * 1024
20
+ _MANAGED_RUNTIME = bool(os.environ.get("TILEBOX_RUNTIME_DIR") and os.environ.get("TILEBOX_RUNTIME_TOKEN"))
21
+
22
+
23
+ def _any_value(value: Any) -> AnyValue: # noqa: PLR0911 -- each protobuf union arm is clearest explicitly
24
+ """Convert an already-normalized JSON value to OTLP."""
25
+ if value is None:
26
+ return AnyValue()
27
+ if isinstance(value, bool):
28
+ return AnyValue(bool_value=value)
29
+ if isinstance(value, int):
30
+ if abs(value) > 2**63 - 1:
31
+ return AnyValue(string_value=str(value))
32
+ return AnyValue(int_value=value)
33
+ if isinstance(value, float):
34
+ return AnyValue(double_value=value)
35
+ if isinstance(value, dict):
36
+ return AnyValue(
37
+ kvlist_value=KeyValueList(values=[KeyValue(key=key, value=_any_value(item)) for key, item in value.items()])
38
+ )
39
+ if isinstance(value, list):
40
+ return AnyValue(array_value=ArrayValue(values=[_any_value(item) for item in value]))
41
+ return AnyValue(string_value=value)
42
+
43
+
44
+ class _LogQueue:
45
+ """Non-blocking bounded queue with explicit sealed-and-empty termination."""
46
+
47
+ def __init__(self, capacity: int = 256, max_record_size: int = _MAX_RECORD_SIZE) -> None:
48
+ self._capacity = capacity
49
+ self._max_record_size = max_record_size
50
+ self._records: deque[LogRecord] = deque()
51
+ self._condition = threading.Condition()
52
+ self._subscriber = False
53
+ self._sealed = False
54
+ self._dropped = 0
55
+
56
+ def submit(self, record: LogRecord) -> None:
57
+ with self._condition:
58
+ if self._sealed:
59
+ return
60
+ if record.ByteSize() > self._max_record_size or len(self._records) >= self._capacity:
61
+ self._dropped += 1
62
+ self._condition.notify_all()
63
+ return
64
+ if self._dropped:
65
+ self._records.append(_drop_record(self._dropped))
66
+ self._dropped = 0
67
+ if len(self._records) >= self._capacity:
68
+ self._dropped += 1
69
+ self._condition.notify_all()
70
+ return
71
+ self._records.append(record)
72
+ self._condition.notify_all()
73
+
74
+ def subscribe(self, active: Callable[[], bool] = lambda: True) -> Iterator[LogRecord]:
75
+ with self._condition:
76
+ if self._subscriber:
77
+ raise RuntimeError("A log subscriber is already connected")
78
+ self._subscriber = True
79
+ try:
80
+ while active():
81
+ with self._condition:
82
+ if self._records:
83
+ record = self._records.popleft()
84
+ self._condition.notify_all()
85
+ elif self._dropped:
86
+ record = _drop_record(self._dropped)
87
+ self._dropped = 0
88
+ self._condition.notify_all()
89
+ elif self._sealed:
90
+ return
91
+ else:
92
+ self._condition.wait(timeout=0.1)
93
+ continue
94
+ yield record
95
+ finally:
96
+ with self._condition:
97
+ self._subscriber = False
98
+ self._condition.notify_all()
99
+
100
+ def seal(self) -> None:
101
+ with self._condition:
102
+ self._sealed = True
103
+ self._condition.notify_all()
104
+
105
+ def wait_empty(self, timeout: float) -> bool:
106
+ deadline = time.monotonic() + timeout
107
+ with self._condition:
108
+ while self._records or self._dropped:
109
+ remaining = deadline - time.monotonic()
110
+ if remaining <= 0:
111
+ return False
112
+ self._condition.wait(timeout=remaining)
113
+ return True
114
+
115
+
116
+ def _drop_record(count: int) -> LogRecord:
117
+ return LogRecord(
118
+ time_unix_nano=time.time_ns(),
119
+ severity_number=_severity_number(logging.WARNING),
120
+ severity_text="WARN",
121
+ body=AnyValue(string_value=f"Dropped {count} local log records"),
122
+ )
123
+
124
+
125
+ class _OTLPQueueHandler(logging.Handler):
126
+ def __init__(self, records: _LogQueue) -> None:
127
+ super().__init__(logging.NOTSET)
128
+ self._records = records
129
+
130
+ def emit(self, record: logging.LogRecord) -> None:
131
+ try:
132
+ attributes = dict(_record_attributes(record))
133
+ trace_id = _hex_id(attributes.pop("trace_id", ""), 16)
134
+ span_id = _hex_id(attributes.pop("span_id", ""), 8)
135
+ if record.exc_info:
136
+ exctype, value, tb = record.exc_info
137
+ if exctype is not None:
138
+ attributes[exception_attributes.EXCEPTION_TYPE] = exctype.__name__
139
+ if value is not None:
140
+ attributes[exception_attributes.EXCEPTION_MESSAGE] = str(value)
141
+ if tb is not None:
142
+ attributes[exception_attributes.EXCEPTION_STACKTRACE] = "".join(
143
+ traceback.format_exception(*record.exc_info)
144
+ )
145
+ self._records.submit(
146
+ LogRecord(
147
+ time_unix_nano=int(record.created * 1e9),
148
+ severity_number=_severity_number(record.levelno),
149
+ severity_text={"WARNING": "WARN", "CRITICAL": "FATAL"}.get(record.levelname, record.levelname),
150
+ body=_any_value(record.getMessage()),
151
+ attributes=[KeyValue(key=str(key), value=_any_value(value)) for key, value in attributes.items()],
152
+ trace_id=trace_id,
153
+ span_id=span_id,
154
+ )
155
+ )
156
+ except Exception: # noqa: BLE001 -- logging must never break workflow execution
157
+ self.handleError(record)
158
+
159
+
160
+ def _hex_id(value: Any, size: int) -> bytes:
161
+ try:
162
+ result = bytes.fromhex(value) if isinstance(value, str) else b""
163
+ except ValueError:
164
+ return b""
165
+ return result if len(result) == size else b""
166
+
167
+
168
+ def _severity_number(level: int) -> SeverityNumber.ValueType:
169
+ return SeverityNumber.Value(f"SEVERITY_NUMBER_{std_to_otel(level).name}")
170
+
171
+
172
+ managed_log_queue = _LogQueue() if _MANAGED_RUNTIME else None