dagster-cloud 1.8.2__py3-none-any.whl → 1.12.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. dagster_cloud/__init__.py +3 -3
  2. dagster_cloud/agent/__init__.py +4 -4
  3. dagster_cloud/agent/cli/__init__.py +56 -17
  4. dagster_cloud/agent/dagster_cloud_agent.py +360 -172
  5. dagster_cloud/agent/instrumentation/__init__.py +0 -0
  6. dagster_cloud/agent/instrumentation/constants.py +2 -0
  7. dagster_cloud/agent/instrumentation/run_launch.py +23 -0
  8. dagster_cloud/agent/instrumentation/schedule.py +34 -0
  9. dagster_cloud/agent/instrumentation/sensor.py +34 -0
  10. dagster_cloud/anomaly_detection/__init__.py +2 -2
  11. dagster_cloud/anomaly_detection/defs.py +17 -12
  12. dagster_cloud/anomaly_detection/types.py +3 -3
  13. dagster_cloud/api/dagster_cloud_api.py +209 -293
  14. dagster_cloud/auth/constants.py +21 -5
  15. dagster_cloud/batching/__init__.py +1 -0
  16. dagster_cloud/batching/batcher.py +210 -0
  17. dagster_cloud/dagster_insights/__init__.py +12 -6
  18. dagster_cloud/dagster_insights/bigquery/bigquery_utils.py +3 -2
  19. dagster_cloud/dagster_insights/bigquery/dbt_wrapper.py +39 -12
  20. dagster_cloud/dagster_insights/bigquery/insights_bigquery_resource.py +8 -6
  21. dagster_cloud/dagster_insights/insights_utils.py +18 -8
  22. dagster_cloud/dagster_insights/metrics_utils.py +12 -12
  23. dagster_cloud/dagster_insights/snowflake/dagster_snowflake_insights.py +5 -12
  24. dagster_cloud/dagster_insights/snowflake/dbt_wrapper.py +34 -8
  25. dagster_cloud/dagster_insights/snowflake/definitions.py +38 -12
  26. dagster_cloud/dagster_insights/snowflake/insights_snowflake_resource.py +11 -23
  27. dagster_cloud/definitions/__init__.py +0 -0
  28. dagster_cloud/definitions/job_selection.py +36 -0
  29. dagster_cloud/execution/cloud_run_launcher/k8s.py +1 -1
  30. dagster_cloud/execution/cloud_run_launcher/process.py +3 -3
  31. dagster_cloud/execution/monitoring/__init__.py +27 -33
  32. dagster_cloud/execution/utils/process.py +3 -3
  33. dagster_cloud/instance/__init__.py +125 -38
  34. dagster_cloud/instrumentation/__init__.py +32 -0
  35. dagster_cloud/metadata/source_code.py +13 -8
  36. dagster_cloud/metrics/__init__.py +0 -0
  37. dagster_cloud/metrics/tracer.py +59 -0
  38. dagster_cloud/opentelemetry/__init__.py +0 -0
  39. dagster_cloud/opentelemetry/config/__init__.py +73 -0
  40. dagster_cloud/opentelemetry/config/exporter.py +81 -0
  41. dagster_cloud/opentelemetry/config/log_record_processor.py +40 -0
  42. dagster_cloud/opentelemetry/config/logging_handler.py +14 -0
  43. dagster_cloud/opentelemetry/config/meter_provider.py +9 -0
  44. dagster_cloud/opentelemetry/config/metric_reader.py +39 -0
  45. dagster_cloud/opentelemetry/controller.py +319 -0
  46. dagster_cloud/opentelemetry/enum.py +58 -0
  47. dagster_cloud/opentelemetry/factories/__init__.py +1 -0
  48. dagster_cloud/opentelemetry/factories/logs.py +113 -0
  49. dagster_cloud/opentelemetry/factories/metrics.py +121 -0
  50. dagster_cloud/opentelemetry/metrics/__init__.py +0 -0
  51. dagster_cloud/opentelemetry/metrics/meter.py +140 -0
  52. dagster_cloud/opentelemetry/observers/__init__.py +0 -0
  53. dagster_cloud/opentelemetry/observers/dagster_exception_handler.py +40 -0
  54. dagster_cloud/opentelemetry/observers/execution_observer.py +178 -0
  55. dagster_cloud/pex/grpc/__generated__/multi_pex_api_pb2.pyi +175 -0
  56. dagster_cloud/pex/grpc/__init__.py +2 -2
  57. dagster_cloud/pex/grpc/client.py +4 -4
  58. dagster_cloud/pex/grpc/compile.py +2 -2
  59. dagster_cloud/pex/grpc/server/__init__.py +2 -2
  60. dagster_cloud/pex/grpc/server/cli/__init__.py +31 -19
  61. dagster_cloud/pex/grpc/server/manager.py +60 -42
  62. dagster_cloud/pex/grpc/server/registry.py +28 -21
  63. dagster_cloud/pex/grpc/server/server.py +23 -14
  64. dagster_cloud/pex/grpc/types.py +5 -5
  65. dagster_cloud/py.typed +0 -0
  66. dagster_cloud/secrets/__init__.py +1 -1
  67. dagster_cloud/secrets/loader.py +3 -3
  68. dagster_cloud/serverless/__init__.py +1 -1
  69. dagster_cloud/serverless/io_manager.py +36 -53
  70. dagster_cloud/storage/client.py +54 -17
  71. dagster_cloud/storage/compute_logs/__init__.py +3 -1
  72. dagster_cloud/storage/compute_logs/compute_log_manager.py +22 -17
  73. dagster_cloud/storage/defs_state/__init__.py +3 -0
  74. dagster_cloud/storage/defs_state/queries.py +15 -0
  75. dagster_cloud/storage/defs_state/storage.py +113 -0
  76. dagster_cloud/storage/event_logs/__init__.py +3 -1
  77. dagster_cloud/storage/event_logs/queries.py +102 -4
  78. dagster_cloud/storage/event_logs/storage.py +266 -73
  79. dagster_cloud/storage/event_logs/utils.py +88 -7
  80. dagster_cloud/storage/runs/__init__.py +1 -1
  81. dagster_cloud/storage/runs/queries.py +17 -2
  82. dagster_cloud/storage/runs/storage.py +88 -42
  83. dagster_cloud/storage/schedules/__init__.py +1 -1
  84. dagster_cloud/storage/schedules/storage.py +6 -8
  85. dagster_cloud/storage/tags.py +66 -1
  86. dagster_cloud/util/__init__.py +10 -12
  87. dagster_cloud/util/errors.py +49 -64
  88. dagster_cloud/version.py +1 -1
  89. dagster_cloud/workspace/config_schema/__init__.py +55 -13
  90. dagster_cloud/workspace/docker/__init__.py +76 -25
  91. dagster_cloud/workspace/docker/utils.py +1 -1
  92. dagster_cloud/workspace/ecs/__init__.py +1 -1
  93. dagster_cloud/workspace/ecs/client.py +51 -33
  94. dagster_cloud/workspace/ecs/launcher.py +76 -22
  95. dagster_cloud/workspace/ecs/run_launcher.py +3 -3
  96. dagster_cloud/workspace/ecs/utils.py +14 -5
  97. dagster_cloud/workspace/kubernetes/__init__.py +1 -1
  98. dagster_cloud/workspace/kubernetes/launcher.py +61 -29
  99. dagster_cloud/workspace/kubernetes/utils.py +34 -22
  100. dagster_cloud/workspace/user_code_launcher/__init__.py +5 -3
  101. dagster_cloud/workspace/user_code_launcher/process.py +16 -14
  102. dagster_cloud/workspace/user_code_launcher/user_code_launcher.py +552 -172
  103. dagster_cloud/workspace/user_code_launcher/utils.py +105 -1
  104. {dagster_cloud-1.8.2.dist-info → dagster_cloud-1.12.6.dist-info}/METADATA +48 -42
  105. dagster_cloud-1.12.6.dist-info/RECORD +134 -0
  106. {dagster_cloud-1.8.2.dist-info → dagster_cloud-1.12.6.dist-info}/WHEEL +1 -1
  107. dagster_cloud-1.8.2.dist-info/RECORD +0 -100
  108. {dagster_cloud-1.8.2.dist-info → dagster_cloud-1.12.6.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,121 @@
1
+ from typing import Optional
2
+
3
+ from opentelemetry.metrics import MeterProvider as APIMeterProvider
4
+ from opentelemetry.sdk.metrics.export import MetricExporter, MetricReader
5
+
6
+ from dagster_cloud.opentelemetry.config.exporter import metrics_instrument_types_enum
7
+ from dagster_cloud.opentelemetry.enum import MetricsExporterEnum, MetricsReaderEnum
8
+
9
+
10
+ def _validate_dict_keys_as_metric_instruments(src: dict, dict_name: str) -> None:
11
+ """Validate that all keys in a dictionary are valid metric instrument types."""
12
+ for key in src:
13
+ if key not in metrics_instrument_types_enum.enum_values:
14
+ raise ValueError(
15
+ f"OpenTelemetry configuration error - Invalid instrument type '{key}' in {dict_name}."
16
+ )
17
+
18
+
19
+ def build_metric_exporter(exporter_config: dict) -> MetricExporter:
20
+ """Build an OpenTelemetry metric exporter.
21
+ params:
22
+ exporter: dict: The exporter configuration.
23
+ - type: str: The exporter type.
24
+ - params: dict: The exporter parameters (varies).
25
+
26
+ Returns:
27
+ MetricExporter: The metric exporter instance.
28
+ """
29
+ exporter_type = exporter_config.get("type", MetricsExporterEnum.ConsoleMetricExporter.value)
30
+ exporter_params = exporter_config.get("params", {})
31
+
32
+ if "preferred_temporality" in exporter_params:
33
+ _validate_dict_keys_as_metric_instruments(
34
+ exporter_params["preferred_temporality"], "preferred_temporality"
35
+ )
36
+
37
+ if "preferred_aggregation" in exporter_params:
38
+ _validate_dict_keys_as_metric_instruments(
39
+ exporter_params["preferred_aggregation"], "preferred_aggregation"
40
+ )
41
+
42
+ if exporter_type == MetricsExporterEnum.HttpProtoOTLPExporter.value:
43
+ from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
44
+
45
+ return OTLPMetricExporter(**exporter_params)
46
+ elif exporter_type == MetricsExporterEnum.GrpcProtoOTLPExporter.value:
47
+ from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
48
+
49
+ return OTLPMetricExporter(**exporter_params)
50
+ elif exporter_type == MetricsExporterEnum.ConsoleMetricExporter.value:
51
+ from opentelemetry.sdk.metrics.export import ConsoleMetricExporter
52
+
53
+ return ConsoleMetricExporter(**exporter_params)
54
+
55
+ raise ValueError(f"Invalid exporter type: {exporter_type}")
56
+
57
+
58
+ def build_metric_reader(
59
+ exporter: Optional[MetricExporter] = None, reader_config: Optional[dict] = None
60
+ ) -> MetricReader:
61
+ """Build an OpenTelemetry metric reader.
62
+ params:
63
+ reader: dict: The reader configuration.
64
+
65
+ Returns:
66
+ MetricReader: The metric reader instance.
67
+ """
68
+ reader_config = reader_config or {}
69
+ reader_type = reader_config.get("type", MetricsReaderEnum.PeriodicExportingMetricReader.value)
70
+ reader_params = reader_config.get("params", {})
71
+
72
+ if reader_type == MetricsReaderEnum.PeriodicExportingMetricReader.value:
73
+ from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
74
+
75
+ if exporter is None:
76
+ raise ValueError("Exporter is required for PeriodicExportingMetricReader.")
77
+ return PeriodicExportingMetricReader(exporter=exporter, **reader_params)
78
+ elif reader_type == MetricsReaderEnum.InMemoryMetricReader.value:
79
+ from opentelemetry.sdk.metrics.export import InMemoryMetricReader
80
+
81
+ return InMemoryMetricReader(**reader_params)
82
+
83
+ raise ValueError(f"Invalid metric reader type: {reader_type}")
84
+
85
+
86
+ def build_meter_provider(
87
+ metric_readers: list[MetricReader],
88
+ resource_attributes: Optional[dict[str, str]] = None,
89
+ ) -> APIMeterProvider:
90
+ """Build an OpenTelemetry meter provider.
91
+ params:
92
+ metric_reader: MetricReader: The metric reader instance.
93
+ resource: Resource (optional): The resource instance.
94
+
95
+ Returns:
96
+ MeterProvider: The meter provider instance.
97
+ """
98
+ from opentelemetry.sdk.metrics import MeterProvider
99
+ from opentelemetry.sdk.resources import Resource
100
+
101
+ resource = (
102
+ Resource.create(attributes=resource_attributes)
103
+ if resource_attributes
104
+ else Resource.create({})
105
+ )
106
+
107
+ return MeterProvider(
108
+ metric_readers=metric_readers,
109
+ resource=resource,
110
+ shutdown_on_exit=False,
111
+ # TODO - implement support for views?
112
+ # https://opentelemetry-python.readthedocs.io/en/latest/_modules/opentelemetry/sdk/metrics/_internal/view.html#View
113
+ # views=views,
114
+ )
115
+
116
+
117
+ def build_noop_meter_provider() -> APIMeterProvider:
118
+ """Build a no-op OpenTelemetry meter provider."""
119
+ from opentelemetry.metrics import NoOpMeterProvider
120
+
121
+ return NoOpMeterProvider()
File without changes
@@ -0,0 +1,140 @@
1
+ from collections.abc import Generator, Iterable, Sequence
2
+ from typing import Callable, Optional, Union, cast
3
+
4
+ # (Gauge is only exposed as a private class)
5
+ from opentelemetry.metrics import (
6
+ CallbackOptions,
7
+ Counter,
8
+ Histogram,
9
+ Instrument,
10
+ ObservableCounter,
11
+ ObservableGauge,
12
+ ObservableUpDownCounter,
13
+ Observation,
14
+ UpDownCounter,
15
+ )
16
+ from opentelemetry.metrics._internal import Meter as SDKMeter
17
+
18
+ InstrumentKeyType = tuple[str, str, str]
19
+ ObservableCallback = Callable[[CallbackOptions], Iterable[Observation]]
20
+ ObservableGenerator = Generator[Iterable[Observation], CallbackOptions, None]
21
+ ObserverType = Union[ObservableCallback, ObservableGenerator]
22
+
23
+
24
+ class Meter:
25
+ def __init__(self, meter: SDKMeter):
26
+ self._meter = meter
27
+ self._instruments: dict[InstrumentKeyType, Instrument] = {}
28
+
29
+ @property
30
+ def name(self) -> str:
31
+ return self._meter.name
32
+
33
+ @property
34
+ def version(self) -> Optional[str]:
35
+ return self._meter.version
36
+
37
+ @property
38
+ def schema_url(self) -> Optional[str]:
39
+ return self._meter.schema_url
40
+
41
+ def get_counter(self, name: str, description: str = "", unit: str = "") -> Counter:
42
+ key = (name, description, unit)
43
+ instrument = self._instruments.get(key)
44
+ if not instrument:
45
+ instrument = self._meter.create_counter(
46
+ name=name,
47
+ description=description,
48
+ unit=unit,
49
+ )
50
+ self._instruments[key] = instrument
51
+
52
+ return cast("Counter", instrument)
53
+
54
+ def get_up_down_counter(
55
+ self, name: str, description: str = "", unit: str = ""
56
+ ) -> UpDownCounter:
57
+ key = (name, description, unit)
58
+ instrument = self._instruments.get(key)
59
+ if not instrument:
60
+ instrument = self._meter.create_up_down_counter(
61
+ name=name,
62
+ description=description,
63
+ unit=unit,
64
+ )
65
+ self._instruments[key] = instrument
66
+
67
+ return cast("UpDownCounter", instrument)
68
+
69
+ def get_histogram(self, name: str, description: str = "", unit: str = "") -> Histogram:
70
+ key = (name, description, unit)
71
+ instrument = self._instruments.get(key)
72
+ if not instrument:
73
+ instrument = self._meter.create_histogram(
74
+ name=name,
75
+ description=description,
76
+ unit=unit,
77
+ )
78
+ self._instruments[key] = instrument
79
+
80
+ return cast("Histogram", instrument)
81
+
82
+ def get_observable_counter(
83
+ self,
84
+ name: str,
85
+ description: str = "",
86
+ unit: str = "",
87
+ observers: Optional[Sequence[ObserverType]] = None,
88
+ ) -> ObservableCounter:
89
+ key = (name, description, unit)
90
+ instrument = self._instruments.get(key)
91
+ if not instrument:
92
+ instrument = self._meter.create_observable_counter(
93
+ name=name,
94
+ description=description,
95
+ unit=unit,
96
+ callbacks=observers,
97
+ )
98
+ self._instruments[key] = instrument
99
+
100
+ return cast("ObservableCounter", instrument)
101
+
102
+ def get_observable_up_down_counter(
103
+ self,
104
+ name: str,
105
+ description: str = "",
106
+ unit: str = "",
107
+ observers: Optional[Sequence[ObserverType]] = None,
108
+ ) -> ObservableUpDownCounter:
109
+ key = (name, description, unit)
110
+ instrument = self._instruments.get(key)
111
+ if not instrument:
112
+ instrument = self._meter.create_observable_up_down_counter(
113
+ name=name,
114
+ description=description,
115
+ unit=unit,
116
+ callbacks=observers,
117
+ )
118
+ self._instruments[key] = instrument
119
+
120
+ return cast("ObservableUpDownCounter", instrument)
121
+
122
+ def get_observable_gauge(
123
+ self,
124
+ name: str,
125
+ description: str = "",
126
+ unit: str = "",
127
+ observers: Optional[Sequence[ObserverType]] = None,
128
+ ) -> ObservableGauge:
129
+ key = (name, description, unit)
130
+ instrument = self._instruments.get(key)
131
+ if not instrument:
132
+ instrument = self._meter.create_observable_gauge(
133
+ name=name,
134
+ description=description,
135
+ unit=unit,
136
+ callbacks=observers,
137
+ )
138
+ self._instruments[key] = instrument
139
+
140
+ return cast("ObservableGauge", instrument)
File without changes
@@ -0,0 +1,40 @@
1
+ from dagster import DagsterError
2
+ from dagster._utils.error import SerializableErrorInfo
3
+ from grpc import Call as GrpcCall
4
+
5
+
6
+ def extract_dagster_error_attributes(error: DagsterError) -> dict[str, str]:
7
+ """Extracts attributes from a DagsterError that can be used for logging or metrics."""
8
+ error_attributes = {
9
+ "exception": type(error).__name__,
10
+ }
11
+
12
+ # DagsterError subtypes often have a field that contains a SerializableErrorInfo object
13
+ # which identify the cause of the error. This field names vary between error types and is not
14
+ # always present, but if they are they often identify the root cause of the error.
15
+ for field in dir(error):
16
+ if field.startswith("__"):
17
+ # Skip magic methods and fields
18
+ continue
19
+
20
+ attr = getattr(error, field)
21
+ if type(attr) is list and all(isinstance(item, SerializableErrorInfo) for item in attr):
22
+ serializable_error_infos: list[SerializableErrorInfo] = getattr(error, field)
23
+ for serializable_error_info in serializable_error_infos:
24
+ if serializable_error_info.cause and serializable_error_info.cause.cls_name:
25
+ error_attributes["cause"] = serializable_error_info.cause.cls_name
26
+ return error_attributes
27
+
28
+ # If not obtained from SerializableErrorInfo, use __cause__ to determine the error attributes
29
+ if hasattr(error, "__cause__") and error.__cause__:
30
+ error_cause = error.__cause__
31
+ cause_name = type(error_cause).__name__
32
+ if cause_name:
33
+ error_attributes["cause"] = cause_name
34
+
35
+ if isinstance(error_cause, GrpcCall):
36
+ grpc_call: GrpcCall = error_cause
37
+ code = grpc_call.code()
38
+ error_attributes["code"] = code.name
39
+
40
+ return error_attributes
@@ -0,0 +1,178 @@
1
+ from abc import ABC, abstractmethod
2
+ from contextlib import contextmanager
3
+ from typing import Callable, Optional
4
+
5
+ from dagster import DagsterError
6
+ from dagster._time import get_current_timestamp
7
+
8
+ from dagster_cloud.opentelemetry.controller import OpenTelemetryController
9
+ from dagster_cloud.opentelemetry.observers.dagster_exception_handler import (
10
+ extract_dagster_error_attributes,
11
+ )
12
+
13
+ ResultEvaluatorCallback = Callable[..., str]
14
+
15
+
16
+ class ExecutionObserverInstruments:
17
+ """OpenTelemetry instruments used to record generic code block executions."""
18
+
19
+ def __init__(
20
+ self, opentelemetry: OpenTelemetryController, metric_base_name: str, description: str
21
+ ):
22
+ self._meter = opentelemetry.get_meter(
23
+ name=metric_base_name,
24
+ attributes={
25
+ "description": description,
26
+ },
27
+ )
28
+
29
+ self._executions = self._meter.get_counter(
30
+ name=metric_base_name,
31
+ description=f"Number of {description}",
32
+ )
33
+
34
+ self._completions = self._meter.get_counter(
35
+ name=f"{metric_base_name}.completions",
36
+ description=f"Number of {description} completions by status",
37
+ )
38
+
39
+ self._exceptions = self._meter.get_counter(
40
+ name=f"{metric_base_name}.exceptions",
41
+ description=f"Number of exceptions caught in {description}",
42
+ )
43
+
44
+ self._duration = self._meter.get_histogram(
45
+ name=f"{metric_base_name}.duration",
46
+ description=f"Duration of {description}",
47
+ unit="seconds",
48
+ )
49
+
50
+ @property
51
+ def meter(self):
52
+ """Get the OpenTelemetry meter. This is useful for adding custom metrics."""
53
+ return self._meter
54
+
55
+ @property
56
+ def executions(self):
57
+ """Get the OpenTelemetry counter for the number of executions of the observed code block."""
58
+ return self._executions
59
+
60
+ @property
61
+ def completions(self):
62
+ """Get the OpenTelemetry counter for the number of completions of the observed code block."""
63
+ return self._completions
64
+
65
+ @property
66
+ def exceptions(self):
67
+ """Get the OpenTelemetry counter for the number of exceptions caught in the observed code block."""
68
+ return self._exceptions
69
+
70
+ @property
71
+ def duration(self):
72
+ """Get the OpenTelemetry gauge for the duration of the observed code block."""
73
+ return self._duration
74
+
75
+
76
+ class ExecutionResultObserver(ABC):
77
+ @abstractmethod
78
+ def evaluate_result(self, *args, **kwargs):
79
+ raise NotImplementedError("Subclasses must implement this method")
80
+
81
+
82
+ class _NoopExecutionResultObserver(ExecutionResultObserver):
83
+ """A no-op observer that does not record any metrics."""
84
+
85
+ def evaluate_result(self, *args, **kwargs):
86
+ pass
87
+
88
+
89
+ class _DagsterExecutionResultObserver(ExecutionResultObserver):
90
+ def __init__(
91
+ self,
92
+ instruments: ExecutionObserverInstruments,
93
+ attributes: dict[str, str],
94
+ result_evaluator_callback: Optional[ResultEvaluatorCallback] = None,
95
+ ):
96
+ self._result_evaluator_callback = result_evaluator_callback
97
+ self._instruments = instruments
98
+ self._attributes = attributes or {}
99
+ self._status = None
100
+
101
+ @property
102
+ def meter(self):
103
+ """Get the OpenTelemetry meter. This is useful for adding custom metrics."""
104
+ return self._instruments.meter
105
+
106
+ def evaluate_result(self, *args, **kwargs):
107
+ """Evaluate the result of the observed code block.
108
+
109
+ If a result evaluator callback is present, it will be called with the arguments passed to
110
+ this method, as well as the attributes and instruments passed to the observer.
111
+
112
+ If no result evaluator callback is present, the status will be set to "success".
113
+ """
114
+ if self._result_evaluator_callback:
115
+ self._status = self._result_evaluator_callback(
116
+ *args, attributes=self._attributes, instruments=self._instruments, **kwargs
117
+ )
118
+ else:
119
+ self._status = "success"
120
+
121
+ def set_status(self, status: str):
122
+ self._status = status
123
+
124
+ def get_status(self) -> Optional[str]:
125
+ return self._status
126
+
127
+
128
+ @contextmanager
129
+ def observe_execution(
130
+ opentelemetry: Optional[OpenTelemetryController],
131
+ event_key: str,
132
+ short_description: str,
133
+ result_evaluator_callback: Optional[ResultEvaluatorCallback] = None,
134
+ attributes: Optional[dict[str, str]] = None,
135
+ ):
136
+ """This context manager is used to observe the execution of a code block.
137
+
138
+ It records the number of executions, completions, exceptions, and the duration of the code block.
139
+ An optional evaluator callback can be provided to evaluate the result of the code block,
140
+ setting a completion status and also optionally implementing other side effects such as
141
+ recording additional metrics or logging based on the results.
142
+ """
143
+ attributes = attributes or {}
144
+ if opentelemetry and opentelemetry.metrics_enabled:
145
+ instruments = ExecutionObserverInstruments(
146
+ opentelemetry=opentelemetry,
147
+ metric_base_name=event_key,
148
+ description=short_description,
149
+ )
150
+ instruments.executions.add(1, attributes)
151
+
152
+ start_time = get_current_timestamp()
153
+ observer = _DagsterExecutionResultObserver(
154
+ instruments=instruments,
155
+ attributes=attributes,
156
+ result_evaluator_callback=result_evaluator_callback,
157
+ )
158
+
159
+ try:
160
+ yield observer
161
+ except DagsterError as e:
162
+ err_attributes = extract_dagster_error_attributes(e)
163
+ observer.set_status("exception")
164
+ instruments.exceptions.add(1, attributes={**attributes, **err_attributes})
165
+ raise
166
+ except Exception as e:
167
+ err_attributes = {"exception": type(e).__name__, **attributes}
168
+ observer.set_status("exception")
169
+ instruments.exceptions.add(1, err_attributes)
170
+ raise
171
+ finally:
172
+ instruments.completions.add(
173
+ 1, attributes={**attributes, "status": observer.get_status() or "unknown"}
174
+ )
175
+ instruments.duration.record(get_current_timestamp() - start_time, attributes)
176
+
177
+ else:
178
+ yield _NoopExecutionResultObserver()
@@ -0,0 +1,175 @@
1
+ """
2
+ @generated by mypy-protobuf. Do not edit manually!
3
+ isort:skip_file
4
+ If you make changes to this file, run "python -m dagster_cloud.pex.grpc.compile" after."""
5
+ import builtins
6
+ import google.protobuf.descriptor
7
+ import google.protobuf.message
8
+ import sys
9
+
10
+ if sys.version_info >= (3, 8):
11
+ import typing as typing_extensions
12
+ else:
13
+ import typing_extensions
14
+
15
+ DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
16
+
17
+ @typing_extensions.final
18
+ class Empty(google.protobuf.message.Message):
19
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
20
+
21
+ def __init__(
22
+ self,
23
+ ) -> None: ...
24
+
25
+ global___Empty = Empty
26
+
27
+ @typing_extensions.final
28
+ class CreatePexServerRequest(google.protobuf.message.Message):
29
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
30
+
31
+ CREATE_PEX_SERVER_ARGS_FIELD_NUMBER: builtins.int
32
+ create_pex_server_args: builtins.str
33
+ def __init__(
34
+ self,
35
+ *,
36
+ create_pex_server_args: builtins.str = ...,
37
+ ) -> None: ...
38
+ def ClearField(self, field_name: typing_extensions.Literal["create_pex_server_args", b"create_pex_server_args"]) -> None: ...
39
+
40
+ global___CreatePexServerRequest = CreatePexServerRequest
41
+
42
+ @typing_extensions.final
43
+ class CreatePexServerReply(google.protobuf.message.Message):
44
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
45
+
46
+ CREATE_PEX_SERVER_RESPONSE_FIELD_NUMBER: builtins.int
47
+ create_pex_server_response: builtins.str
48
+ def __init__(
49
+ self,
50
+ *,
51
+ create_pex_server_response: builtins.str = ...,
52
+ ) -> None: ...
53
+ def ClearField(self, field_name: typing_extensions.Literal["create_pex_server_response", b"create_pex_server_response"]) -> None: ...
54
+
55
+ global___CreatePexServerReply = CreatePexServerReply
56
+
57
+ @typing_extensions.final
58
+ class GetPexServersRequest(google.protobuf.message.Message):
59
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
60
+
61
+ GET_PEX_SERVERS_ARGS_FIELD_NUMBER: builtins.int
62
+ get_pex_servers_args: builtins.str
63
+ def __init__(
64
+ self,
65
+ *,
66
+ get_pex_servers_args: builtins.str = ...,
67
+ ) -> None: ...
68
+ def ClearField(self, field_name: typing_extensions.Literal["get_pex_servers_args", b"get_pex_servers_args"]) -> None: ...
69
+
70
+ global___GetPexServersRequest = GetPexServersRequest
71
+
72
+ @typing_extensions.final
73
+ class GetPexServersReply(google.protobuf.message.Message):
74
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
75
+
76
+ GET_PEX_SERVERS_RESPONSE_FIELD_NUMBER: builtins.int
77
+ get_pex_servers_response: builtins.str
78
+ def __init__(
79
+ self,
80
+ *,
81
+ get_pex_servers_response: builtins.str = ...,
82
+ ) -> None: ...
83
+ def ClearField(self, field_name: typing_extensions.Literal["get_pex_servers_response", b"get_pex_servers_response"]) -> None: ...
84
+
85
+ global___GetPexServersReply = GetPexServersReply
86
+
87
+ @typing_extensions.final
88
+ class GetCrashedPexServersRequest(google.protobuf.message.Message):
89
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
90
+
91
+ GET_CRASHED_PEX_SERVERS_ARGS_FIELD_NUMBER: builtins.int
92
+ get_crashed_pex_servers_args: builtins.str
93
+ def __init__(
94
+ self,
95
+ *,
96
+ get_crashed_pex_servers_args: builtins.str = ...,
97
+ ) -> None: ...
98
+ def ClearField(self, field_name: typing_extensions.Literal["get_crashed_pex_servers_args", b"get_crashed_pex_servers_args"]) -> None: ...
99
+
100
+ global___GetCrashedPexServersRequest = GetCrashedPexServersRequest
101
+
102
+ @typing_extensions.final
103
+ class GetCrashedPexServersReply(google.protobuf.message.Message):
104
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
105
+
106
+ GET_CRASHED_PEX_SERVERS_RESPONSE_FIELD_NUMBER: builtins.int
107
+ get_crashed_pex_servers_response: builtins.str
108
+ def __init__(
109
+ self,
110
+ *,
111
+ get_crashed_pex_servers_response: builtins.str = ...,
112
+ ) -> None: ...
113
+ def ClearField(self, field_name: typing_extensions.Literal["get_crashed_pex_servers_response", b"get_crashed_pex_servers_response"]) -> None: ...
114
+
115
+ global___GetCrashedPexServersReply = GetCrashedPexServersReply
116
+
117
+ @typing_extensions.final
118
+ class ShutdownPexServerRequest(google.protobuf.message.Message):
119
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
120
+
121
+ SHUTDOWN_PEX_SERVER_ARGS_FIELD_NUMBER: builtins.int
122
+ shutdown_pex_server_args: builtins.str
123
+ def __init__(
124
+ self,
125
+ *,
126
+ shutdown_pex_server_args: builtins.str = ...,
127
+ ) -> None: ...
128
+ def ClearField(self, field_name: typing_extensions.Literal["shutdown_pex_server_args", b"shutdown_pex_server_args"]) -> None: ...
129
+
130
+ global___ShutdownPexServerRequest = ShutdownPexServerRequest
131
+
132
+ @typing_extensions.final
133
+ class ShutdownPexServerReply(google.protobuf.message.Message):
134
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
135
+
136
+ SHUTDOWN_PEX_SERVER_RESPONSE_FIELD_NUMBER: builtins.int
137
+ shutdown_pex_server_response: builtins.str
138
+ def __init__(
139
+ self,
140
+ *,
141
+ shutdown_pex_server_response: builtins.str = ...,
142
+ ) -> None: ...
143
+ def ClearField(self, field_name: typing_extensions.Literal["shutdown_pex_server_response", b"shutdown_pex_server_response"]) -> None: ...
144
+
145
+ global___ShutdownPexServerReply = ShutdownPexServerReply
146
+
147
+ @typing_extensions.final
148
+ class PingRequest(google.protobuf.message.Message):
149
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
150
+
151
+ ECHO_FIELD_NUMBER: builtins.int
152
+ echo: builtins.str
153
+ def __init__(
154
+ self,
155
+ *,
156
+ echo: builtins.str = ...,
157
+ ) -> None: ...
158
+ def ClearField(self, field_name: typing_extensions.Literal["echo", b"echo"]) -> None: ...
159
+
160
+ global___PingRequest = PingRequest
161
+
162
+ @typing_extensions.final
163
+ class PingReply(google.protobuf.message.Message):
164
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
165
+
166
+ ECHO_FIELD_NUMBER: builtins.int
167
+ echo: builtins.str
168
+ def __init__(
169
+ self,
170
+ *,
171
+ echo: builtins.str = ...,
172
+ ) -> None: ...
173
+ def ClearField(self, field_name: typing_extensions.Literal["echo", b"echo"]) -> None: ...
174
+
175
+ global___PingReply = PingReply
@@ -1,8 +1,8 @@
1
- from .client import (
1
+ from dagster_cloud.pex.grpc.client import (
2
2
  MultiPexGrpcClient as MultiPexGrpcClient,
3
3
  wait_for_grpc_server as wait_for_grpc_server,
4
4
  )
5
- from .types import (
5
+ from dagster_cloud.pex.grpc.types import (
6
6
  CreatePexServerArgs as CreatePexServerArgs,
7
7
  GetPexServersArgs as GetPexServersArgs,
8
8
  PexServerHandle as PexServerHandle,
@@ -9,11 +9,11 @@ from dagster._core.errors import DagsterUserCodeProcessError, DagsterUserCodeUnr
9
9
  from dagster._grpc.client import DEFAULT_GRPC_TIMEOUT
10
10
  from dagster._grpc.utils import max_rx_bytes, max_send_bytes
11
11
  from dagster._serdes import serialize_value
12
- from dagster._serdes.serdes import deserialize_value
13
12
  from dagster._utils.error import SerializableErrorInfo, serializable_error_info_from_exc_info
13
+ from dagster_shared.serdes.serdes import deserialize_value
14
14
 
15
- from .__generated__ import MultiPexApiStub, multi_pex_api_pb2
16
- from .types import (
15
+ from dagster_cloud.pex.grpc.__generated__ import MultiPexApiStub, multi_pex_api_pb2
16
+ from dagster_cloud.pex.grpc.types import (
17
17
  CreatePexServerArgs,
18
18
  CreatePexServerResponse,
19
19
  GetCrashedPexServersArgs,
@@ -33,7 +33,7 @@ class MultiPexGrpcClient:
33
33
  if port:
34
34
  self._server_address = host + ":" + str(port)
35
35
  else:
36
- self._server_address = "unix:" + os.path.abspath(socket)
36
+ self._server_address = "unix:" + os.path.abspath(socket) # pyright: ignore[reportArgumentType,reportCallIssue]
37
37
 
38
38
  def create_pex_server(self, create_pex_server_args: CreatePexServerArgs):
39
39
  check.inst_param(create_pex_server_args, "create_pex_server_args", CreatePexServerArgs)