dagster-cloud 1.11.11__tar.gz → 1.11.13__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/PKG-INFO +7 -7
- dagster_cloud-1.11.13/dagster_cloud/__init__.py +7 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/agent/dagster_cloud_agent.py +13 -14
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/anomaly_detection/__init__.py +2 -2
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/anomaly_detection/defs.py +2 -2
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/auth/constants.py +21 -5
- dagster_cloud-1.11.13/dagster_cloud/batching/__init__.py +1 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/dagster_insights/__init__.py +12 -6
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/dagster_insights/bigquery/dbt_wrapper.py +8 -2
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/dagster_insights/bigquery/insights_bigquery_resource.py +4 -2
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/dagster_insights/snowflake/dagster_snowflake_insights.py +1 -9
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/dagster_insights/snowflake/dbt_wrapper.py +9 -2
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/dagster_insights/snowflake/definitions.py +5 -4
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/dagster_insights/snowflake/insights_snowflake_resource.py +1 -2
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/execution/utils/process.py +1 -1
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/instance/__init__.py +57 -39
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/opentelemetry/config/exporter.py +1 -1
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/opentelemetry/observers/execution_observer.py +4 -2
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/pex/grpc/__init__.py +2 -2
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/pex/grpc/client.py +2 -2
- dagster_cloud-1.11.13/dagster_cloud/pex/grpc/server/__init__.py +2 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/pex/grpc/server/cli/__init__.py +2 -2
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/pex/grpc/server/manager.py +2 -3
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/pex/grpc/server/registry.py +5 -1
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/pex/grpc/server/server.py +4 -4
- dagster_cloud-1.11.13/dagster_cloud/secrets/__init__.py +1 -0
- dagster_cloud-1.11.13/dagster_cloud/serverless/__init__.py +1 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/serverless/io_manager.py +4 -1
- dagster_cloud-1.11.13/dagster_cloud/storage/compute_logs/__init__.py +3 -0
- dagster_cloud-1.11.13/dagster_cloud/storage/defs_state/__init__.py +3 -0
- dagster_cloud-1.11.11/dagster_cloud/storage/defs_state/base_storage.py → dagster_cloud-1.11.13/dagster_cloud/storage/defs_state/storage.py +40 -17
- dagster_cloud-1.11.13/dagster_cloud/storage/event_logs/__init__.py +3 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/storage/event_logs/storage.py +1 -1
- dagster_cloud-1.11.13/dagster_cloud/storage/runs/__init__.py +1 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/storage/runs/storage.py +1 -1
- dagster_cloud-1.11.13/dagster_cloud/storage/schedules/__init__.py +1 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/storage/schedules/storage.py +1 -1
- dagster_cloud-1.11.13/dagster_cloud/version.py +1 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/workspace/config_schema/__init__.py +3 -3
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/workspace/docker/__init__.py +8 -7
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/workspace/docker/utils.py +1 -1
- dagster_cloud-1.11.13/dagster_cloud/workspace/ecs/__init__.py +1 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/workspace/ecs/launcher.py +7 -5
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/workspace/ecs/run_launcher.py +1 -2
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/workspace/ecs/utils.py +4 -1
- dagster_cloud-1.11.13/dagster_cloud/workspace/kubernetes/__init__.py +1 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/workspace/kubernetes/launcher.py +11 -12
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/workspace/kubernetes/utils.py +1 -2
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/workspace/user_code_launcher/__init__.py +5 -3
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/workspace/user_code_launcher/process.py +2 -3
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud.egg-info/PKG-INFO +7 -7
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud.egg-info/SOURCES.txt +0 -1
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud.egg-info/requires.txt +6 -6
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/pyproject.toml +6 -6
- dagster_cloud-1.11.11/dagster_cloud/__init__.py +0 -7
- dagster_cloud-1.11.11/dagster_cloud/batching/__init__.py +0 -1
- dagster_cloud-1.11.11/dagster_cloud/pex/grpc/server/__init__.py +0 -2
- dagster_cloud-1.11.11/dagster_cloud/secrets/__init__.py +0 -1
- dagster_cloud-1.11.11/dagster_cloud/serverless/__init__.py +0 -1
- dagster_cloud-1.11.11/dagster_cloud/storage/compute_logs/__init__.py +0 -1
- dagster_cloud-1.11.11/dagster_cloud/storage/defs_state/__init__.py +0 -1
- dagster_cloud-1.11.11/dagster_cloud/storage/defs_state/storage.py +0 -55
- dagster_cloud-1.11.11/dagster_cloud/storage/event_logs/__init__.py +0 -1
- dagster_cloud-1.11.11/dagster_cloud/storage/runs/__init__.py +0 -1
- dagster_cloud-1.11.11/dagster_cloud/storage/schedules/__init__.py +0 -1
- dagster_cloud-1.11.11/dagster_cloud/version.py +0 -1
- dagster_cloud-1.11.11/dagster_cloud/workspace/ecs/__init__.py +0 -1
- dagster_cloud-1.11.11/dagster_cloud/workspace/kubernetes/__init__.py +0 -1
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/README.md +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/agent/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/agent/cli/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/agent/instrumentation/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/agent/instrumentation/constants.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/agent/instrumentation/run_launch.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/agent/instrumentation/schedule.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/agent/instrumentation/sensor.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/agent/queries.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/anomaly_detection/mutation.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/anomaly_detection/types.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/api/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/api/dagster_cloud_api.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/artifacts/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/auth/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/batching/batcher.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/constants.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/dagster_insights/bigquery/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/dagster_insights/bigquery/bigquery_utils.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/dagster_insights/errors.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/dagster_insights/insights_utils.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/dagster_insights/metrics_utils.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/dagster_insights/query.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/dagster_insights/snowflake/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/dagster_insights/snowflake/snowflake_utils.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/definitions/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/definitions/job_selection.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/execution/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/execution/cloud_run_launcher/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/execution/cloud_run_launcher/k8s.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/execution/cloud_run_launcher/process.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/execution/monitoring/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/execution/utils/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/instrumentation/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/metadata/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/metadata/source_code.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/metrics/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/metrics/tracer.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/opentelemetry/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/opentelemetry/config/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/opentelemetry/config/log_record_processor.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/opentelemetry/config/logging_handler.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/opentelemetry/config/meter_provider.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/opentelemetry/config/metric_reader.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/opentelemetry/controller.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/opentelemetry/enum.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/opentelemetry/factories/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/opentelemetry/factories/logs.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/opentelemetry/factories/metrics.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/opentelemetry/metrics/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/opentelemetry/metrics/meter.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/opentelemetry/observers/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/opentelemetry/observers/dagster_exception_handler.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/pex/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/pex/grpc/__generated__/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/pex/grpc/__generated__/multi_pex_api_pb2.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/pex/grpc/__generated__/multi_pex_api_pb2.pyi +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/pex/grpc/__generated__/multi_pex_api_pb2_grpc.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/pex/grpc/compile.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/pex/grpc/types.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/py.typed +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/secrets/loader.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/storage/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/storage/client.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/storage/compute_logs/compute_log_manager.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/storage/defs_state/queries.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/storage/event_logs/queries.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/storage/event_logs/utils.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/storage/runs/queries.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/storage/schedules/queries.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/storage/tags.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/util/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/util/container_resources.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/util/errors.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/workspace/__init__.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/workspace/config_schema/docker.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/workspace/config_schema/ecs.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/workspace/config_schema/kubernetes.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/workspace/ecs/client.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/workspace/ecs/service.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/workspace/user_code_launcher/user_code_launcher.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/workspace/user_code_launcher/utils.py +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud.egg-info/dependency_links.txt +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud.egg-info/top_level.txt +0 -0
- {dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dagster-cloud
|
|
3
|
-
Version: 1.11.
|
|
3
|
+
Version: 1.11.13
|
|
4
4
|
Author-email: Elementl <support@elementl.com>
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
Project-URL: Homepage, https://dagster.io/cloud
|
|
@@ -28,9 +28,9 @@ Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
|
|
|
28
28
|
Classifier: Operating System :: OS Independent
|
|
29
29
|
Requires-Python: <3.14,>=3.9
|
|
30
30
|
Description-Content-Type: text/markdown
|
|
31
|
-
Requires-Dist: dagster==1.11.
|
|
32
|
-
Requires-Dist: dagster-shared==1.11.
|
|
33
|
-
Requires-Dist: dagster-cloud-cli==1.11.
|
|
31
|
+
Requires-Dist: dagster==1.11.13
|
|
32
|
+
Requires-Dist: dagster-shared==1.11.13
|
|
33
|
+
Requires-Dist: dagster-cloud-cli==1.11.13
|
|
34
34
|
Requires-Dist: opentelemetry-api<2,>=1.27.0
|
|
35
35
|
Requires-Dist: opentelemetry-sdk<2,>=1.27.0
|
|
36
36
|
Requires-Dist: opentelemetry-exporter-otlp-proto-grpc<2,>=1.27.0
|
|
@@ -65,12 +65,12 @@ Provides-Extra: insights
|
|
|
65
65
|
Requires-Dist: pyarrow; extra == "insights"
|
|
66
66
|
Provides-Extra: docker
|
|
67
67
|
Requires-Dist: docker; extra == "docker"
|
|
68
|
-
Requires-Dist: dagster-docker==0.27.
|
|
68
|
+
Requires-Dist: dagster-docker==0.27.13; extra == "docker"
|
|
69
69
|
Provides-Extra: kubernetes
|
|
70
70
|
Requires-Dist: kubernetes; extra == "kubernetes"
|
|
71
|
-
Requires-Dist: dagster-k8s==0.27.
|
|
71
|
+
Requires-Dist: dagster-k8s==0.27.13; extra == "kubernetes"
|
|
72
72
|
Provides-Extra: ecs
|
|
73
|
-
Requires-Dist: dagster-aws==0.27.
|
|
73
|
+
Requires-Dist: dagster-aws==0.27.13; extra == "ecs"
|
|
74
74
|
Requires-Dist: boto3; extra == "ecs"
|
|
75
75
|
Provides-Extra: sandbox
|
|
76
76
|
Requires-Dist: supervisor; extra == "sandbox"
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
from dagster._core.libraries import DagsterLibraryRegistry
|
|
2
|
+
|
|
3
|
+
from dagster_cloud.instance import DagsterCloudAgentInstance as DagsterCloudAgentInstance
|
|
4
|
+
from dagster_cloud.storage.compute_logs import CloudComputeLogManager as CloudComputeLogManager
|
|
5
|
+
from dagster_cloud.version import __version__
|
|
6
|
+
|
|
7
|
+
DagsterLibraryRegistry.register("dagster-cloud", __version__)
|
|
@@ -31,6 +31,16 @@ from dagster._utils.typed_dict import init_optional_typeddict
|
|
|
31
31
|
from dagster_cloud_cli.core.errors import DagsterCloudHTTPError, raise_http_error
|
|
32
32
|
from dagster_cloud_cli.core.workspace import CodeLocationDeployData
|
|
33
33
|
|
|
34
|
+
from dagster_cloud.agent.instrumentation.constants import DAGSTER_CLOUD_AGENT_METRIC_PREFIX
|
|
35
|
+
from dagster_cloud.agent.instrumentation.run_launch import extract_run_attributes
|
|
36
|
+
from dagster_cloud.agent.instrumentation.schedule import inspect_schedule_result
|
|
37
|
+
from dagster_cloud.agent.instrumentation.sensor import inspect_sensor_result
|
|
38
|
+
from dagster_cloud.agent.queries import (
|
|
39
|
+
ADD_AGENT_HEARTBEATS_MUTATION,
|
|
40
|
+
DEPLOYMENTS_QUERY,
|
|
41
|
+
GET_USER_CLOUD_REQUESTS_QUERY,
|
|
42
|
+
WORKSPACE_ENTRIES_QUERY,
|
|
43
|
+
)
|
|
34
44
|
from dagster_cloud.api.dagster_cloud_api import (
|
|
35
45
|
AgentHeartbeat,
|
|
36
46
|
AgentUtilizationMetrics,
|
|
@@ -48,26 +58,15 @@ from dagster_cloud.api.dagster_cloud_api import (
|
|
|
48
58
|
)
|
|
49
59
|
from dagster_cloud.batching import Batcher
|
|
50
60
|
from dagster_cloud.instance import DagsterCloudAgentInstance
|
|
61
|
+
from dagster_cloud.opentelemetry.observers.execution_observer import observe_execution
|
|
62
|
+
from dagster_cloud.util import SERVER_HANDLE_TAG, compressed_namedtuple_upload_file, is_isolated_run
|
|
63
|
+
from dagster_cloud.version import __version__
|
|
51
64
|
from dagster_cloud.workspace.user_code_launcher import (
|
|
52
65
|
DagsterCloudUserCodeLauncher,
|
|
53
66
|
UserCodeLauncherEntry,
|
|
54
67
|
)
|
|
55
68
|
from dagster_cloud.workspace.user_code_launcher.utils import get_instance_ref_for_user_code
|
|
56
69
|
|
|
57
|
-
from ..opentelemetry.observers.execution_observer import observe_execution
|
|
58
|
-
from ..util import SERVER_HANDLE_TAG, compressed_namedtuple_upload_file, is_isolated_run
|
|
59
|
-
from ..version import __version__
|
|
60
|
-
from .instrumentation.constants import DAGSTER_CLOUD_AGENT_METRIC_PREFIX
|
|
61
|
-
from .instrumentation.run_launch import extract_run_attributes
|
|
62
|
-
from .instrumentation.schedule import inspect_schedule_result
|
|
63
|
-
from .instrumentation.sensor import inspect_sensor_result
|
|
64
|
-
from .queries import (
|
|
65
|
-
ADD_AGENT_HEARTBEATS_MUTATION,
|
|
66
|
-
DEPLOYMENTS_QUERY,
|
|
67
|
-
GET_USER_CLOUD_REQUESTS_QUERY,
|
|
68
|
-
WORKSPACE_ENTRIES_QUERY,
|
|
69
|
-
)
|
|
70
|
-
|
|
71
70
|
if TYPE_CHECKING:
|
|
72
71
|
import datetime
|
|
73
72
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
from .defs import (
|
|
1
|
+
from dagster_cloud.anomaly_detection.defs import (
|
|
2
2
|
build_anomaly_detection_freshness_checks as build_anomaly_detection_freshness_checks,
|
|
3
3
|
)
|
|
4
|
-
from .types import (
|
|
4
|
+
from dagster_cloud.anomaly_detection.types import (
|
|
5
5
|
AnomalyDetectionModelParams as AnomalyDetectionModelParams,
|
|
6
6
|
BetaFreshnessAnomalyDetectionParams as BetaFreshnessAnomalyDetectionParams,
|
|
7
7
|
)
|
|
@@ -39,8 +39,8 @@ from dagster_cloud_cli.core.graphql_client import (
|
|
|
39
39
|
create_cloud_webserver_client,
|
|
40
40
|
)
|
|
41
41
|
|
|
42
|
-
from .mutation import ANOMALY_DETECTION_INFERENCE_MUTATION
|
|
43
|
-
from .types import (
|
|
42
|
+
from dagster_cloud.anomaly_detection.mutation import ANOMALY_DETECTION_INFERENCE_MUTATION
|
|
43
|
+
from dagster_cloud.anomaly_detection.types import (
|
|
44
44
|
AnomalyDetectionModelParams,
|
|
45
45
|
BetaFreshnessAnomalyDetectionParams,
|
|
46
46
|
FreshnessAnomalyDetectionResult,
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import uuid
|
|
1
2
|
from typing import Optional
|
|
2
3
|
|
|
3
4
|
from dagster._core.errors import DagsterInvariantViolationError
|
|
@@ -19,12 +20,27 @@ def get_organization_public_id_from_api_token(api_token: str) -> Optional[str]:
|
|
|
19
20
|
return split_token[2]
|
|
20
21
|
|
|
21
22
|
|
|
22
|
-
def
|
|
23
|
+
def decode_region_from_uuid(regional_token: str) -> Optional[str]:
|
|
24
|
+
try:
|
|
25
|
+
regional_uuid = uuid.UUID(regional_token)
|
|
26
|
+
except ValueError:
|
|
27
|
+
# if it's not an actual uuid, we can't decode region
|
|
28
|
+
return None
|
|
29
|
+
|
|
30
|
+
# custom uuids contain region subdomains in the first 2 bytes
|
|
31
|
+
if regional_uuid.version != 8 or regional_uuid.variant != uuid.RFC_4122:
|
|
32
|
+
return None
|
|
33
|
+
|
|
34
|
+
uuid_bytes = regional_uuid.bytes
|
|
35
|
+
return uuid_bytes[:2].decode("ascii")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def decode_agent_token(agent_token: str) -> tuple[Optional[str], Optional[str]]:
|
|
23
39
|
split_token = agent_token.split(":")
|
|
24
40
|
|
|
25
41
|
# Legacy agent token format - organization must be specified in dagster.yaml
|
|
26
42
|
if len(split_token) == 1:
|
|
27
|
-
return None
|
|
43
|
+
return None, None
|
|
28
44
|
|
|
29
45
|
token_type, *token = split_token
|
|
30
46
|
|
|
@@ -35,6 +51,6 @@ def get_organization_name_from_agent_token(agent_token: str) -> Optional[str]:
|
|
|
35
51
|
"Generate a new agent token in Dagster Cloud."
|
|
36
52
|
)
|
|
37
53
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
return organization
|
|
54
|
+
# token format: agent:<org>:<uuid>
|
|
55
|
+
organization, uuid_str = token
|
|
56
|
+
return organization, decode_region_from_uuid(uuid_str)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from dagster_cloud.batching.batcher import Batcher as Batcher
|
|
@@ -1,15 +1,19 @@
|
|
|
1
1
|
import sys
|
|
2
2
|
from typing import Any
|
|
3
3
|
|
|
4
|
-
from .snowflake.dbt_wrapper import
|
|
5
|
-
|
|
4
|
+
from dagster_cloud.dagster_insights.snowflake.dbt_wrapper import (
|
|
5
|
+
dbt_with_snowflake_insights as dbt_with_snowflake_insights,
|
|
6
|
+
)
|
|
7
|
+
from dagster_cloud.dagster_insights.snowflake.definitions import (
|
|
6
8
|
create_snowflake_insights_asset_and_schedule as create_snowflake_insights_asset_and_schedule,
|
|
7
9
|
)
|
|
8
|
-
from .snowflake.snowflake_utils import
|
|
10
|
+
from dagster_cloud.dagster_insights.snowflake.snowflake_utils import (
|
|
11
|
+
meter_snowflake_query as meter_snowflake_query,
|
|
12
|
+
)
|
|
9
13
|
|
|
10
14
|
dagster_snowflake_req_imports = {"InsightsSnowflakeResource"}
|
|
11
15
|
try:
|
|
12
|
-
from .snowflake.insights_snowflake_resource import (
|
|
16
|
+
from dagster_cloud.dagster_insights.snowflake.insights_snowflake_resource import (
|
|
13
17
|
InsightsSnowflakeResource as InsightsSnowflakeResource,
|
|
14
18
|
)
|
|
15
19
|
except ImportError:
|
|
@@ -17,8 +21,10 @@ except ImportError:
|
|
|
17
21
|
|
|
18
22
|
dagster_bigquery_req_imports = {"InsightsBigQueryResource", "dbt_with_bigquery_insights"}
|
|
19
23
|
try:
|
|
20
|
-
from .bigquery.dbt_wrapper import
|
|
21
|
-
|
|
24
|
+
from dagster_cloud.dagster_insights.bigquery.dbt_wrapper import (
|
|
25
|
+
dbt_with_bigquery_insights as dbt_with_bigquery_insights,
|
|
26
|
+
)
|
|
27
|
+
from dagster_cloud.dagster_insights.bigquery.insights_bigquery_resource import (
|
|
22
28
|
InsightsBigQueryResource as InsightsBigQueryResource,
|
|
23
29
|
)
|
|
24
30
|
except ImportError:
|
|
@@ -18,8 +18,14 @@ from dagster_dbt import DbtCliInvocation
|
|
|
18
18
|
from dagster_dbt.version import __version__ as dagster_dbt_version
|
|
19
19
|
from packaging import version
|
|
20
20
|
|
|
21
|
-
from
|
|
22
|
-
|
|
21
|
+
from dagster_cloud.dagster_insights.bigquery.bigquery_utils import (
|
|
22
|
+
build_bigquery_cost_metadata,
|
|
23
|
+
marker_asset_key_for_job,
|
|
24
|
+
)
|
|
25
|
+
from dagster_cloud.dagster_insights.insights_utils import (
|
|
26
|
+
extract_asset_info_from_event,
|
|
27
|
+
handle_raise_on_error,
|
|
28
|
+
)
|
|
23
29
|
|
|
24
30
|
if TYPE_CHECKING:
|
|
25
31
|
from dbt.adapters.base.impl import BaseAdapter
|
|
@@ -7,10 +7,12 @@ from dagster_gcp import BigQueryResource
|
|
|
7
7
|
from dagster_gcp.bigquery.utils import setup_gcp_creds
|
|
8
8
|
from google.cloud import bigquery
|
|
9
9
|
|
|
10
|
+
from dagster_cloud.dagster_insights.bigquery.bigquery_utils import (
|
|
11
|
+
build_bigquery_cost_metadata,
|
|
12
|
+
marker_asset_key_for_job,
|
|
13
|
+
)
|
|
10
14
|
from dagster_cloud.dagster_insights.insights_utils import get_current_context_and_asset_key
|
|
11
15
|
|
|
12
|
-
from .bigquery_utils import build_bigquery_cost_metadata, marker_asset_key_for_job
|
|
13
|
-
|
|
14
16
|
OUTPUT_NON_ASSET_SIGIL = "__bigquery_query_metadata_"
|
|
15
17
|
|
|
16
18
|
|
|
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Optional
|
|
|
6
6
|
|
|
7
7
|
from dagster import AssetKey, AssetsDefinition, ScheduleDefinition
|
|
8
8
|
|
|
9
|
-
from .snowflake_utils import OPAQUE_ID_SQL_SIGIL
|
|
9
|
+
from dagster_cloud.dagster_insights.snowflake.snowflake_utils import OPAQUE_ID_SQL_SIGIL
|
|
10
10
|
|
|
11
11
|
if TYPE_CHECKING:
|
|
12
12
|
from dagster_snowflake import SnowflakeConnection
|
|
@@ -78,10 +78,6 @@ HAVING ARRAY_SIZE(opaque_ids) > 0
|
|
|
78
78
|
|
|
79
79
|
costs: list[tuple[str, float, str]] = []
|
|
80
80
|
|
|
81
|
-
print(
|
|
82
|
-
f"{len(results) if results else 0} annotated queries returned from snowflake query_history"
|
|
83
|
-
)
|
|
84
|
-
|
|
85
81
|
if not results:
|
|
86
82
|
return []
|
|
87
83
|
|
|
@@ -94,8 +90,4 @@ HAVING ARRAY_SIZE(opaque_ids) > 0
|
|
|
94
90
|
for opaque_id in opaque_ids:
|
|
95
91
|
costs.append((opaque_id, float(cost), query_id))
|
|
96
92
|
|
|
97
|
-
print(
|
|
98
|
-
f"Reported costs for {len(costs)} of {total} asset materializations found in the"
|
|
99
|
-
" query_history."
|
|
100
|
-
)
|
|
101
93
|
return costs
|
|
@@ -13,8 +13,15 @@ from dagster import (
|
|
|
13
13
|
Output,
|
|
14
14
|
)
|
|
15
15
|
|
|
16
|
-
from
|
|
17
|
-
|
|
16
|
+
from dagster_cloud.dagster_insights.insights_utils import (
|
|
17
|
+
extract_asset_info_from_event,
|
|
18
|
+
handle_raise_on_error,
|
|
19
|
+
)
|
|
20
|
+
from dagster_cloud.dagster_insights.snowflake.snowflake_utils import (
|
|
21
|
+
OPAQUE_ID_SQL_SIGIL,
|
|
22
|
+
build_opaque_id_metadata,
|
|
23
|
+
marker_asset_key_for_job,
|
|
24
|
+
)
|
|
18
25
|
|
|
19
26
|
if TYPE_CHECKING:
|
|
20
27
|
from dagster_dbt import DbtCliInvocation
|
|
@@ -2,7 +2,6 @@ import warnings
|
|
|
2
2
|
from collections.abc import Sequence
|
|
3
3
|
from dataclasses import dataclass
|
|
4
4
|
from datetime import date, datetime, timedelta, timezone
|
|
5
|
-
from pprint import pprint
|
|
6
5
|
from typing import TYPE_CHECKING, Optional, Union
|
|
7
6
|
|
|
8
7
|
from dagster import (
|
|
@@ -27,8 +26,10 @@ from dagster._core.storage.tags import (
|
|
|
27
26
|
ASSET_PARTITION_RANGE_START_TAG,
|
|
28
27
|
)
|
|
29
28
|
|
|
30
|
-
from
|
|
31
|
-
from .dagster_snowflake_insights import
|
|
29
|
+
from dagster_cloud.dagster_insights.metrics_utils import put_cost_information
|
|
30
|
+
from dagster_cloud.dagster_insights.snowflake.dagster_snowflake_insights import (
|
|
31
|
+
get_cost_data_for_hour,
|
|
32
|
+
)
|
|
32
33
|
|
|
33
34
|
if TYPE_CHECKING:
|
|
34
35
|
from dagster_snowflake import SnowflakeConnection
|
|
@@ -154,7 +155,7 @@ def create_snowflake_insights_asset_and_schedule(
|
|
|
154
155
|
)
|
|
155
156
|
|
|
156
157
|
if dry_run:
|
|
157
|
-
|
|
158
|
+
pass
|
|
158
159
|
else:
|
|
159
160
|
context.log.info(
|
|
160
161
|
f"Submitting cost information for {len(costs)} queries to Dagster Insights"
|
|
@@ -20,8 +20,7 @@ from dagster_snowflake import SnowflakeConnection, SnowflakeResource
|
|
|
20
20
|
from snowflake.connector.cursor import SnowflakeCursor
|
|
21
21
|
|
|
22
22
|
from dagster_cloud.dagster_insights.insights_utils import get_current_context_and_asset_key
|
|
23
|
-
|
|
24
|
-
from .snowflake_utils import meter_snowflake_query
|
|
23
|
+
from dagster_cloud.dagster_insights.snowflake.snowflake_utils import meter_snowflake_query
|
|
25
24
|
|
|
26
25
|
|
|
27
26
|
def get_current_context_and_asset_key_or_warn() -> tuple[
|
|
@@ -32,14 +32,13 @@ from dagster_cloud_cli.core.headers.auth import DagsterCloudInstanceScope
|
|
|
32
32
|
from urllib3 import Retry
|
|
33
33
|
|
|
34
34
|
from dagster_cloud.agent import AgentQueuesConfig
|
|
35
|
+
from dagster_cloud.auth.constants import decode_agent_token
|
|
36
|
+
from dagster_cloud.opentelemetry.config import opentelemetry_config_schema
|
|
37
|
+
from dagster_cloud.opentelemetry.controller import OpenTelemetryController
|
|
38
|
+
from dagster_cloud.storage.client import dagster_cloud_api_config
|
|
39
|
+
from dagster_cloud.util import get_env_names_from_config, is_isolated_run
|
|
35
40
|
from dagster_cloud.version import __version__
|
|
36
41
|
|
|
37
|
-
from ..auth.constants import get_organization_name_from_agent_token
|
|
38
|
-
from ..opentelemetry.config import opentelemetry_config_schema
|
|
39
|
-
from ..opentelemetry.controller import OpenTelemetryController
|
|
40
|
-
from ..storage.client import dagster_cloud_api_config
|
|
41
|
-
from ..util import get_env_names_from_config, is_isolated_run
|
|
42
|
-
|
|
43
42
|
if TYPE_CHECKING:
|
|
44
43
|
from requests import Session
|
|
45
44
|
|
|
@@ -87,14 +86,18 @@ class DagsterCloudAgentInstance(DagsterCloudInstance):
|
|
|
87
86
|
super().__init__(*args, **kwargs)
|
|
88
87
|
|
|
89
88
|
self._unprocessed_dagster_cloud_api_config = dagster_cloud_api
|
|
90
|
-
self._dagster_cloud_api_config =
|
|
91
|
-
|
|
89
|
+
self._dagster_cloud_api_config = check.not_none(
|
|
90
|
+
self._get_processed_config(
|
|
91
|
+
"dagster_cloud_api",
|
|
92
|
+
dagster_cloud_api,
|
|
93
|
+
dagster_cloud_api_config(),
|
|
94
|
+
)
|
|
92
95
|
)
|
|
93
96
|
|
|
94
97
|
check.invariant(
|
|
95
98
|
not (
|
|
96
|
-
self._dagster_cloud_api_config.get("deployment")
|
|
97
|
-
and self._dagster_cloud_api_config.get("deployments")
|
|
99
|
+
self._dagster_cloud_api_config.get("deployment")
|
|
100
|
+
and self._dagster_cloud_api_config.get("deployments")
|
|
98
101
|
),
|
|
99
102
|
"Cannot set both deployment and deployments in `dagster_cloud_api`",
|
|
100
103
|
)
|
|
@@ -175,16 +178,16 @@ class DagsterCloudAgentInstance(DagsterCloudInstance):
|
|
|
175
178
|
return processed_config.value
|
|
176
179
|
|
|
177
180
|
def _dagster_cloud_api_config_for_deployment(self, deployment_name: Optional[str]):
|
|
178
|
-
new_api_config = dict(copy.deepcopy(self._dagster_cloud_api_config))
|
|
181
|
+
new_api_config = dict(copy.deepcopy(self._dagster_cloud_api_config))
|
|
179
182
|
if deployment_name:
|
|
180
|
-
new_api_config["deployment"] = deployment_name
|
|
183
|
+
new_api_config["deployment"] = deployment_name
|
|
181
184
|
if self.includes_branch_deployments:
|
|
182
|
-
del new_api_config["branch_deployments"]
|
|
183
|
-
new_api_config.pop("deployments", None)
|
|
184
|
-
new_api_config.pop("all_serverless_deployments", None)
|
|
185
|
+
del new_api_config["branch_deployments"]
|
|
186
|
+
new_api_config.pop("deployments", None)
|
|
187
|
+
new_api_config.pop("all_serverless_deployments", None)
|
|
185
188
|
else:
|
|
186
|
-
new_api_config.pop("deployment", None)
|
|
187
|
-
new_api_config.pop("deployments", None)
|
|
189
|
+
new_api_config.pop("deployment", None)
|
|
190
|
+
new_api_config.pop("deployments", None)
|
|
188
191
|
|
|
189
192
|
return new_api_config
|
|
190
193
|
|
|
@@ -238,11 +241,11 @@ class DagsterCloudAgentInstance(DagsterCloudInstance):
|
|
|
238
241
|
return socket_option
|
|
239
242
|
|
|
240
243
|
def _socket_options(self):
|
|
241
|
-
if self._dagster_cloud_api_config.get("socket_options") is None:
|
|
244
|
+
if self._dagster_cloud_api_config.get("socket_options") is None:
|
|
242
245
|
return None
|
|
243
246
|
|
|
244
247
|
translated_socket_options = []
|
|
245
|
-
for socket_option in self._dagster_cloud_api_config["socket_options"]:
|
|
248
|
+
for socket_option in self._dagster_cloud_api_config["socket_options"]:
|
|
246
249
|
check.invariant(
|
|
247
250
|
len(socket_option) == 3, "Each socket option must be a list of three values"
|
|
248
251
|
)
|
|
@@ -284,7 +287,7 @@ class DagsterCloudAgentInstance(DagsterCloudInstance):
|
|
|
284
287
|
adapter_kwargs=dict(
|
|
285
288
|
max_retries=Retry(
|
|
286
289
|
total=self.dagster_cloud_api_retries,
|
|
287
|
-
backoff_factor=self._dagster_cloud_api_config["backoff_factor"],
|
|
290
|
+
backoff_factor=self._dagster_cloud_api_config["backoff_factor"],
|
|
288
291
|
),
|
|
289
292
|
socket_options=self._socket_options(),
|
|
290
293
|
)
|
|
@@ -314,21 +317,26 @@ class DagsterCloudAgentInstance(DagsterCloudInstance):
|
|
|
314
317
|
|
|
315
318
|
@property
|
|
316
319
|
def dagster_cloud_url(self):
|
|
317
|
-
if "url" in self._dagster_cloud_api_config:
|
|
318
|
-
return self._dagster_cloud_api_config["url"]
|
|
320
|
+
if "url" in self._dagster_cloud_api_config:
|
|
321
|
+
return self._dagster_cloud_api_config["url"]
|
|
319
322
|
|
|
320
|
-
organization =
|
|
323
|
+
organization, region = decode_agent_token(self.dagster_cloud_agent_token)
|
|
321
324
|
if not organization:
|
|
322
325
|
raise DagsterInvariantViolationError(
|
|
323
326
|
"Could not derive Dagster Cloud URL from agent token. Create a new agent token or"
|
|
324
327
|
" set the `url` field under `dagster_cloud_api` in your `dagster.yaml`."
|
|
325
328
|
)
|
|
326
329
|
|
|
327
|
-
return
|
|
330
|
+
return (
|
|
331
|
+
f"https://{organization}.agent.{region}.dagster.cloud"
|
|
332
|
+
if region
|
|
333
|
+
else f"https://{organization}.agent.dagster.cloud"
|
|
334
|
+
)
|
|
328
335
|
|
|
329
336
|
@property
|
|
330
337
|
def organization_name(self) -> Optional[str]:
|
|
331
|
-
|
|
338
|
+
organization, _ = decode_agent_token(self.dagster_cloud_agent_token)
|
|
339
|
+
return organization
|
|
332
340
|
|
|
333
341
|
@property
|
|
334
342
|
def deployment_name(self) -> Optional[str]:
|
|
@@ -343,26 +351,31 @@ class DagsterCloudAgentInstance(DagsterCloudInstance):
|
|
|
343
351
|
|
|
344
352
|
@property
|
|
345
353
|
def deployment_names(self) -> list[str]:
|
|
346
|
-
if self._dagster_cloud_api_config.get("deployment"):
|
|
347
|
-
return [self._dagster_cloud_api_config["deployment"]]
|
|
354
|
+
if self._dagster_cloud_api_config.get("deployment"):
|
|
355
|
+
return [self._dagster_cloud_api_config["deployment"]]
|
|
348
356
|
|
|
349
|
-
return self._dagster_cloud_api_config.get("deployments", [])
|
|
357
|
+
return self._dagster_cloud_api_config.get("deployments", [])
|
|
350
358
|
|
|
351
359
|
@property
|
|
352
360
|
def include_all_serverless_deployments(self) -> bool:
|
|
353
|
-
return self._dagster_cloud_api_config.get("all_serverless_deployments")
|
|
361
|
+
return self._dagster_cloud_api_config.get("all_serverless_deployments") or False
|
|
354
362
|
|
|
355
363
|
@property
|
|
356
364
|
def dagit_url(self):
|
|
357
|
-
organization =
|
|
365
|
+
organization, region = decode_agent_token(self.dagster_cloud_agent_token)
|
|
358
366
|
if not organization:
|
|
359
367
|
raise Exception(
|
|
360
368
|
"Could not derive Dagster Cloud URL from agent token to generate a Dagit URL."
|
|
361
369
|
" Generate a new agent token in the Dagit UI."
|
|
362
370
|
)
|
|
363
371
|
|
|
364
|
-
deployment = self._dagster_cloud_api_config.get("deployment")
|
|
365
|
-
|
|
372
|
+
deployment = self._dagster_cloud_api_config.get("deployment")
|
|
373
|
+
base_url = (
|
|
374
|
+
f"https://{organization}.{region}.dagster.cloud/"
|
|
375
|
+
if region
|
|
376
|
+
else f"https://{organization}.dagster.cloud/"
|
|
377
|
+
)
|
|
378
|
+
return base_url + (f"{deployment}/" if deployment else "")
|
|
366
379
|
|
|
367
380
|
@property
|
|
368
381
|
def dagster_cloud_graphql_url(self):
|
|
@@ -412,33 +425,38 @@ class DagsterCloudAgentInstance(DagsterCloudInstance):
|
|
|
412
425
|
return get_agent_headers(self._dagster_cloud_api_config, scope=scope) # pyright: ignore[reportArgumentType]
|
|
413
426
|
|
|
414
427
|
@property
|
|
415
|
-
def dagster_cloud_agent_token(self):
|
|
416
|
-
|
|
428
|
+
def dagster_cloud_agent_token(self) -> str:
|
|
429
|
+
check.invariant(
|
|
430
|
+
self._dagster_cloud_api_config.get("agent_token") is not None,
|
|
431
|
+
"No agent token found in dagster_cloud_api configuration. An agent token is required"
|
|
432
|
+
" for Dagster Cloud authentication.",
|
|
433
|
+
)
|
|
434
|
+
return self._dagster_cloud_api_config["agent_token"]
|
|
417
435
|
|
|
418
436
|
@property
|
|
419
437
|
def dagster_cloud_api_retries(self) -> int:
|
|
420
|
-
return self._dagster_cloud_api_config["retries"]
|
|
438
|
+
return self._dagster_cloud_api_config["retries"]
|
|
421
439
|
|
|
422
440
|
@property
|
|
423
441
|
def dagster_cloud_api_timeout(self) -> int:
|
|
424
|
-
return self._dagster_cloud_api_config["timeout"]
|
|
442
|
+
return self._dagster_cloud_api_config["timeout"]
|
|
425
443
|
|
|
426
444
|
@property
|
|
427
445
|
def dagster_cloud_api_proxies(self) -> Optional[dict[str, str]]:
|
|
428
446
|
# Requests library modifies the proxies key so create a copy
|
|
429
447
|
return (
|
|
430
448
|
self._dagster_cloud_api_config.get("proxies").copy() # pyright: ignore[reportOptionalMemberAccess]
|
|
431
|
-
if self._dagster_cloud_api_config.get("proxies")
|
|
449
|
+
if self._dagster_cloud_api_config.get("proxies")
|
|
432
450
|
else {}
|
|
433
451
|
)
|
|
434
452
|
|
|
435
453
|
@property
|
|
436
454
|
def dagster_cloud_api_agent_label(self) -> Optional[str]:
|
|
437
|
-
return self._dagster_cloud_api_config.get("agent_label")
|
|
455
|
+
return self._dagster_cloud_api_config.get("agent_label")
|
|
438
456
|
|
|
439
457
|
@property
|
|
440
458
|
def includes_branch_deployments(self) -> bool:
|
|
441
|
-
return self._dagster_cloud_api_config.get("branch_deployments", False)
|
|
459
|
+
return self._dagster_cloud_api_config.get("branch_deployments", False)
|
|
442
460
|
|
|
443
461
|
@property
|
|
444
462
|
def instance_uuid(self) -> str:
|
|
@@ -5,8 +5,10 @@ from typing import Callable, Optional
|
|
|
5
5
|
from dagster import DagsterError
|
|
6
6
|
from dagster._time import get_current_timestamp
|
|
7
7
|
|
|
8
|
-
from
|
|
9
|
-
from .dagster_exception_handler import
|
|
8
|
+
from dagster_cloud.opentelemetry.controller import OpenTelemetryController
|
|
9
|
+
from dagster_cloud.opentelemetry.observers.dagster_exception_handler import (
|
|
10
|
+
extract_dagster_error_attributes,
|
|
11
|
+
)
|
|
10
12
|
|
|
11
13
|
ResultEvaluatorCallback = Callable[..., str]
|
|
12
14
|
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
from .client import (
|
|
1
|
+
from dagster_cloud.pex.grpc.client import (
|
|
2
2
|
MultiPexGrpcClient as MultiPexGrpcClient,
|
|
3
3
|
wait_for_grpc_server as wait_for_grpc_server,
|
|
4
4
|
)
|
|
5
|
-
from .types import (
|
|
5
|
+
from dagster_cloud.pex.grpc.types import (
|
|
6
6
|
CreatePexServerArgs as CreatePexServerArgs,
|
|
7
7
|
GetPexServersArgs as GetPexServersArgs,
|
|
8
8
|
PexServerHandle as PexServerHandle,
|
|
@@ -12,8 +12,8 @@ from dagster._serdes import serialize_value
|
|
|
12
12
|
from dagster._utils.error import SerializableErrorInfo, serializable_error_info_from_exc_info
|
|
13
13
|
from dagster_shared.serdes.serdes import deserialize_value
|
|
14
14
|
|
|
15
|
-
from .__generated__ import MultiPexApiStub, multi_pex_api_pb2
|
|
16
|
-
from .types import (
|
|
15
|
+
from dagster_cloud.pex.grpc.__generated__ import MultiPexApiStub, multi_pex_api_pb2
|
|
16
|
+
from dagster_cloud.pex.grpc.types import (
|
|
17
17
|
CreatePexServerArgs,
|
|
18
18
|
CreatePexServerResponse,
|
|
19
19
|
GetCrashedPexServersArgs,
|
{dagster_cloud-1.11.11 → dagster_cloud-1.11.13}/dagster_cloud/pex/grpc/server/cli/__init__.py
RENAMED
|
@@ -9,8 +9,8 @@ from dagster._utils.interrupts import setup_interrupt_handlers
|
|
|
9
9
|
from dagster_cloud_cli.core.workspace import PexMetadata
|
|
10
10
|
from typer import Option, Typer
|
|
11
11
|
|
|
12
|
-
from
|
|
13
|
-
from
|
|
12
|
+
from dagster_cloud.pex.grpc.server.registry import PexS3Registry
|
|
13
|
+
from dagster_cloud.pex.grpc.server.server import run_multipex_server
|
|
14
14
|
|
|
15
15
|
app = Typer(hidden=True)
|
|
16
16
|
|
|
@@ -18,11 +18,10 @@ from dagster_shared import seven
|
|
|
18
18
|
from dagster_shared.ipc import open_ipc_subprocess
|
|
19
19
|
from pydantic import BaseModel, Extra
|
|
20
20
|
|
|
21
|
+
from dagster_cloud.pex.grpc.server.registry import PexS3Registry
|
|
22
|
+
from dagster_cloud.pex.grpc.types import PexServerHandle
|
|
21
23
|
from dagster_cloud.workspace.user_code_launcher.utils import get_grpc_server_env
|
|
22
24
|
|
|
23
|
-
from ..types import PexServerHandle
|
|
24
|
-
from .registry import PexS3Registry
|
|
25
|
-
|
|
26
25
|
logger = logging.getLogger("dagster.multipex")
|
|
27
26
|
|
|
28
27
|
|
|
@@ -28,7 +28,11 @@ def _download_from_s3(filename: str, local_filepath: str):
|
|
|
28
28
|
|
|
29
29
|
config = Config(retries={"max_attempts": 3, "mode": "standard"})
|
|
30
30
|
|
|
31
|
-
s3 = boto3.client(
|
|
31
|
+
s3 = boto3.client(
|
|
32
|
+
"s3",
|
|
33
|
+
region_name=os.getenv("DAGSTER_CLOUD_SERVERLESS_REGION", "us-west-2"),
|
|
34
|
+
config=config,
|
|
35
|
+
)
|
|
32
36
|
|
|
33
37
|
# TODO: move the bucket and prefix to pex_metdata
|
|
34
38
|
s3_bucket_name = os.environ["DAGSTER_CLOUD_SERVERLESS_STORAGE_S3_BUCKET"]
|