dagster-cloud 1.10.11__py3-none-any.whl → 1.12.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dagster_cloud/__init__.py +3 -3
- dagster_cloud/agent/dagster_cloud_agent.py +64 -20
- dagster_cloud/agent/instrumentation/run_launch.py +2 -2
- dagster_cloud/agent/instrumentation/schedule.py +1 -1
- dagster_cloud/agent/instrumentation/sensor.py +1 -1
- dagster_cloud/anomaly_detection/__init__.py +2 -2
- dagster_cloud/anomaly_detection/defs.py +11 -8
- dagster_cloud/api/dagster_cloud_api.py +7 -5
- dagster_cloud/auth/constants.py +21 -5
- dagster_cloud/batching/__init__.py +1 -1
- dagster_cloud/dagster_insights/__init__.py +12 -6
- dagster_cloud/dagster_insights/bigquery/dbt_wrapper.py +8 -2
- dagster_cloud/dagster_insights/bigquery/insights_bigquery_resource.py +4 -2
- dagster_cloud/dagster_insights/insights_utils.py +1 -1
- dagster_cloud/dagster_insights/metrics_utils.py +1 -1
- dagster_cloud/dagster_insights/snowflake/dagster_snowflake_insights.py +1 -9
- dagster_cloud/dagster_insights/snowflake/dbt_wrapper.py +9 -2
- dagster_cloud/dagster_insights/snowflake/definitions.py +5 -4
- dagster_cloud/dagster_insights/snowflake/insights_snowflake_resource.py +1 -2
- dagster_cloud/definitions/__init__.py +0 -0
- dagster_cloud/definitions/job_selection.py +36 -0
- dagster_cloud/execution/utils/process.py +1 -1
- dagster_cloud/instance/__init__.py +81 -42
- dagster_cloud/metadata/source_code.py +3 -1
- dagster_cloud/opentelemetry/config/exporter.py +1 -1
- dagster_cloud/opentelemetry/controller.py +1 -1
- dagster_cloud/opentelemetry/observers/dagster_exception_handler.py +1 -1
- dagster_cloud/opentelemetry/observers/execution_observer.py +4 -2
- dagster_cloud/pex/grpc/__init__.py +2 -2
- dagster_cloud/pex/grpc/client.py +2 -2
- dagster_cloud/pex/grpc/server/__init__.py +2 -2
- dagster_cloud/pex/grpc/server/cli/__init__.py +2 -2
- dagster_cloud/pex/grpc/server/manager.py +5 -4
- dagster_cloud/pex/grpc/server/registry.py +15 -8
- dagster_cloud/pex/grpc/server/server.py +17 -8
- dagster_cloud/secrets/__init__.py +1 -1
- dagster_cloud/serverless/__init__.py +1 -1
- dagster_cloud/serverless/io_manager.py +4 -1
- dagster_cloud/storage/compute_logs/__init__.py +3 -1
- dagster_cloud/storage/compute_logs/compute_log_manager.py +17 -13
- dagster_cloud/storage/defs_state/__init__.py +3 -0
- dagster_cloud/storage/defs_state/queries.py +15 -0
- dagster_cloud/storage/defs_state/storage.py +113 -0
- dagster_cloud/storage/event_logs/__init__.py +3 -1
- dagster_cloud/storage/event_logs/storage.py +9 -2
- dagster_cloud/storage/event_logs/utils.py +1 -3
- dagster_cloud/storage/runs/__init__.py +1 -1
- dagster_cloud/storage/runs/queries.py +15 -0
- dagster_cloud/storage/runs/storage.py +30 -3
- dagster_cloud/storage/schedules/__init__.py +1 -1
- dagster_cloud/storage/schedules/storage.py +1 -1
- dagster_cloud/util/errors.py +0 -91
- dagster_cloud/version.py +1 -1
- dagster_cloud/workspace/config_schema/__init__.py +43 -5
- dagster_cloud/workspace/docker/__init__.py +8 -7
- dagster_cloud/workspace/docker/utils.py +1 -1
- dagster_cloud/workspace/ecs/__init__.py +1 -1
- dagster_cloud/workspace/ecs/client.py +23 -18
- dagster_cloud/workspace/ecs/launcher.py +19 -5
- dagster_cloud/workspace/ecs/run_launcher.py +1 -2
- dagster_cloud/workspace/ecs/utils.py +5 -2
- dagster_cloud/workspace/kubernetes/__init__.py +1 -1
- dagster_cloud/workspace/kubernetes/launcher.py +11 -12
- dagster_cloud/workspace/kubernetes/utils.py +1 -2
- dagster_cloud/workspace/user_code_launcher/__init__.py +5 -3
- dagster_cloud/workspace/user_code_launcher/process.py +2 -3
- dagster_cloud/workspace/user_code_launcher/user_code_launcher.py +71 -34
- dagster_cloud/workspace/user_code_launcher/utils.py +7 -0
- {dagster_cloud-1.10.11.dist-info → dagster_cloud-1.12.6.dist-info}/METADATA +9 -8
- dagster_cloud-1.12.6.dist-info/RECORD +134 -0
- {dagster_cloud-1.10.11.dist-info → dagster_cloud-1.12.6.dist-info}/WHEEL +1 -1
- dagster_cloud-1.10.11.dist-info/RECORD +0 -129
- {dagster_cloud-1.10.11.dist-info → dagster_cloud-1.12.6.dist-info}/top_level.txt +0 -0
dagster_cloud/__init__.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from dagster._core.libraries import DagsterLibraryRegistry
|
|
2
2
|
|
|
3
|
-
from .instance import DagsterCloudAgentInstance as DagsterCloudAgentInstance
|
|
4
|
-
from .storage.compute_logs import CloudComputeLogManager as CloudComputeLogManager
|
|
5
|
-
from .version import __version__
|
|
3
|
+
from dagster_cloud.instance import DagsterCloudAgentInstance as DagsterCloudAgentInstance
|
|
4
|
+
from dagster_cloud.storage.compute_logs import CloudComputeLogManager as CloudComputeLogManager
|
|
5
|
+
from dagster_cloud.version import __version__
|
|
6
6
|
|
|
7
7
|
DagsterLibraryRegistry.register("dagster-cloud", __version__)
|
|
@@ -12,8 +12,7 @@ from typing import TYPE_CHECKING, Any, Optional, Union, cast
|
|
|
12
12
|
import dagster._check as check
|
|
13
13
|
from dagster import DagsterInstance
|
|
14
14
|
from dagster._core.launcher.base import LaunchRunContext
|
|
15
|
-
from dagster._core.
|
|
16
|
-
from dagster._core.remote_representation.origin import RegisteredCodeLocationOrigin
|
|
15
|
+
from dagster._core.remote_origin import CodeLocationOrigin, RegisteredCodeLocationOrigin
|
|
17
16
|
from dagster._core.utils import FuturesAwareThreadPoolExecutor
|
|
18
17
|
from dagster._grpc.client import DagsterGrpcClient
|
|
19
18
|
from dagster._grpc.types import CancelExecutionRequest
|
|
@@ -21,13 +20,27 @@ from dagster._serdes import deserialize_value, serialize_value
|
|
|
21
20
|
from dagster._time import get_current_datetime, get_current_timestamp
|
|
22
21
|
from dagster._utils.cached_method import cached_method
|
|
23
22
|
from dagster._utils.container import retrieve_containerized_utilization_metrics
|
|
24
|
-
from dagster._utils.error import
|
|
23
|
+
from dagster._utils.error import (
|
|
24
|
+
SerializableErrorInfo,
|
|
25
|
+
serializable_error_info_from_exc_info,
|
|
26
|
+
truncate_serialized_error,
|
|
27
|
+
)
|
|
25
28
|
from dagster._utils.interrupts import raise_interrupts_as
|
|
26
29
|
from dagster._utils.merger import merge_dicts
|
|
27
30
|
from dagster._utils.typed_dict import init_optional_typeddict
|
|
28
31
|
from dagster_cloud_cli.core.errors import DagsterCloudHTTPError, raise_http_error
|
|
29
32
|
from dagster_cloud_cli.core.workspace import CodeLocationDeployData
|
|
30
33
|
|
|
34
|
+
from dagster_cloud.agent.instrumentation.constants import DAGSTER_CLOUD_AGENT_METRIC_PREFIX
|
|
35
|
+
from dagster_cloud.agent.instrumentation.run_launch import extract_run_attributes
|
|
36
|
+
from dagster_cloud.agent.instrumentation.schedule import inspect_schedule_result
|
|
37
|
+
from dagster_cloud.agent.instrumentation.sensor import inspect_sensor_result
|
|
38
|
+
from dagster_cloud.agent.queries import (
|
|
39
|
+
ADD_AGENT_HEARTBEATS_MUTATION,
|
|
40
|
+
DEPLOYMENTS_QUERY,
|
|
41
|
+
GET_USER_CLOUD_REQUESTS_QUERY,
|
|
42
|
+
WORKSPACE_ENTRIES_QUERY,
|
|
43
|
+
)
|
|
31
44
|
from dagster_cloud.api.dagster_cloud_api import (
|
|
32
45
|
AgentHeartbeat,
|
|
33
46
|
AgentUtilizationMetrics,
|
|
@@ -45,27 +58,15 @@ from dagster_cloud.api.dagster_cloud_api import (
|
|
|
45
58
|
)
|
|
46
59
|
from dagster_cloud.batching import Batcher
|
|
47
60
|
from dagster_cloud.instance import DagsterCloudAgentInstance
|
|
48
|
-
from dagster_cloud.
|
|
61
|
+
from dagster_cloud.opentelemetry.observers.execution_observer import observe_execution
|
|
62
|
+
from dagster_cloud.util import SERVER_HANDLE_TAG, compressed_namedtuple_upload_file, is_isolated_run
|
|
63
|
+
from dagster_cloud.version import __version__
|
|
49
64
|
from dagster_cloud.workspace.user_code_launcher import (
|
|
50
65
|
DagsterCloudUserCodeLauncher,
|
|
51
66
|
UserCodeLauncherEntry,
|
|
52
67
|
)
|
|
53
68
|
from dagster_cloud.workspace.user_code_launcher.utils import get_instance_ref_for_user_code
|
|
54
69
|
|
|
55
|
-
from ..opentelemetry.observers.execution_observer import observe_execution
|
|
56
|
-
from ..util import SERVER_HANDLE_TAG, compressed_namedtuple_upload_file, is_isolated_run
|
|
57
|
-
from ..version import __version__
|
|
58
|
-
from .instrumentation.constants import DAGSTER_CLOUD_AGENT_METRIC_PREFIX
|
|
59
|
-
from .instrumentation.run_launch import extract_run_attributes
|
|
60
|
-
from .instrumentation.schedule import inspect_schedule_result
|
|
61
|
-
from .instrumentation.sensor import inspect_sensor_result
|
|
62
|
-
from .queries import (
|
|
63
|
-
ADD_AGENT_HEARTBEATS_MUTATION,
|
|
64
|
-
DEPLOYMENTS_QUERY,
|
|
65
|
-
GET_USER_CLOUD_REQUESTS_QUERY,
|
|
66
|
-
WORKSPACE_ENTRIES_QUERY,
|
|
67
|
-
)
|
|
68
|
-
|
|
69
70
|
if TYPE_CHECKING:
|
|
70
71
|
import datetime
|
|
71
72
|
|
|
@@ -162,6 +163,8 @@ class DagsterCloudAgent:
|
|
|
162
163
|
|
|
163
164
|
self._last_liveness_check_time = None
|
|
164
165
|
|
|
166
|
+
self._warned_about_long_in_progress_reconcile = False
|
|
167
|
+
|
|
165
168
|
def __enter__(self):
|
|
166
169
|
return self
|
|
167
170
|
|
|
@@ -305,6 +308,19 @@ class DagsterCloudAgent:
|
|
|
305
308
|
|
|
306
309
|
self._check_initial_deployment_names()
|
|
307
310
|
|
|
311
|
+
serving = []
|
|
312
|
+
queues = list(filter(None, self._instance.agent_queues_config.queues))
|
|
313
|
+
if queues:
|
|
314
|
+
serving.append(f"queues{queues}")
|
|
315
|
+
if self._instance.deployment_names:
|
|
316
|
+
serving.append(f"deployments{self._instance.deployment_names}")
|
|
317
|
+
if self._instance.include_all_serverless_deployments:
|
|
318
|
+
serving.append("all serverless deployments")
|
|
319
|
+
if self._instance.includes_branch_deployments:
|
|
320
|
+
serving.append("branch deployments")
|
|
321
|
+
|
|
322
|
+
self._logger.info(f"Agent is serving: {', '.join(serving)}")
|
|
323
|
+
|
|
308
324
|
self._check_update_workspace(
|
|
309
325
|
user_code_launcher,
|
|
310
326
|
upload_all=user_code_launcher.upload_snapshots_on_startup,
|
|
@@ -352,6 +368,8 @@ class DagsterCloudAgent:
|
|
|
352
368
|
except Exception:
|
|
353
369
|
self._logger.exception("Failed to add heartbeat")
|
|
354
370
|
|
|
371
|
+
self._check_for_long_running_reconcile(user_code_launcher)
|
|
372
|
+
|
|
355
373
|
# Check for any received interrupts
|
|
356
374
|
with raise_interrupts_as(KeyboardInterrupt):
|
|
357
375
|
pass
|
|
@@ -394,6 +412,26 @@ class DagsterCloudAgent:
|
|
|
394
412
|
self._logger.error(f"Failed to write liveness sentinel and disabling it: {e}")
|
|
395
413
|
self._last_liveness_check_time = False
|
|
396
414
|
|
|
415
|
+
def _check_for_long_running_reconcile(self, user_code_launcher):
|
|
416
|
+
"""Detect from the main thread if the background reconcile thread is running behind or has gotten stuck."""
|
|
417
|
+
in_progress_reconcile_start_time = user_code_launcher.in_progress_reconcile_start_time
|
|
418
|
+
|
|
419
|
+
reconcile_start_time_warning = int(
|
|
420
|
+
os.getenv("DAGSTER_CLOUD_AGENT_RECONCILE_START_TIME_WARNING", "3600")
|
|
421
|
+
)
|
|
422
|
+
|
|
423
|
+
if (
|
|
424
|
+
in_progress_reconcile_start_time is not None
|
|
425
|
+
and (time.time() - in_progress_reconcile_start_time) >= reconcile_start_time_warning
|
|
426
|
+
):
|
|
427
|
+
if not self._warned_about_long_in_progress_reconcile:
|
|
428
|
+
self._logger.warning(
|
|
429
|
+
f"Agent has been redeploying code servers for more than {reconcile_start_time_warning} seconds. This may indicate the background thread that performs the redeploys is stuck."
|
|
430
|
+
)
|
|
431
|
+
self._warned_about_long_in_progress_reconcile = True
|
|
432
|
+
else:
|
|
433
|
+
self._warned_about_long_in_progress_reconcile = False
|
|
434
|
+
|
|
397
435
|
def _check_update_workspace(self, user_code_launcher, upload_all):
|
|
398
436
|
curr_time = get_current_datetime()
|
|
399
437
|
|
|
@@ -1044,10 +1082,16 @@ class DagsterCloudAgent:
|
|
|
1044
1082
|
client.cancel_execution(CancelExecutionRequest(run_id=run.run_id))
|
|
1045
1083
|
|
|
1046
1084
|
return DagsterCloudApiSuccess()
|
|
1047
|
-
|
|
1085
|
+
elif api_name in (
|
|
1086
|
+
DagsterCloudApi.CHECK_STEP_HEALTH,
|
|
1087
|
+
DagsterCloudApi.TERMINATE_STEP,
|
|
1088
|
+
DagsterCloudApi.LAUNCH_STEP,
|
|
1089
|
+
DagsterCloudApi.CHECK_RUN_HEALTH,
|
|
1090
|
+
DagsterCloudApi.LOAD_REPOSITORIES,
|
|
1091
|
+
):
|
|
1092
|
+
check.failed(f"Unexpected deprecated request type {api_name}")
|
|
1048
1093
|
else:
|
|
1049
1094
|
check.assert_never(api_name)
|
|
1050
|
-
raise Exception(f"Unexpected dagster cloud api call {api_name}")
|
|
1051
1095
|
|
|
1052
1096
|
def _process_api_request(
|
|
1053
1097
|
self,
|
|
@@ -19,5 +19,5 @@ def extract_run_attributes(deployment_name, run: DagsterRun) -> dict[str, str]:
|
|
|
19
19
|
attributes["location"] = run.remote_job_origin.location_name or "unknown"
|
|
20
20
|
except Exception as e: # pylint: disable=broad-except
|
|
21
21
|
logging.warning("Failed to extract additional run attributes", exc_info=e)
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
|
|
23
|
+
return attributes
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from dagster import deserialize_value
|
|
2
2
|
from dagster._core.definitions.schedule_definition import ScheduleExecutionData
|
|
3
|
-
from dagster._core.remote_representation import ScheduleExecutionErrorSnap
|
|
3
|
+
from dagster._core.remote_representation.external_data import ScheduleExecutionErrorSnap
|
|
4
4
|
|
|
5
5
|
from dagster_cloud.opentelemetry.observers.execution_observer import ExecutionObserverInstruments
|
|
6
6
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from dagster import deserialize_value
|
|
2
2
|
from dagster._core.definitions.sensor_definition import SensorExecutionData
|
|
3
|
-
from dagster._core.remote_representation import SensorExecutionErrorSnap
|
|
3
|
+
from dagster._core.remote_representation.external_data import SensorExecutionErrorSnap
|
|
4
4
|
|
|
5
5
|
from dagster_cloud.opentelemetry.observers.execution_observer import ExecutionObserverInstruments
|
|
6
6
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
from .defs import (
|
|
1
|
+
from dagster_cloud.anomaly_detection.defs import (
|
|
2
2
|
build_anomaly_detection_freshness_checks as build_anomaly_detection_freshness_checks,
|
|
3
3
|
)
|
|
4
|
-
from .types import (
|
|
4
|
+
from dagster_cloud.anomaly_detection.types import (
|
|
5
5
|
AnomalyDetectionModelParams as AnomalyDetectionModelParams,
|
|
6
6
|
BetaFreshnessAnomalyDetectionParams as BetaFreshnessAnomalyDetectionParams,
|
|
7
7
|
)
|
|
@@ -7,24 +7,27 @@ from dagster import (
|
|
|
7
7
|
MetadataValue,
|
|
8
8
|
_check as check,
|
|
9
9
|
)
|
|
10
|
-
from dagster._core.definitions.asset_check_factories.freshness_checks.last_update import (
|
|
10
|
+
from dagster._core.definitions.asset_checks.asset_check_factories.freshness_checks.last_update import (
|
|
11
11
|
construct_description,
|
|
12
12
|
)
|
|
13
|
-
from dagster._core.definitions.asset_check_factories.utils import (
|
|
13
|
+
from dagster._core.definitions.asset_checks.asset_check_factories.utils import (
|
|
14
14
|
FRESH_UNTIL_METADATA_KEY,
|
|
15
15
|
LAST_UPDATED_TIMESTAMP_METADATA_KEY,
|
|
16
16
|
LOWER_BOUND_TIMESTAMP_METADATA_KEY,
|
|
17
17
|
assets_to_keys,
|
|
18
18
|
)
|
|
19
|
-
from dagster._core.definitions.asset_check_result import AssetCheckResult
|
|
20
|
-
from dagster._core.definitions.asset_check_spec import (
|
|
19
|
+
from dagster._core.definitions.asset_checks.asset_check_result import AssetCheckResult
|
|
20
|
+
from dagster._core.definitions.asset_checks.asset_check_spec import (
|
|
21
21
|
AssetCheckKey,
|
|
22
22
|
AssetCheckSeverity,
|
|
23
23
|
AssetCheckSpec,
|
|
24
24
|
)
|
|
25
|
-
from dagster._core.definitions.asset_checks import AssetChecksDefinition
|
|
25
|
+
from dagster._core.definitions.asset_checks.asset_checks_definition import AssetChecksDefinition
|
|
26
26
|
from dagster._core.definitions.asset_key import AssetKey
|
|
27
|
-
from dagster._core.definitions.assets import
|
|
27
|
+
from dagster._core.definitions.assets.definition.assets_definition import (
|
|
28
|
+
AssetsDefinition,
|
|
29
|
+
unique_id_from_asset_and_check_keys,
|
|
30
|
+
)
|
|
28
31
|
from dagster._core.definitions.decorators.asset_check_decorator import multi_asset_check
|
|
29
32
|
from dagster._core.definitions.events import CoercibleToAssetKey
|
|
30
33
|
from dagster._core.definitions.source_asset import SourceAsset
|
|
@@ -36,8 +39,8 @@ from dagster_cloud_cli.core.graphql_client import (
|
|
|
36
39
|
create_cloud_webserver_client,
|
|
37
40
|
)
|
|
38
41
|
|
|
39
|
-
from .mutation import ANOMALY_DETECTION_INFERENCE_MUTATION
|
|
40
|
-
from .types import (
|
|
42
|
+
from dagster_cloud.anomaly_detection.mutation import ANOMALY_DETECTION_INFERENCE_MUTATION
|
|
43
|
+
from dagster_cloud.anomaly_detection.types import (
|
|
41
44
|
AnomalyDetectionModelParams,
|
|
42
45
|
BetaFreshnessAnomalyDetectionParams,
|
|
43
46
|
FreshnessAnomalyDetectionResult,
|
|
@@ -7,7 +7,8 @@ from typing import Any, Optional, TypedDict, Union
|
|
|
7
7
|
from dagster._core.code_pointer import CodePointer
|
|
8
8
|
from dagster._core.definitions.selector import JobSelector
|
|
9
9
|
from dagster._core.events.log import EventLogEntry
|
|
10
|
-
from dagster._core.
|
|
10
|
+
from dagster._core.remote_origin import CodeLocationOrigin
|
|
11
|
+
from dagster._core.remote_representation.external_data import RepositorySnap
|
|
11
12
|
from dagster._core.storage.dagster_run import DagsterRun
|
|
12
13
|
from dagster._core.utils import RequestUtilizationMetrics
|
|
13
14
|
from dagster._record import IHaveNew, copy, record, record_custom
|
|
@@ -84,7 +85,6 @@ class DagsterCloudUploadWorkspaceResponse:
|
|
|
84
85
|
@whitelist_for_serdes
|
|
85
86
|
class DagsterCloudApi(Enum):
|
|
86
87
|
CHECK_FOR_WORKSPACE_UPDATES = "CHECK_FOR_WORKSPACE_UPDATES"
|
|
87
|
-
LOAD_REPOSITORIES = "LOAD_REPOSITORIES"
|
|
88
88
|
GET_EXTERNAL_EXECUTION_PLAN = "GET_EXTERNAL_EXECUTION_PLAN"
|
|
89
89
|
GET_SUBSET_EXTERNAL_PIPELINE_RESULT = "GET_SUBSET_EXTERNAL_PIPELINE_RESULT"
|
|
90
90
|
GET_EXTERNAL_PARTITION_CONFIG = "GET_EXTERNAL_PARTITION_CONFIG"
|
|
@@ -98,13 +98,15 @@ class DagsterCloudApi(Enum):
|
|
|
98
98
|
GET_EXTERNAL_NOTEBOOK_DATA = "GET_EXTERNAL_NOTEBOOK_DATA"
|
|
99
99
|
|
|
100
100
|
LAUNCH_RUN = "LAUNCH_RUN"
|
|
101
|
-
CHECK_RUN_HEALTH = "CHECK_RUN_HEALTH" # deprecated, agents now surface this in heartbeats
|
|
102
101
|
TERMINATE_RUN = "TERMINATE_RUN"
|
|
102
|
+
|
|
103
|
+
PING_LOCATION = "PING_LOCATION" # Signal that a location is in use and should keep servers up
|
|
104
|
+
|
|
105
|
+
CHECK_RUN_HEALTH = "CHECK_RUN_HEALTH" # deprecated, agents now surface this in heartbeats
|
|
103
106
|
LAUNCH_STEP = "LAUNCH_STEP" # deprecated with cloud executor
|
|
104
107
|
CHECK_STEP_HEALTH = "CHECK_STEP_HEALTH" # deprecated with cloud executor
|
|
105
108
|
TERMINATE_STEP = "TERMINATE_STEP" # deprecated with cloud executor
|
|
106
|
-
|
|
107
|
-
PING_LOCATION = "PING_LOCATION" # Signal that a location is in use and should keep servers up
|
|
109
|
+
LOAD_REPOSITORIES = "LOAD_REPOSITORIES" # deprecated
|
|
108
110
|
|
|
109
111
|
def __structlog__(self):
|
|
110
112
|
return self.name
|
dagster_cloud/auth/constants.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import uuid
|
|
1
2
|
from typing import Optional
|
|
2
3
|
|
|
3
4
|
from dagster._core.errors import DagsterInvariantViolationError
|
|
@@ -19,12 +20,27 @@ def get_organization_public_id_from_api_token(api_token: str) -> Optional[str]:
|
|
|
19
20
|
return split_token[2]
|
|
20
21
|
|
|
21
22
|
|
|
22
|
-
def
|
|
23
|
+
def decode_region_from_uuid(regional_token: str) -> Optional[str]:
|
|
24
|
+
try:
|
|
25
|
+
regional_uuid = uuid.UUID(regional_token)
|
|
26
|
+
except ValueError:
|
|
27
|
+
# if it's not an actual uuid, we can't decode region
|
|
28
|
+
return None
|
|
29
|
+
|
|
30
|
+
# custom uuids contain region subdomains in the first 2 bytes
|
|
31
|
+
if regional_uuid.version != 8 or regional_uuid.variant != uuid.RFC_4122:
|
|
32
|
+
return None
|
|
33
|
+
|
|
34
|
+
uuid_bytes = regional_uuid.bytes
|
|
35
|
+
return uuid_bytes[:2].decode("ascii")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def decode_agent_token(agent_token: str) -> tuple[Optional[str], Optional[str]]:
|
|
23
39
|
split_token = agent_token.split(":")
|
|
24
40
|
|
|
25
41
|
# Legacy agent token format - organization must be specified in dagster.yaml
|
|
26
42
|
if len(split_token) == 1:
|
|
27
|
-
return None
|
|
43
|
+
return None, None
|
|
28
44
|
|
|
29
45
|
token_type, *token = split_token
|
|
30
46
|
|
|
@@ -35,6 +51,6 @@ def get_organization_name_from_agent_token(agent_token: str) -> Optional[str]:
|
|
|
35
51
|
"Generate a new agent token in Dagster Cloud."
|
|
36
52
|
)
|
|
37
53
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
return organization
|
|
54
|
+
# token format: agent:<org>:<uuid>
|
|
55
|
+
organization, uuid_str = token
|
|
56
|
+
return organization, decode_region_from_uuid(uuid_str)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
from .batcher import Batcher as Batcher
|
|
1
|
+
from dagster_cloud.batching.batcher import Batcher as Batcher
|
|
@@ -1,15 +1,19 @@
|
|
|
1
1
|
import sys
|
|
2
2
|
from typing import Any
|
|
3
3
|
|
|
4
|
-
from .snowflake.dbt_wrapper import
|
|
5
|
-
|
|
4
|
+
from dagster_cloud.dagster_insights.snowflake.dbt_wrapper import (
|
|
5
|
+
dbt_with_snowflake_insights as dbt_with_snowflake_insights,
|
|
6
|
+
)
|
|
7
|
+
from dagster_cloud.dagster_insights.snowflake.definitions import (
|
|
6
8
|
create_snowflake_insights_asset_and_schedule as create_snowflake_insights_asset_and_schedule,
|
|
7
9
|
)
|
|
8
|
-
from .snowflake.snowflake_utils import
|
|
10
|
+
from dagster_cloud.dagster_insights.snowflake.snowflake_utils import (
|
|
11
|
+
meter_snowflake_query as meter_snowflake_query,
|
|
12
|
+
)
|
|
9
13
|
|
|
10
14
|
dagster_snowflake_req_imports = {"InsightsSnowflakeResource"}
|
|
11
15
|
try:
|
|
12
|
-
from .snowflake.insights_snowflake_resource import (
|
|
16
|
+
from dagster_cloud.dagster_insights.snowflake.insights_snowflake_resource import (
|
|
13
17
|
InsightsSnowflakeResource as InsightsSnowflakeResource,
|
|
14
18
|
)
|
|
15
19
|
except ImportError:
|
|
@@ -17,8 +21,10 @@ except ImportError:
|
|
|
17
21
|
|
|
18
22
|
dagster_bigquery_req_imports = {"InsightsBigQueryResource", "dbt_with_bigquery_insights"}
|
|
19
23
|
try:
|
|
20
|
-
from .bigquery.dbt_wrapper import
|
|
21
|
-
|
|
24
|
+
from dagster_cloud.dagster_insights.bigquery.dbt_wrapper import (
|
|
25
|
+
dbt_with_bigquery_insights as dbt_with_bigquery_insights,
|
|
26
|
+
)
|
|
27
|
+
from dagster_cloud.dagster_insights.bigquery.insights_bigquery_resource import (
|
|
22
28
|
InsightsBigQueryResource as InsightsBigQueryResource,
|
|
23
29
|
)
|
|
24
30
|
except ImportError:
|
|
@@ -18,8 +18,14 @@ from dagster_dbt import DbtCliInvocation
|
|
|
18
18
|
from dagster_dbt.version import __version__ as dagster_dbt_version
|
|
19
19
|
from packaging import version
|
|
20
20
|
|
|
21
|
-
from
|
|
22
|
-
|
|
21
|
+
from dagster_cloud.dagster_insights.bigquery.bigquery_utils import (
|
|
22
|
+
build_bigquery_cost_metadata,
|
|
23
|
+
marker_asset_key_for_job,
|
|
24
|
+
)
|
|
25
|
+
from dagster_cloud.dagster_insights.insights_utils import (
|
|
26
|
+
extract_asset_info_from_event,
|
|
27
|
+
handle_raise_on_error,
|
|
28
|
+
)
|
|
23
29
|
|
|
24
30
|
if TYPE_CHECKING:
|
|
25
31
|
from dbt.adapters.base.impl import BaseAdapter
|
|
@@ -7,10 +7,12 @@ from dagster_gcp import BigQueryResource
|
|
|
7
7
|
from dagster_gcp.bigquery.utils import setup_gcp_creds
|
|
8
8
|
from google.cloud import bigquery
|
|
9
9
|
|
|
10
|
+
from dagster_cloud.dagster_insights.bigquery.bigquery_utils import (
|
|
11
|
+
build_bigquery_cost_metadata,
|
|
12
|
+
marker_asset_key_for_job,
|
|
13
|
+
)
|
|
10
14
|
from dagster_cloud.dagster_insights.insights_utils import get_current_context_and_asset_key
|
|
11
15
|
|
|
12
|
-
from .bigquery_utils import build_bigquery_cost_metadata, marker_asset_key_for_job
|
|
13
|
-
|
|
14
16
|
OUTPUT_NON_ASSET_SIGIL = "__bigquery_query_metadata_"
|
|
15
17
|
|
|
16
18
|
|
|
@@ -33,7 +33,7 @@ def get_current_context_and_asset_key() -> tuple[
|
|
|
33
33
|
def get_asset_key_for_output(
|
|
34
34
|
context: Union[OpExecutionContext, AssetExecutionContext], output_name: str
|
|
35
35
|
) -> Optional[AssetKey]:
|
|
36
|
-
asset_key = context.job_def.asset_layer.
|
|
36
|
+
asset_key = context.job_def.asset_layer.get_asset_key_for_node_output(
|
|
37
37
|
node_handle=context.op_handle, output_name=output_name
|
|
38
38
|
)
|
|
39
39
|
if asset_key is None:
|
|
@@ -28,7 +28,7 @@ def get_url_and_token_from_instance(instance: DagsterInstance) -> tuple[str, str
|
|
|
28
28
|
if not isinstance(instance, DagsterCloudAgentInstance):
|
|
29
29
|
raise RuntimeError("This asset only functions in a running Dagster Cloud instance")
|
|
30
30
|
|
|
31
|
-
return f"{instance.dagit_url}graphql", instance.dagster_cloud_agent_token
|
|
31
|
+
return f"{instance.dagit_url}graphql", instance.dagster_cloud_agent_token
|
|
32
32
|
|
|
33
33
|
|
|
34
34
|
def get_insights_upload_request_params(
|
|
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Optional
|
|
|
6
6
|
|
|
7
7
|
from dagster import AssetKey, AssetsDefinition, ScheduleDefinition
|
|
8
8
|
|
|
9
|
-
from .snowflake_utils import OPAQUE_ID_SQL_SIGIL
|
|
9
|
+
from dagster_cloud.dagster_insights.snowflake.snowflake_utils import OPAQUE_ID_SQL_SIGIL
|
|
10
10
|
|
|
11
11
|
if TYPE_CHECKING:
|
|
12
12
|
from dagster_snowflake import SnowflakeConnection
|
|
@@ -78,10 +78,6 @@ HAVING ARRAY_SIZE(opaque_ids) > 0
|
|
|
78
78
|
|
|
79
79
|
costs: list[tuple[str, float, str]] = []
|
|
80
80
|
|
|
81
|
-
print(
|
|
82
|
-
f"{len(results) if results else 0} annotated queries returned from snowflake query_history"
|
|
83
|
-
)
|
|
84
|
-
|
|
85
81
|
if not results:
|
|
86
82
|
return []
|
|
87
83
|
|
|
@@ -94,8 +90,4 @@ HAVING ARRAY_SIZE(opaque_ids) > 0
|
|
|
94
90
|
for opaque_id in opaque_ids:
|
|
95
91
|
costs.append((opaque_id, float(cost), query_id))
|
|
96
92
|
|
|
97
|
-
print(
|
|
98
|
-
f"Reported costs for {len(costs)} of {total} asset materializations found in the"
|
|
99
|
-
" query_history."
|
|
100
|
-
)
|
|
101
93
|
return costs
|
|
@@ -13,8 +13,15 @@ from dagster import (
|
|
|
13
13
|
Output,
|
|
14
14
|
)
|
|
15
15
|
|
|
16
|
-
from
|
|
17
|
-
|
|
16
|
+
from dagster_cloud.dagster_insights.insights_utils import (
|
|
17
|
+
extract_asset_info_from_event,
|
|
18
|
+
handle_raise_on_error,
|
|
19
|
+
)
|
|
20
|
+
from dagster_cloud.dagster_insights.snowflake.snowflake_utils import (
|
|
21
|
+
OPAQUE_ID_SQL_SIGIL,
|
|
22
|
+
build_opaque_id_metadata,
|
|
23
|
+
marker_asset_key_for_job,
|
|
24
|
+
)
|
|
18
25
|
|
|
19
26
|
if TYPE_CHECKING:
|
|
20
27
|
from dagster_dbt import DbtCliInvocation
|
|
@@ -2,7 +2,6 @@ import warnings
|
|
|
2
2
|
from collections.abc import Sequence
|
|
3
3
|
from dataclasses import dataclass
|
|
4
4
|
from datetime import date, datetime, timedelta, timezone
|
|
5
|
-
from pprint import pprint
|
|
6
5
|
from typing import TYPE_CHECKING, Optional, Union
|
|
7
6
|
|
|
8
7
|
from dagster import (
|
|
@@ -27,8 +26,10 @@ from dagster._core.storage.tags import (
|
|
|
27
26
|
ASSET_PARTITION_RANGE_START_TAG,
|
|
28
27
|
)
|
|
29
28
|
|
|
30
|
-
from
|
|
31
|
-
from .dagster_snowflake_insights import
|
|
29
|
+
from dagster_cloud.dagster_insights.metrics_utils import put_cost_information
|
|
30
|
+
from dagster_cloud.dagster_insights.snowflake.dagster_snowflake_insights import (
|
|
31
|
+
get_cost_data_for_hour,
|
|
32
|
+
)
|
|
32
33
|
|
|
33
34
|
if TYPE_CHECKING:
|
|
34
35
|
from dagster_snowflake import SnowflakeConnection
|
|
@@ -154,7 +155,7 @@ def create_snowflake_insights_asset_and_schedule(
|
|
|
154
155
|
)
|
|
155
156
|
|
|
156
157
|
if dry_run:
|
|
157
|
-
|
|
158
|
+
pass
|
|
158
159
|
else:
|
|
159
160
|
context.log.info(
|
|
160
161
|
f"Submitting cost information for {len(costs)} queries to Dagster Insights"
|
|
@@ -20,8 +20,7 @@ from dagster_snowflake import SnowflakeConnection, SnowflakeResource
|
|
|
20
20
|
from snowflake.connector.cursor import SnowflakeCursor
|
|
21
21
|
|
|
22
22
|
from dagster_cloud.dagster_insights.insights_utils import get_current_context_and_asset_key
|
|
23
|
-
|
|
24
|
-
from .snowflake_utils import meter_snowflake_query
|
|
23
|
+
from dagster_cloud.dagster_insights.snowflake.snowflake_utils import meter_snowflake_query
|
|
25
24
|
|
|
26
25
|
|
|
27
26
|
def get_current_context_and_asset_key_or_warn() -> tuple[
|
|
File without changes
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
from collections.abc import Sequence
|
|
3
|
+
from typing import AbstractSet # noqa: UP035
|
|
4
|
+
|
|
5
|
+
import dagster._check as check
|
|
6
|
+
from dagster._record import record
|
|
7
|
+
from dagster_shared.serdes import whitelist_for_serdes
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@whitelist_for_serdes
|
|
11
|
+
@record
|
|
12
|
+
class JobSelection(ABC):
|
|
13
|
+
code_location_name: str
|
|
14
|
+
repository_name: str
|
|
15
|
+
|
|
16
|
+
@abstractmethod
|
|
17
|
+
def resolve_job_names(self) -> AbstractSet[str]:
|
|
18
|
+
raise NotImplementedError()
|
|
19
|
+
|
|
20
|
+
@staticmethod
|
|
21
|
+
def names(
|
|
22
|
+
names: Sequence[str], code_location_name: str, repository_name: str
|
|
23
|
+
) -> "NamesJobSelection":
|
|
24
|
+
check.invariant(len(names) == 1, "Only one job name is supported at this time")
|
|
25
|
+
return NamesJobSelection(
|
|
26
|
+
job_names=names, code_location_name=code_location_name, repository_name=repository_name
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@whitelist_for_serdes
|
|
31
|
+
@record
|
|
32
|
+
class NamesJobSelection(JobSelection):
|
|
33
|
+
job_names: Sequence[str]
|
|
34
|
+
|
|
35
|
+
def resolve_job_names(self) -> AbstractSet[str]:
|
|
36
|
+
return set(self.job_names)
|