dagster-cloud 1.10.11__py3-none-any.whl → 1.12.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. dagster_cloud/__init__.py +3 -3
  2. dagster_cloud/agent/dagster_cloud_agent.py +64 -20
  3. dagster_cloud/agent/instrumentation/run_launch.py +2 -2
  4. dagster_cloud/agent/instrumentation/schedule.py +1 -1
  5. dagster_cloud/agent/instrumentation/sensor.py +1 -1
  6. dagster_cloud/anomaly_detection/__init__.py +2 -2
  7. dagster_cloud/anomaly_detection/defs.py +11 -8
  8. dagster_cloud/api/dagster_cloud_api.py +7 -5
  9. dagster_cloud/auth/constants.py +21 -5
  10. dagster_cloud/batching/__init__.py +1 -1
  11. dagster_cloud/dagster_insights/__init__.py +12 -6
  12. dagster_cloud/dagster_insights/bigquery/dbt_wrapper.py +8 -2
  13. dagster_cloud/dagster_insights/bigquery/insights_bigquery_resource.py +4 -2
  14. dagster_cloud/dagster_insights/insights_utils.py +1 -1
  15. dagster_cloud/dagster_insights/metrics_utils.py +1 -1
  16. dagster_cloud/dagster_insights/snowflake/dagster_snowflake_insights.py +1 -9
  17. dagster_cloud/dagster_insights/snowflake/dbt_wrapper.py +9 -2
  18. dagster_cloud/dagster_insights/snowflake/definitions.py +5 -4
  19. dagster_cloud/dagster_insights/snowflake/insights_snowflake_resource.py +1 -2
  20. dagster_cloud/definitions/__init__.py +0 -0
  21. dagster_cloud/definitions/job_selection.py +36 -0
  22. dagster_cloud/execution/utils/process.py +1 -1
  23. dagster_cloud/instance/__init__.py +81 -42
  24. dagster_cloud/metadata/source_code.py +3 -1
  25. dagster_cloud/opentelemetry/config/exporter.py +1 -1
  26. dagster_cloud/opentelemetry/controller.py +1 -1
  27. dagster_cloud/opentelemetry/observers/dagster_exception_handler.py +1 -1
  28. dagster_cloud/opentelemetry/observers/execution_observer.py +4 -2
  29. dagster_cloud/pex/grpc/__init__.py +2 -2
  30. dagster_cloud/pex/grpc/client.py +2 -2
  31. dagster_cloud/pex/grpc/server/__init__.py +2 -2
  32. dagster_cloud/pex/grpc/server/cli/__init__.py +2 -2
  33. dagster_cloud/pex/grpc/server/manager.py +5 -4
  34. dagster_cloud/pex/grpc/server/registry.py +15 -8
  35. dagster_cloud/pex/grpc/server/server.py +17 -8
  36. dagster_cloud/secrets/__init__.py +1 -1
  37. dagster_cloud/serverless/__init__.py +1 -1
  38. dagster_cloud/serverless/io_manager.py +4 -1
  39. dagster_cloud/storage/compute_logs/__init__.py +3 -1
  40. dagster_cloud/storage/compute_logs/compute_log_manager.py +17 -13
  41. dagster_cloud/storage/defs_state/__init__.py +3 -0
  42. dagster_cloud/storage/defs_state/queries.py +15 -0
  43. dagster_cloud/storage/defs_state/storage.py +113 -0
  44. dagster_cloud/storage/event_logs/__init__.py +3 -1
  45. dagster_cloud/storage/event_logs/storage.py +9 -2
  46. dagster_cloud/storage/event_logs/utils.py +1 -3
  47. dagster_cloud/storage/runs/__init__.py +1 -1
  48. dagster_cloud/storage/runs/queries.py +15 -0
  49. dagster_cloud/storage/runs/storage.py +30 -3
  50. dagster_cloud/storage/schedules/__init__.py +1 -1
  51. dagster_cloud/storage/schedules/storage.py +1 -1
  52. dagster_cloud/util/errors.py +0 -91
  53. dagster_cloud/version.py +1 -1
  54. dagster_cloud/workspace/config_schema/__init__.py +43 -5
  55. dagster_cloud/workspace/docker/__init__.py +8 -7
  56. dagster_cloud/workspace/docker/utils.py +1 -1
  57. dagster_cloud/workspace/ecs/__init__.py +1 -1
  58. dagster_cloud/workspace/ecs/client.py +23 -18
  59. dagster_cloud/workspace/ecs/launcher.py +19 -5
  60. dagster_cloud/workspace/ecs/run_launcher.py +1 -2
  61. dagster_cloud/workspace/ecs/utils.py +5 -2
  62. dagster_cloud/workspace/kubernetes/__init__.py +1 -1
  63. dagster_cloud/workspace/kubernetes/launcher.py +11 -12
  64. dagster_cloud/workspace/kubernetes/utils.py +1 -2
  65. dagster_cloud/workspace/user_code_launcher/__init__.py +5 -3
  66. dagster_cloud/workspace/user_code_launcher/process.py +2 -3
  67. dagster_cloud/workspace/user_code_launcher/user_code_launcher.py +71 -34
  68. dagster_cloud/workspace/user_code_launcher/utils.py +7 -0
  69. {dagster_cloud-1.10.11.dist-info → dagster_cloud-1.12.6.dist-info}/METADATA +9 -8
  70. dagster_cloud-1.12.6.dist-info/RECORD +134 -0
  71. {dagster_cloud-1.10.11.dist-info → dagster_cloud-1.12.6.dist-info}/WHEEL +1 -1
  72. dagster_cloud-1.10.11.dist-info/RECORD +0 -129
  73. {dagster_cloud-1.10.11.dist-info → dagster_cloud-1.12.6.dist-info}/top_level.txt +0 -0
dagster_cloud/__init__.py CHANGED
@@ -1,7 +1,7 @@
1
1
  from dagster._core.libraries import DagsterLibraryRegistry
2
2
 
3
- from .instance import DagsterCloudAgentInstance as DagsterCloudAgentInstance
4
- from .storage.compute_logs import CloudComputeLogManager as CloudComputeLogManager
5
- from .version import __version__
3
+ from dagster_cloud.instance import DagsterCloudAgentInstance as DagsterCloudAgentInstance
4
+ from dagster_cloud.storage.compute_logs import CloudComputeLogManager as CloudComputeLogManager
5
+ from dagster_cloud.version import __version__
6
6
 
7
7
  DagsterLibraryRegistry.register("dagster-cloud", __version__)
@@ -12,8 +12,7 @@ from typing import TYPE_CHECKING, Any, Optional, Union, cast
12
12
  import dagster._check as check
13
13
  from dagster import DagsterInstance
14
14
  from dagster._core.launcher.base import LaunchRunContext
15
- from dagster._core.remote_representation import CodeLocationOrigin
16
- from dagster._core.remote_representation.origin import RegisteredCodeLocationOrigin
15
+ from dagster._core.remote_origin import CodeLocationOrigin, RegisteredCodeLocationOrigin
17
16
  from dagster._core.utils import FuturesAwareThreadPoolExecutor
18
17
  from dagster._grpc.client import DagsterGrpcClient
19
18
  from dagster._grpc.types import CancelExecutionRequest
@@ -21,13 +20,27 @@ from dagster._serdes import deserialize_value, serialize_value
21
20
  from dagster._time import get_current_datetime, get_current_timestamp
22
21
  from dagster._utils.cached_method import cached_method
23
22
  from dagster._utils.container import retrieve_containerized_utilization_metrics
24
- from dagster._utils.error import SerializableErrorInfo, serializable_error_info_from_exc_info
23
+ from dagster._utils.error import (
24
+ SerializableErrorInfo,
25
+ serializable_error_info_from_exc_info,
26
+ truncate_serialized_error,
27
+ )
25
28
  from dagster._utils.interrupts import raise_interrupts_as
26
29
  from dagster._utils.merger import merge_dicts
27
30
  from dagster._utils.typed_dict import init_optional_typeddict
28
31
  from dagster_cloud_cli.core.errors import DagsterCloudHTTPError, raise_http_error
29
32
  from dagster_cloud_cli.core.workspace import CodeLocationDeployData
30
33
 
34
+ from dagster_cloud.agent.instrumentation.constants import DAGSTER_CLOUD_AGENT_METRIC_PREFIX
35
+ from dagster_cloud.agent.instrumentation.run_launch import extract_run_attributes
36
+ from dagster_cloud.agent.instrumentation.schedule import inspect_schedule_result
37
+ from dagster_cloud.agent.instrumentation.sensor import inspect_sensor_result
38
+ from dagster_cloud.agent.queries import (
39
+ ADD_AGENT_HEARTBEATS_MUTATION,
40
+ DEPLOYMENTS_QUERY,
41
+ GET_USER_CLOUD_REQUESTS_QUERY,
42
+ WORKSPACE_ENTRIES_QUERY,
43
+ )
31
44
  from dagster_cloud.api.dagster_cloud_api import (
32
45
  AgentHeartbeat,
33
46
  AgentUtilizationMetrics,
@@ -45,27 +58,15 @@ from dagster_cloud.api.dagster_cloud_api import (
45
58
  )
46
59
  from dagster_cloud.batching import Batcher
47
60
  from dagster_cloud.instance import DagsterCloudAgentInstance
48
- from dagster_cloud.util.errors import truncate_serialized_error
61
+ from dagster_cloud.opentelemetry.observers.execution_observer import observe_execution
62
+ from dagster_cloud.util import SERVER_HANDLE_TAG, compressed_namedtuple_upload_file, is_isolated_run
63
+ from dagster_cloud.version import __version__
49
64
  from dagster_cloud.workspace.user_code_launcher import (
50
65
  DagsterCloudUserCodeLauncher,
51
66
  UserCodeLauncherEntry,
52
67
  )
53
68
  from dagster_cloud.workspace.user_code_launcher.utils import get_instance_ref_for_user_code
54
69
 
55
- from ..opentelemetry.observers.execution_observer import observe_execution
56
- from ..util import SERVER_HANDLE_TAG, compressed_namedtuple_upload_file, is_isolated_run
57
- from ..version import __version__
58
- from .instrumentation.constants import DAGSTER_CLOUD_AGENT_METRIC_PREFIX
59
- from .instrumentation.run_launch import extract_run_attributes
60
- from .instrumentation.schedule import inspect_schedule_result
61
- from .instrumentation.sensor import inspect_sensor_result
62
- from .queries import (
63
- ADD_AGENT_HEARTBEATS_MUTATION,
64
- DEPLOYMENTS_QUERY,
65
- GET_USER_CLOUD_REQUESTS_QUERY,
66
- WORKSPACE_ENTRIES_QUERY,
67
- )
68
-
69
70
  if TYPE_CHECKING:
70
71
  import datetime
71
72
 
@@ -162,6 +163,8 @@ class DagsterCloudAgent:
162
163
 
163
164
  self._last_liveness_check_time = None
164
165
 
166
+ self._warned_about_long_in_progress_reconcile = False
167
+
165
168
  def __enter__(self):
166
169
  return self
167
170
 
@@ -305,6 +308,19 @@ class DagsterCloudAgent:
305
308
 
306
309
  self._check_initial_deployment_names()
307
310
 
311
+ serving = []
312
+ queues = list(filter(None, self._instance.agent_queues_config.queues))
313
+ if queues:
314
+ serving.append(f"queues{queues}")
315
+ if self._instance.deployment_names:
316
+ serving.append(f"deployments{self._instance.deployment_names}")
317
+ if self._instance.include_all_serverless_deployments:
318
+ serving.append("all serverless deployments")
319
+ if self._instance.includes_branch_deployments:
320
+ serving.append("branch deployments")
321
+
322
+ self._logger.info(f"Agent is serving: {', '.join(serving)}")
323
+
308
324
  self._check_update_workspace(
309
325
  user_code_launcher,
310
326
  upload_all=user_code_launcher.upload_snapshots_on_startup,
@@ -352,6 +368,8 @@ class DagsterCloudAgent:
352
368
  except Exception:
353
369
  self._logger.exception("Failed to add heartbeat")
354
370
 
371
+ self._check_for_long_running_reconcile(user_code_launcher)
372
+
355
373
  # Check for any received interrupts
356
374
  with raise_interrupts_as(KeyboardInterrupt):
357
375
  pass
@@ -394,6 +412,26 @@ class DagsterCloudAgent:
394
412
  self._logger.error(f"Failed to write liveness sentinel and disabling it: {e}")
395
413
  self._last_liveness_check_time = False
396
414
 
415
+ def _check_for_long_running_reconcile(self, user_code_launcher):
416
+ """Detect from the main thread if the background reconcile thread is running behind or has gotten stuck."""
417
+ in_progress_reconcile_start_time = user_code_launcher.in_progress_reconcile_start_time
418
+
419
+ reconcile_start_time_warning = int(
420
+ os.getenv("DAGSTER_CLOUD_AGENT_RECONCILE_START_TIME_WARNING", "3600")
421
+ )
422
+
423
+ if (
424
+ in_progress_reconcile_start_time is not None
425
+ and (time.time() - in_progress_reconcile_start_time) >= reconcile_start_time_warning
426
+ ):
427
+ if not self._warned_about_long_in_progress_reconcile:
428
+ self._logger.warning(
429
+ f"Agent has been redeploying code servers for more than {reconcile_start_time_warning} seconds. This may indicate the background thread that performs the redeploys is stuck."
430
+ )
431
+ self._warned_about_long_in_progress_reconcile = True
432
+ else:
433
+ self._warned_about_long_in_progress_reconcile = False
434
+
397
435
  def _check_update_workspace(self, user_code_launcher, upload_all):
398
436
  curr_time = get_current_datetime()
399
437
 
@@ -1044,10 +1082,16 @@ class DagsterCloudAgent:
1044
1082
  client.cancel_execution(CancelExecutionRequest(run_id=run.run_id))
1045
1083
 
1046
1084
  return DagsterCloudApiSuccess()
1047
-
1085
+ elif api_name in (
1086
+ DagsterCloudApi.CHECK_STEP_HEALTH,
1087
+ DagsterCloudApi.TERMINATE_STEP,
1088
+ DagsterCloudApi.LAUNCH_STEP,
1089
+ DagsterCloudApi.CHECK_RUN_HEALTH,
1090
+ DagsterCloudApi.LOAD_REPOSITORIES,
1091
+ ):
1092
+ check.failed(f"Unexpected deprecated request type {api_name}")
1048
1093
  else:
1049
1094
  check.assert_never(api_name)
1050
- raise Exception(f"Unexpected dagster cloud api call {api_name}")
1051
1095
 
1052
1096
  def _process_api_request(
1053
1097
  self,
@@ -19,5 +19,5 @@ def extract_run_attributes(deployment_name, run: DagsterRun) -> dict[str, str]:
19
19
  attributes["location"] = run.remote_job_origin.location_name or "unknown"
20
20
  except Exception as e: # pylint: disable=broad-except
21
21
  logging.warning("Failed to extract additional run attributes", exc_info=e)
22
- finally:
23
- return attributes
22
+
23
+ return attributes
@@ -1,6 +1,6 @@
1
1
  from dagster import deserialize_value
2
2
  from dagster._core.definitions.schedule_definition import ScheduleExecutionData
3
- from dagster._core.remote_representation import ScheduleExecutionErrorSnap
3
+ from dagster._core.remote_representation.external_data import ScheduleExecutionErrorSnap
4
4
 
5
5
  from dagster_cloud.opentelemetry.observers.execution_observer import ExecutionObserverInstruments
6
6
 
@@ -1,6 +1,6 @@
1
1
  from dagster import deserialize_value
2
2
  from dagster._core.definitions.sensor_definition import SensorExecutionData
3
- from dagster._core.remote_representation import SensorExecutionErrorSnap
3
+ from dagster._core.remote_representation.external_data import SensorExecutionErrorSnap
4
4
 
5
5
  from dagster_cloud.opentelemetry.observers.execution_observer import ExecutionObserverInstruments
6
6
 
@@ -1,7 +1,7 @@
1
- from .defs import (
1
+ from dagster_cloud.anomaly_detection.defs import (
2
2
  build_anomaly_detection_freshness_checks as build_anomaly_detection_freshness_checks,
3
3
  )
4
- from .types import (
4
+ from dagster_cloud.anomaly_detection.types import (
5
5
  AnomalyDetectionModelParams as AnomalyDetectionModelParams,
6
6
  BetaFreshnessAnomalyDetectionParams as BetaFreshnessAnomalyDetectionParams,
7
7
  )
@@ -7,24 +7,27 @@ from dagster import (
7
7
  MetadataValue,
8
8
  _check as check,
9
9
  )
10
- from dagster._core.definitions.asset_check_factories.freshness_checks.last_update import (
10
+ from dagster._core.definitions.asset_checks.asset_check_factories.freshness_checks.last_update import (
11
11
  construct_description,
12
12
  )
13
- from dagster._core.definitions.asset_check_factories.utils import (
13
+ from dagster._core.definitions.asset_checks.asset_check_factories.utils import (
14
14
  FRESH_UNTIL_METADATA_KEY,
15
15
  LAST_UPDATED_TIMESTAMP_METADATA_KEY,
16
16
  LOWER_BOUND_TIMESTAMP_METADATA_KEY,
17
17
  assets_to_keys,
18
18
  )
19
- from dagster._core.definitions.asset_check_result import AssetCheckResult
20
- from dagster._core.definitions.asset_check_spec import (
19
+ from dagster._core.definitions.asset_checks.asset_check_result import AssetCheckResult
20
+ from dagster._core.definitions.asset_checks.asset_check_spec import (
21
21
  AssetCheckKey,
22
22
  AssetCheckSeverity,
23
23
  AssetCheckSpec,
24
24
  )
25
- from dagster._core.definitions.asset_checks import AssetChecksDefinition
25
+ from dagster._core.definitions.asset_checks.asset_checks_definition import AssetChecksDefinition
26
26
  from dagster._core.definitions.asset_key import AssetKey
27
- from dagster._core.definitions.assets import AssetsDefinition, unique_id_from_asset_and_check_keys
27
+ from dagster._core.definitions.assets.definition.assets_definition import (
28
+ AssetsDefinition,
29
+ unique_id_from_asset_and_check_keys,
30
+ )
28
31
  from dagster._core.definitions.decorators.asset_check_decorator import multi_asset_check
29
32
  from dagster._core.definitions.events import CoercibleToAssetKey
30
33
  from dagster._core.definitions.source_asset import SourceAsset
@@ -36,8 +39,8 @@ from dagster_cloud_cli.core.graphql_client import (
36
39
  create_cloud_webserver_client,
37
40
  )
38
41
 
39
- from .mutation import ANOMALY_DETECTION_INFERENCE_MUTATION
40
- from .types import (
42
+ from dagster_cloud.anomaly_detection.mutation import ANOMALY_DETECTION_INFERENCE_MUTATION
43
+ from dagster_cloud.anomaly_detection.types import (
41
44
  AnomalyDetectionModelParams,
42
45
  BetaFreshnessAnomalyDetectionParams,
43
46
  FreshnessAnomalyDetectionResult,
@@ -7,7 +7,8 @@ from typing import Any, Optional, TypedDict, Union
7
7
  from dagster._core.code_pointer import CodePointer
8
8
  from dagster._core.definitions.selector import JobSelector
9
9
  from dagster._core.events.log import EventLogEntry
10
- from dagster._core.remote_representation import CodeLocationOrigin, RepositorySnap
10
+ from dagster._core.remote_origin import CodeLocationOrigin
11
+ from dagster._core.remote_representation.external_data import RepositorySnap
11
12
  from dagster._core.storage.dagster_run import DagsterRun
12
13
  from dagster._core.utils import RequestUtilizationMetrics
13
14
  from dagster._record import IHaveNew, copy, record, record_custom
@@ -84,7 +85,6 @@ class DagsterCloudUploadWorkspaceResponse:
84
85
  @whitelist_for_serdes
85
86
  class DagsterCloudApi(Enum):
86
87
  CHECK_FOR_WORKSPACE_UPDATES = "CHECK_FOR_WORKSPACE_UPDATES"
87
- LOAD_REPOSITORIES = "LOAD_REPOSITORIES"
88
88
  GET_EXTERNAL_EXECUTION_PLAN = "GET_EXTERNAL_EXECUTION_PLAN"
89
89
  GET_SUBSET_EXTERNAL_PIPELINE_RESULT = "GET_SUBSET_EXTERNAL_PIPELINE_RESULT"
90
90
  GET_EXTERNAL_PARTITION_CONFIG = "GET_EXTERNAL_PARTITION_CONFIG"
@@ -98,13 +98,15 @@ class DagsterCloudApi(Enum):
98
98
  GET_EXTERNAL_NOTEBOOK_DATA = "GET_EXTERNAL_NOTEBOOK_DATA"
99
99
 
100
100
  LAUNCH_RUN = "LAUNCH_RUN"
101
- CHECK_RUN_HEALTH = "CHECK_RUN_HEALTH" # deprecated, agents now surface this in heartbeats
102
101
  TERMINATE_RUN = "TERMINATE_RUN"
102
+
103
+ PING_LOCATION = "PING_LOCATION" # Signal that a location is in use and should keep servers up
104
+
105
+ CHECK_RUN_HEALTH = "CHECK_RUN_HEALTH" # deprecated, agents now surface this in heartbeats
103
106
  LAUNCH_STEP = "LAUNCH_STEP" # deprecated with cloud executor
104
107
  CHECK_STEP_HEALTH = "CHECK_STEP_HEALTH" # deprecated with cloud executor
105
108
  TERMINATE_STEP = "TERMINATE_STEP" # deprecated with cloud executor
106
-
107
- PING_LOCATION = "PING_LOCATION" # Signal that a location is in use and should keep servers up
109
+ LOAD_REPOSITORIES = "LOAD_REPOSITORIES" # deprecated
108
110
 
109
111
  def __structlog__(self):
110
112
  return self.name
@@ -1,3 +1,4 @@
1
+ import uuid
1
2
  from typing import Optional
2
3
 
3
4
  from dagster._core.errors import DagsterInvariantViolationError
@@ -19,12 +20,27 @@ def get_organization_public_id_from_api_token(api_token: str) -> Optional[str]:
19
20
  return split_token[2]
20
21
 
21
22
 
22
- def get_organization_name_from_agent_token(agent_token: str) -> Optional[str]:
23
+ def decode_region_from_uuid(regional_token: str) -> Optional[str]:
24
+ try:
25
+ regional_uuid = uuid.UUID(regional_token)
26
+ except ValueError:
27
+ # if it's not an actual uuid, we can't decode region
28
+ return None
29
+
30
+ # custom uuids contain region subdomains in the first 2 bytes
31
+ if regional_uuid.version != 8 or regional_uuid.variant != uuid.RFC_4122:
32
+ return None
33
+
34
+ uuid_bytes = regional_uuid.bytes
35
+ return uuid_bytes[:2].decode("ascii")
36
+
37
+
38
+ def decode_agent_token(agent_token: str) -> tuple[Optional[str], Optional[str]]:
23
39
  split_token = agent_token.split(":")
24
40
 
25
41
  # Legacy agent token format - organization must be specified in dagster.yaml
26
42
  if len(split_token) == 1:
27
- return None
43
+ return None, None
28
44
 
29
45
  token_type, *token = split_token
30
46
 
@@ -35,6 +51,6 @@ def get_organization_name_from_agent_token(agent_token: str) -> Optional[str]:
35
51
  "Generate a new agent token in Dagster Cloud."
36
52
  )
37
53
 
38
- organization, _identifier = token
39
-
40
- return organization
54
+ # token format: agent:<org>:<uuid>
55
+ organization, uuid_str = token
56
+ return organization, decode_region_from_uuid(uuid_str)
@@ -1 +1 @@
1
- from .batcher import Batcher as Batcher
1
+ from dagster_cloud.batching.batcher import Batcher as Batcher
@@ -1,15 +1,19 @@
1
1
  import sys
2
2
  from typing import Any
3
3
 
4
- from .snowflake.dbt_wrapper import dbt_with_snowflake_insights as dbt_with_snowflake_insights
5
- from .snowflake.definitions import (
4
+ from dagster_cloud.dagster_insights.snowflake.dbt_wrapper import (
5
+ dbt_with_snowflake_insights as dbt_with_snowflake_insights,
6
+ )
7
+ from dagster_cloud.dagster_insights.snowflake.definitions import (
6
8
  create_snowflake_insights_asset_and_schedule as create_snowflake_insights_asset_and_schedule,
7
9
  )
8
- from .snowflake.snowflake_utils import meter_snowflake_query as meter_snowflake_query
10
+ from dagster_cloud.dagster_insights.snowflake.snowflake_utils import (
11
+ meter_snowflake_query as meter_snowflake_query,
12
+ )
9
13
 
10
14
  dagster_snowflake_req_imports = {"InsightsSnowflakeResource"}
11
15
  try:
12
- from .snowflake.insights_snowflake_resource import (
16
+ from dagster_cloud.dagster_insights.snowflake.insights_snowflake_resource import (
13
17
  InsightsSnowflakeResource as InsightsSnowflakeResource,
14
18
  )
15
19
  except ImportError:
@@ -17,8 +21,10 @@ except ImportError:
17
21
 
18
22
  dagster_bigquery_req_imports = {"InsightsBigQueryResource", "dbt_with_bigquery_insights"}
19
23
  try:
20
- from .bigquery.dbt_wrapper import dbt_with_bigquery_insights as dbt_with_bigquery_insights
21
- from .bigquery.insights_bigquery_resource import (
24
+ from dagster_cloud.dagster_insights.bigquery.dbt_wrapper import (
25
+ dbt_with_bigquery_insights as dbt_with_bigquery_insights,
26
+ )
27
+ from dagster_cloud.dagster_insights.bigquery.insights_bigquery_resource import (
22
28
  InsightsBigQueryResource as InsightsBigQueryResource,
23
29
  )
24
30
  except ImportError:
@@ -18,8 +18,14 @@ from dagster_dbt import DbtCliInvocation
18
18
  from dagster_dbt.version import __version__ as dagster_dbt_version
19
19
  from packaging import version
20
20
 
21
- from ..insights_utils import extract_asset_info_from_event, handle_raise_on_error
22
- from .bigquery_utils import build_bigquery_cost_metadata, marker_asset_key_for_job
21
+ from dagster_cloud.dagster_insights.bigquery.bigquery_utils import (
22
+ build_bigquery_cost_metadata,
23
+ marker_asset_key_for_job,
24
+ )
25
+ from dagster_cloud.dagster_insights.insights_utils import (
26
+ extract_asset_info_from_event,
27
+ handle_raise_on_error,
28
+ )
23
29
 
24
30
  if TYPE_CHECKING:
25
31
  from dbt.adapters.base.impl import BaseAdapter
@@ -7,10 +7,12 @@ from dagster_gcp import BigQueryResource
7
7
  from dagster_gcp.bigquery.utils import setup_gcp_creds
8
8
  from google.cloud import bigquery
9
9
 
10
+ from dagster_cloud.dagster_insights.bigquery.bigquery_utils import (
11
+ build_bigquery_cost_metadata,
12
+ marker_asset_key_for_job,
13
+ )
10
14
  from dagster_cloud.dagster_insights.insights_utils import get_current_context_and_asset_key
11
15
 
12
- from .bigquery_utils import build_bigquery_cost_metadata, marker_asset_key_for_job
13
-
14
16
  OUTPUT_NON_ASSET_SIGIL = "__bigquery_query_metadata_"
15
17
 
16
18
 
@@ -33,7 +33,7 @@ def get_current_context_and_asset_key() -> tuple[
33
33
  def get_asset_key_for_output(
34
34
  context: Union[OpExecutionContext, AssetExecutionContext], output_name: str
35
35
  ) -> Optional[AssetKey]:
36
- asset_key = context.job_def.asset_layer.asset_key_for_output(
36
+ asset_key = context.job_def.asset_layer.get_asset_key_for_node_output(
37
37
  node_handle=context.op_handle, output_name=output_name
38
38
  )
39
39
  if asset_key is None:
@@ -28,7 +28,7 @@ def get_url_and_token_from_instance(instance: DagsterInstance) -> tuple[str, str
28
28
  if not isinstance(instance, DagsterCloudAgentInstance):
29
29
  raise RuntimeError("This asset only functions in a running Dagster Cloud instance")
30
30
 
31
- return f"{instance.dagit_url}graphql", instance.dagster_cloud_agent_token # pyright: ignore[reportReturnType]
31
+ return f"{instance.dagit_url}graphql", instance.dagster_cloud_agent_token
32
32
 
33
33
 
34
34
  def get_insights_upload_request_params(
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Optional
6
6
 
7
7
  from dagster import AssetKey, AssetsDefinition, ScheduleDefinition
8
8
 
9
- from .snowflake_utils import OPAQUE_ID_SQL_SIGIL
9
+ from dagster_cloud.dagster_insights.snowflake.snowflake_utils import OPAQUE_ID_SQL_SIGIL
10
10
 
11
11
  if TYPE_CHECKING:
12
12
  from dagster_snowflake import SnowflakeConnection
@@ -78,10 +78,6 @@ HAVING ARRAY_SIZE(opaque_ids) > 0
78
78
 
79
79
  costs: list[tuple[str, float, str]] = []
80
80
 
81
- print(
82
- f"{len(results) if results else 0} annotated queries returned from snowflake query_history"
83
- )
84
-
85
81
  if not results:
86
82
  return []
87
83
 
@@ -94,8 +90,4 @@ HAVING ARRAY_SIZE(opaque_ids) > 0
94
90
  for opaque_id in opaque_ids:
95
91
  costs.append((opaque_id, float(cost), query_id))
96
92
 
97
- print(
98
- f"Reported costs for {len(costs)} of {total} asset materializations found in the"
99
- " query_history."
100
- )
101
93
  return costs
@@ -13,8 +13,15 @@ from dagster import (
13
13
  Output,
14
14
  )
15
15
 
16
- from ..insights_utils import extract_asset_info_from_event, handle_raise_on_error
17
- from .snowflake_utils import OPAQUE_ID_SQL_SIGIL, build_opaque_id_metadata, marker_asset_key_for_job
16
+ from dagster_cloud.dagster_insights.insights_utils import (
17
+ extract_asset_info_from_event,
18
+ handle_raise_on_error,
19
+ )
20
+ from dagster_cloud.dagster_insights.snowflake.snowflake_utils import (
21
+ OPAQUE_ID_SQL_SIGIL,
22
+ build_opaque_id_metadata,
23
+ marker_asset_key_for_job,
24
+ )
18
25
 
19
26
  if TYPE_CHECKING:
20
27
  from dagster_dbt import DbtCliInvocation
@@ -2,7 +2,6 @@ import warnings
2
2
  from collections.abc import Sequence
3
3
  from dataclasses import dataclass
4
4
  from datetime import date, datetime, timedelta, timezone
5
- from pprint import pprint
6
5
  from typing import TYPE_CHECKING, Optional, Union
7
6
 
8
7
  from dagster import (
@@ -27,8 +26,10 @@ from dagster._core.storage.tags import (
27
26
  ASSET_PARTITION_RANGE_START_TAG,
28
27
  )
29
28
 
30
- from ..metrics_utils import put_cost_information
31
- from .dagster_snowflake_insights import get_cost_data_for_hour
29
+ from dagster_cloud.dagster_insights.metrics_utils import put_cost_information
30
+ from dagster_cloud.dagster_insights.snowflake.dagster_snowflake_insights import (
31
+ get_cost_data_for_hour,
32
+ )
32
33
 
33
34
  if TYPE_CHECKING:
34
35
  from dagster_snowflake import SnowflakeConnection
@@ -154,7 +155,7 @@ def create_snowflake_insights_asset_and_schedule(
154
155
  )
155
156
 
156
157
  if dry_run:
157
- pprint(costs)
158
+ pass
158
159
  else:
159
160
  context.log.info(
160
161
  f"Submitting cost information for {len(costs)} queries to Dagster Insights"
@@ -20,8 +20,7 @@ from dagster_snowflake import SnowflakeConnection, SnowflakeResource
20
20
  from snowflake.connector.cursor import SnowflakeCursor
21
21
 
22
22
  from dagster_cloud.dagster_insights.insights_utils import get_current_context_and_asset_key
23
-
24
- from .snowflake_utils import meter_snowflake_query
23
+ from dagster_cloud.dagster_insights.snowflake.snowflake_utils import meter_snowflake_query
25
24
 
26
25
 
27
26
  def get_current_context_and_asset_key_or_warn() -> tuple[
File without changes
@@ -0,0 +1,36 @@
1
+ from abc import ABC, abstractmethod
2
+ from collections.abc import Sequence
3
+ from typing import AbstractSet # noqa: UP035
4
+
5
+ import dagster._check as check
6
+ from dagster._record import record
7
+ from dagster_shared.serdes import whitelist_for_serdes
8
+
9
+
10
+ @whitelist_for_serdes
11
+ @record
12
+ class JobSelection(ABC):
13
+ code_location_name: str
14
+ repository_name: str
15
+
16
+ @abstractmethod
17
+ def resolve_job_names(self) -> AbstractSet[str]:
18
+ raise NotImplementedError()
19
+
20
+ @staticmethod
21
+ def names(
22
+ names: Sequence[str], code_location_name: str, repository_name: str
23
+ ) -> "NamesJobSelection":
24
+ check.invariant(len(names) == 1, "Only one job name is supported at this time")
25
+ return NamesJobSelection(
26
+ job_names=names, code_location_name=code_location_name, repository_name=repository_name
27
+ )
28
+
29
+
30
+ @whitelist_for_serdes
31
+ @record
32
+ class NamesJobSelection(JobSelection):
33
+ job_names: Sequence[str]
34
+
35
+ def resolve_job_names(self) -> AbstractSet[str]:
36
+ return set(self.job_names)
@@ -4,7 +4,7 @@ from collections.abc import Sequence
4
4
 
5
5
  from dagster_shared.ipc import interrupt_ipc_subprocess_pid, open_ipc_subprocess
6
6
 
7
- from . import TaskStatus
7
+ from dagster_cloud.execution.utils import TaskStatus
8
8
 
9
9
 
10
10
  def launch_process(args: Sequence[str]) -> int: