dagster-cloud 1.8.2__py3-none-any.whl → 1.12.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dagster_cloud/__init__.py +3 -3
- dagster_cloud/agent/__init__.py +4 -4
- dagster_cloud/agent/cli/__init__.py +56 -17
- dagster_cloud/agent/dagster_cloud_agent.py +360 -172
- dagster_cloud/agent/instrumentation/__init__.py +0 -0
- dagster_cloud/agent/instrumentation/constants.py +2 -0
- dagster_cloud/agent/instrumentation/run_launch.py +23 -0
- dagster_cloud/agent/instrumentation/schedule.py +34 -0
- dagster_cloud/agent/instrumentation/sensor.py +34 -0
- dagster_cloud/anomaly_detection/__init__.py +2 -2
- dagster_cloud/anomaly_detection/defs.py +17 -12
- dagster_cloud/anomaly_detection/types.py +3 -3
- dagster_cloud/api/dagster_cloud_api.py +209 -293
- dagster_cloud/auth/constants.py +21 -5
- dagster_cloud/batching/__init__.py +1 -0
- dagster_cloud/batching/batcher.py +210 -0
- dagster_cloud/dagster_insights/__init__.py +12 -6
- dagster_cloud/dagster_insights/bigquery/bigquery_utils.py +3 -2
- dagster_cloud/dagster_insights/bigquery/dbt_wrapper.py +39 -12
- dagster_cloud/dagster_insights/bigquery/insights_bigquery_resource.py +8 -6
- dagster_cloud/dagster_insights/insights_utils.py +18 -8
- dagster_cloud/dagster_insights/metrics_utils.py +12 -12
- dagster_cloud/dagster_insights/snowflake/dagster_snowflake_insights.py +5 -12
- dagster_cloud/dagster_insights/snowflake/dbt_wrapper.py +34 -8
- dagster_cloud/dagster_insights/snowflake/definitions.py +38 -12
- dagster_cloud/dagster_insights/snowflake/insights_snowflake_resource.py +11 -23
- dagster_cloud/definitions/__init__.py +0 -0
- dagster_cloud/definitions/job_selection.py +36 -0
- dagster_cloud/execution/cloud_run_launcher/k8s.py +1 -1
- dagster_cloud/execution/cloud_run_launcher/process.py +3 -3
- dagster_cloud/execution/monitoring/__init__.py +27 -33
- dagster_cloud/execution/utils/process.py +3 -3
- dagster_cloud/instance/__init__.py +125 -38
- dagster_cloud/instrumentation/__init__.py +32 -0
- dagster_cloud/metadata/source_code.py +13 -8
- dagster_cloud/metrics/__init__.py +0 -0
- dagster_cloud/metrics/tracer.py +59 -0
- dagster_cloud/opentelemetry/__init__.py +0 -0
- dagster_cloud/opentelemetry/config/__init__.py +73 -0
- dagster_cloud/opentelemetry/config/exporter.py +81 -0
- dagster_cloud/opentelemetry/config/log_record_processor.py +40 -0
- dagster_cloud/opentelemetry/config/logging_handler.py +14 -0
- dagster_cloud/opentelemetry/config/meter_provider.py +9 -0
- dagster_cloud/opentelemetry/config/metric_reader.py +39 -0
- dagster_cloud/opentelemetry/controller.py +319 -0
- dagster_cloud/opentelemetry/enum.py +58 -0
- dagster_cloud/opentelemetry/factories/__init__.py +1 -0
- dagster_cloud/opentelemetry/factories/logs.py +113 -0
- dagster_cloud/opentelemetry/factories/metrics.py +121 -0
- dagster_cloud/opentelemetry/metrics/__init__.py +0 -0
- dagster_cloud/opentelemetry/metrics/meter.py +140 -0
- dagster_cloud/opentelemetry/observers/__init__.py +0 -0
- dagster_cloud/opentelemetry/observers/dagster_exception_handler.py +40 -0
- dagster_cloud/opentelemetry/observers/execution_observer.py +178 -0
- dagster_cloud/pex/grpc/__generated__/multi_pex_api_pb2.pyi +175 -0
- dagster_cloud/pex/grpc/__init__.py +2 -2
- dagster_cloud/pex/grpc/client.py +4 -4
- dagster_cloud/pex/grpc/compile.py +2 -2
- dagster_cloud/pex/grpc/server/__init__.py +2 -2
- dagster_cloud/pex/grpc/server/cli/__init__.py +31 -19
- dagster_cloud/pex/grpc/server/manager.py +60 -42
- dagster_cloud/pex/grpc/server/registry.py +28 -21
- dagster_cloud/pex/grpc/server/server.py +23 -14
- dagster_cloud/pex/grpc/types.py +5 -5
- dagster_cloud/py.typed +0 -0
- dagster_cloud/secrets/__init__.py +1 -1
- dagster_cloud/secrets/loader.py +3 -3
- dagster_cloud/serverless/__init__.py +1 -1
- dagster_cloud/serverless/io_manager.py +36 -53
- dagster_cloud/storage/client.py +54 -17
- dagster_cloud/storage/compute_logs/__init__.py +3 -1
- dagster_cloud/storage/compute_logs/compute_log_manager.py +22 -17
- dagster_cloud/storage/defs_state/__init__.py +3 -0
- dagster_cloud/storage/defs_state/queries.py +15 -0
- dagster_cloud/storage/defs_state/storage.py +113 -0
- dagster_cloud/storage/event_logs/__init__.py +3 -1
- dagster_cloud/storage/event_logs/queries.py +102 -4
- dagster_cloud/storage/event_logs/storage.py +266 -73
- dagster_cloud/storage/event_logs/utils.py +88 -7
- dagster_cloud/storage/runs/__init__.py +1 -1
- dagster_cloud/storage/runs/queries.py +17 -2
- dagster_cloud/storage/runs/storage.py +88 -42
- dagster_cloud/storage/schedules/__init__.py +1 -1
- dagster_cloud/storage/schedules/storage.py +6 -8
- dagster_cloud/storage/tags.py +66 -1
- dagster_cloud/util/__init__.py +10 -12
- dagster_cloud/util/errors.py +49 -64
- dagster_cloud/version.py +1 -1
- dagster_cloud/workspace/config_schema/__init__.py +55 -13
- dagster_cloud/workspace/docker/__init__.py +76 -25
- dagster_cloud/workspace/docker/utils.py +1 -1
- dagster_cloud/workspace/ecs/__init__.py +1 -1
- dagster_cloud/workspace/ecs/client.py +51 -33
- dagster_cloud/workspace/ecs/launcher.py +76 -22
- dagster_cloud/workspace/ecs/run_launcher.py +3 -3
- dagster_cloud/workspace/ecs/utils.py +14 -5
- dagster_cloud/workspace/kubernetes/__init__.py +1 -1
- dagster_cloud/workspace/kubernetes/launcher.py +61 -29
- dagster_cloud/workspace/kubernetes/utils.py +34 -22
- dagster_cloud/workspace/user_code_launcher/__init__.py +5 -3
- dagster_cloud/workspace/user_code_launcher/process.py +16 -14
- dagster_cloud/workspace/user_code_launcher/user_code_launcher.py +552 -172
- dagster_cloud/workspace/user_code_launcher/utils.py +105 -1
- {dagster_cloud-1.8.2.dist-info → dagster_cloud-1.12.6.dist-info}/METADATA +48 -42
- dagster_cloud-1.12.6.dist-info/RECORD +134 -0
- {dagster_cloud-1.8.2.dist-info → dagster_cloud-1.12.6.dist-info}/WHEEL +1 -1
- dagster_cloud-1.8.2.dist-info/RECORD +0 -100
- {dagster_cloud-1.8.2.dist-info → dagster_cloud-1.12.6.dist-info}/top_level.txt +0 -0
|
@@ -105,6 +105,8 @@ ASSET_ENTRY_FRAGMENT = (
|
|
|
105
105
|
lastObservationRecord {
|
|
106
106
|
...EventLogRecordFragment
|
|
107
107
|
}
|
|
108
|
+
lastPlannedMaterializationStorageId
|
|
109
|
+
lastPlannedMaterializationRunId
|
|
108
110
|
}
|
|
109
111
|
"""
|
|
110
112
|
)
|
|
@@ -140,11 +142,28 @@ ASSET_CHECK_STATE_FRAGMENT = (
|
|
|
140
142
|
}
|
|
141
143
|
createTimestamp
|
|
142
144
|
}
|
|
143
|
-
|
|
144
|
-
|
|
145
|
+
lastCompletedCheckExecutionRecord {
|
|
146
|
+
id
|
|
147
|
+
runId
|
|
148
|
+
status
|
|
149
|
+
event {
|
|
150
|
+
...EventLogEntryFragment
|
|
151
|
+
}
|
|
152
|
+
createTimestamp
|
|
153
|
+
}
|
|
154
|
+
lastRunId
|
|
155
|
+
}
|
|
145
156
|
"""
|
|
146
157
|
)
|
|
147
158
|
|
|
159
|
+
GET_MAXIMUM_RECORD_ID = """
|
|
160
|
+
query getMaximumRecordId {
|
|
161
|
+
eventLogs {
|
|
162
|
+
getMaximumRecordId
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
"""
|
|
166
|
+
|
|
148
167
|
GET_RECORDS_FOR_RUN_QUERY = (
|
|
149
168
|
EVENT_RECORD_FRAGMENT
|
|
150
169
|
+ """
|
|
@@ -321,9 +340,9 @@ GET_MATERIALIZATION_COUNT_BY_PARTITION = """
|
|
|
321
340
|
"""
|
|
322
341
|
|
|
323
342
|
GET_LATEST_STORAGE_ID_BY_PARTITION = """
|
|
324
|
-
query getLatestStorageIdByPartition($assetKey: String!, $eventType: String!) {
|
|
343
|
+
query getLatestStorageIdByPartition($assetKey: String!, $eventType: String!, $partitions: [String!]) {
|
|
325
344
|
eventLogs {
|
|
326
|
-
getLatestStorageIdByPartition(assetKey: $assetKey, eventType: $eventType) {
|
|
345
|
+
getLatestStorageIdByPartition(assetKey: $assetKey, eventType: $eventType, partitions: $partitions) {
|
|
327
346
|
... on StorageIdByPartition {
|
|
328
347
|
partition
|
|
329
348
|
storageId
|
|
@@ -465,6 +484,16 @@ WIPE_ASSET_MUTATION = """
|
|
|
465
484
|
}
|
|
466
485
|
"""
|
|
467
486
|
|
|
487
|
+
WIPE_ASSET_PARTITIONS_MUTATION = """
|
|
488
|
+
mutation WipeAssetPartitions($assetKey: String!, $partitionKeys: [String!]!) {
|
|
489
|
+
eventLogs {
|
|
490
|
+
WipeAssetPartitions(assetKey: $assetKey, partitionKeys: $partitionKeys) {
|
|
491
|
+
ok
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
"""
|
|
496
|
+
|
|
468
497
|
WIPE_ASSET_CACHED_STATUS_DATA_MUTATION = """
|
|
469
498
|
mutation WipeAssetCachedStatusData($assetKey: String!) {
|
|
470
499
|
eventLogs {
|
|
@@ -483,6 +512,19 @@ query getDynamicPartitions($partitionsDefName: String!) {
|
|
|
483
512
|
}
|
|
484
513
|
"""
|
|
485
514
|
|
|
515
|
+
GET_PAGINATED_DYNAMIC_PARTITIONS_QUERY = """
|
|
516
|
+
query getPaginatedDynamicPartitions($partitionsDefName: String!, $limit: Int!, $ascending: Boolean!, $cursor: String) {
|
|
517
|
+
eventLogs {
|
|
518
|
+
getPaginatedDynamicPartitions(partitionsDefName: $partitionsDefName, limit: $limit, ascending: $ascending, cursor: $cursor) {
|
|
519
|
+
results
|
|
520
|
+
hasMore
|
|
521
|
+
cursor
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
"""
|
|
526
|
+
|
|
527
|
+
|
|
486
528
|
HAS_DYNAMIC_PARTITION_QUERY = """
|
|
487
529
|
query hasDynamicPartition($partitionsDefName: String!, $partitionKey: String!) {
|
|
488
530
|
eventLogs {
|
|
@@ -566,6 +608,30 @@ query getConcurrencyKeys {
|
|
|
566
608
|
}
|
|
567
609
|
"""
|
|
568
610
|
|
|
611
|
+
GET_POOL_LIMITS_QUERY = """
|
|
612
|
+
query getPoolLimits {
|
|
613
|
+
eventLogs {
|
|
614
|
+
getPoolLimits {
|
|
615
|
+
name
|
|
616
|
+
limit
|
|
617
|
+
fromDefault
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
"""
|
|
622
|
+
|
|
623
|
+
GET_POOL_CONFIG_QUERY = """
|
|
624
|
+
query getPoolConfig {
|
|
625
|
+
eventLogs {
|
|
626
|
+
getPoolConfig {
|
|
627
|
+
poolGranularity
|
|
628
|
+
defaultPoolLimit
|
|
629
|
+
opGranularityRunBuffer
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
"""
|
|
634
|
+
|
|
569
635
|
GET_CONCURRENCY_INFO_QUERY = """
|
|
570
636
|
query getConcurrencyInfo($concurrencyKey: String!) {
|
|
571
637
|
eventLogs {
|
|
@@ -583,6 +649,8 @@ query getConcurrencyInfo($concurrencyKey: String!) {
|
|
|
583
649
|
assignedTimestamp
|
|
584
650
|
priority
|
|
585
651
|
}
|
|
652
|
+
limit
|
|
653
|
+
usingDefaultLimit
|
|
586
654
|
}
|
|
587
655
|
}
|
|
588
656
|
}
|
|
@@ -705,6 +773,23 @@ query getRunStatusChangeEventRecordsQuery($recordsFilter: RunStatusChangeRecords
|
|
|
705
773
|
"""
|
|
706
774
|
)
|
|
707
775
|
|
|
776
|
+
FETCH_FAILED_MATERIALIZATIONS_QUERY = (
|
|
777
|
+
EVENT_RECORD_FRAGMENT
|
|
778
|
+
+ """
|
|
779
|
+
query fetchFailedMaterializationsQuery($recordsFilter: AssetRecordsFilter!, $limit: Int, $cursor: String, $ascending: Boolean) {
|
|
780
|
+
eventLogs {
|
|
781
|
+
fetchFailedMaterializations(recordsFilter: $recordsFilter, limit: $limit, cursor: $cursor, ascending: $ascending) {
|
|
782
|
+
records {
|
|
783
|
+
...EventLogRecordFragment
|
|
784
|
+
}
|
|
785
|
+
cursor
|
|
786
|
+
hasMore
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
"""
|
|
791
|
+
)
|
|
792
|
+
|
|
708
793
|
GET_LATEST_PLANNED_MATERIALIZATION_INFO = """
|
|
709
794
|
query getLatestPlannedMaterializationInfo($assetKey: String!, $partition: String) {
|
|
710
795
|
eventLogs {
|
|
@@ -724,3 +809,16 @@ GET_UPDATED_DATA_VERSION_PARTITIONS = """
|
|
|
724
809
|
}
|
|
725
810
|
}
|
|
726
811
|
"""
|
|
812
|
+
|
|
813
|
+
GET_ASSET_STATUS_CACHE_VALUES = (
|
|
814
|
+
CACHED_STATUS_DATA_FRAGMENT
|
|
815
|
+
+ """
|
|
816
|
+
query getAssetStatusCacheValues($assetKeys: [AssetKeyInput!]!) {
|
|
817
|
+
eventLogs {
|
|
818
|
+
getAssetStatusCacheValues(assetKeys: $assetKeys) {
|
|
819
|
+
...CachedStatusDataFragment
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
"""
|
|
824
|
+
)
|