dagster-cloud 1.8.2__py3-none-any.whl → 1.12.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. dagster_cloud/__init__.py +3 -3
  2. dagster_cloud/agent/__init__.py +4 -4
  3. dagster_cloud/agent/cli/__init__.py +56 -17
  4. dagster_cloud/agent/dagster_cloud_agent.py +360 -172
  5. dagster_cloud/agent/instrumentation/__init__.py +0 -0
  6. dagster_cloud/agent/instrumentation/constants.py +2 -0
  7. dagster_cloud/agent/instrumentation/run_launch.py +23 -0
  8. dagster_cloud/agent/instrumentation/schedule.py +34 -0
  9. dagster_cloud/agent/instrumentation/sensor.py +34 -0
  10. dagster_cloud/anomaly_detection/__init__.py +2 -2
  11. dagster_cloud/anomaly_detection/defs.py +17 -12
  12. dagster_cloud/anomaly_detection/types.py +3 -3
  13. dagster_cloud/api/dagster_cloud_api.py +209 -293
  14. dagster_cloud/auth/constants.py +21 -5
  15. dagster_cloud/batching/__init__.py +1 -0
  16. dagster_cloud/batching/batcher.py +210 -0
  17. dagster_cloud/dagster_insights/__init__.py +12 -6
  18. dagster_cloud/dagster_insights/bigquery/bigquery_utils.py +3 -2
  19. dagster_cloud/dagster_insights/bigquery/dbt_wrapper.py +39 -12
  20. dagster_cloud/dagster_insights/bigquery/insights_bigquery_resource.py +8 -6
  21. dagster_cloud/dagster_insights/insights_utils.py +18 -8
  22. dagster_cloud/dagster_insights/metrics_utils.py +12 -12
  23. dagster_cloud/dagster_insights/snowflake/dagster_snowflake_insights.py +5 -12
  24. dagster_cloud/dagster_insights/snowflake/dbt_wrapper.py +34 -8
  25. dagster_cloud/dagster_insights/snowflake/definitions.py +38 -12
  26. dagster_cloud/dagster_insights/snowflake/insights_snowflake_resource.py +11 -23
  27. dagster_cloud/definitions/__init__.py +0 -0
  28. dagster_cloud/definitions/job_selection.py +36 -0
  29. dagster_cloud/execution/cloud_run_launcher/k8s.py +1 -1
  30. dagster_cloud/execution/cloud_run_launcher/process.py +3 -3
  31. dagster_cloud/execution/monitoring/__init__.py +27 -33
  32. dagster_cloud/execution/utils/process.py +3 -3
  33. dagster_cloud/instance/__init__.py +125 -38
  34. dagster_cloud/instrumentation/__init__.py +32 -0
  35. dagster_cloud/metadata/source_code.py +13 -8
  36. dagster_cloud/metrics/__init__.py +0 -0
  37. dagster_cloud/metrics/tracer.py +59 -0
  38. dagster_cloud/opentelemetry/__init__.py +0 -0
  39. dagster_cloud/opentelemetry/config/__init__.py +73 -0
  40. dagster_cloud/opentelemetry/config/exporter.py +81 -0
  41. dagster_cloud/opentelemetry/config/log_record_processor.py +40 -0
  42. dagster_cloud/opentelemetry/config/logging_handler.py +14 -0
  43. dagster_cloud/opentelemetry/config/meter_provider.py +9 -0
  44. dagster_cloud/opentelemetry/config/metric_reader.py +39 -0
  45. dagster_cloud/opentelemetry/controller.py +319 -0
  46. dagster_cloud/opentelemetry/enum.py +58 -0
  47. dagster_cloud/opentelemetry/factories/__init__.py +1 -0
  48. dagster_cloud/opentelemetry/factories/logs.py +113 -0
  49. dagster_cloud/opentelemetry/factories/metrics.py +121 -0
  50. dagster_cloud/opentelemetry/metrics/__init__.py +0 -0
  51. dagster_cloud/opentelemetry/metrics/meter.py +140 -0
  52. dagster_cloud/opentelemetry/observers/__init__.py +0 -0
  53. dagster_cloud/opentelemetry/observers/dagster_exception_handler.py +40 -0
  54. dagster_cloud/opentelemetry/observers/execution_observer.py +178 -0
  55. dagster_cloud/pex/grpc/__generated__/multi_pex_api_pb2.pyi +175 -0
  56. dagster_cloud/pex/grpc/__init__.py +2 -2
  57. dagster_cloud/pex/grpc/client.py +4 -4
  58. dagster_cloud/pex/grpc/compile.py +2 -2
  59. dagster_cloud/pex/grpc/server/__init__.py +2 -2
  60. dagster_cloud/pex/grpc/server/cli/__init__.py +31 -19
  61. dagster_cloud/pex/grpc/server/manager.py +60 -42
  62. dagster_cloud/pex/grpc/server/registry.py +28 -21
  63. dagster_cloud/pex/grpc/server/server.py +23 -14
  64. dagster_cloud/pex/grpc/types.py +5 -5
  65. dagster_cloud/py.typed +0 -0
  66. dagster_cloud/secrets/__init__.py +1 -1
  67. dagster_cloud/secrets/loader.py +3 -3
  68. dagster_cloud/serverless/__init__.py +1 -1
  69. dagster_cloud/serverless/io_manager.py +36 -53
  70. dagster_cloud/storage/client.py +54 -17
  71. dagster_cloud/storage/compute_logs/__init__.py +3 -1
  72. dagster_cloud/storage/compute_logs/compute_log_manager.py +22 -17
  73. dagster_cloud/storage/defs_state/__init__.py +3 -0
  74. dagster_cloud/storage/defs_state/queries.py +15 -0
  75. dagster_cloud/storage/defs_state/storage.py +113 -0
  76. dagster_cloud/storage/event_logs/__init__.py +3 -1
  77. dagster_cloud/storage/event_logs/queries.py +102 -4
  78. dagster_cloud/storage/event_logs/storage.py +266 -73
  79. dagster_cloud/storage/event_logs/utils.py +88 -7
  80. dagster_cloud/storage/runs/__init__.py +1 -1
  81. dagster_cloud/storage/runs/queries.py +17 -2
  82. dagster_cloud/storage/runs/storage.py +88 -42
  83. dagster_cloud/storage/schedules/__init__.py +1 -1
  84. dagster_cloud/storage/schedules/storage.py +6 -8
  85. dagster_cloud/storage/tags.py +66 -1
  86. dagster_cloud/util/__init__.py +10 -12
  87. dagster_cloud/util/errors.py +49 -64
  88. dagster_cloud/version.py +1 -1
  89. dagster_cloud/workspace/config_schema/__init__.py +55 -13
  90. dagster_cloud/workspace/docker/__init__.py +76 -25
  91. dagster_cloud/workspace/docker/utils.py +1 -1
  92. dagster_cloud/workspace/ecs/__init__.py +1 -1
  93. dagster_cloud/workspace/ecs/client.py +51 -33
  94. dagster_cloud/workspace/ecs/launcher.py +76 -22
  95. dagster_cloud/workspace/ecs/run_launcher.py +3 -3
  96. dagster_cloud/workspace/ecs/utils.py +14 -5
  97. dagster_cloud/workspace/kubernetes/__init__.py +1 -1
  98. dagster_cloud/workspace/kubernetes/launcher.py +61 -29
  99. dagster_cloud/workspace/kubernetes/utils.py +34 -22
  100. dagster_cloud/workspace/user_code_launcher/__init__.py +5 -3
  101. dagster_cloud/workspace/user_code_launcher/process.py +16 -14
  102. dagster_cloud/workspace/user_code_launcher/user_code_launcher.py +552 -172
  103. dagster_cloud/workspace/user_code_launcher/utils.py +105 -1
  104. {dagster_cloud-1.8.2.dist-info → dagster_cloud-1.12.6.dist-info}/METADATA +48 -42
  105. dagster_cloud-1.12.6.dist-info/RECORD +134 -0
  106. {dagster_cloud-1.8.2.dist-info → dagster_cloud-1.12.6.dist-info}/WHEEL +1 -1
  107. dagster_cloud-1.8.2.dist-info/RECORD +0 -100
  108. {dagster_cloud-1.8.2.dist-info → dagster_cloud-1.12.6.dist-info}/top_level.txt +0 -0
@@ -105,6 +105,8 @@ ASSET_ENTRY_FRAGMENT = (
105
105
  lastObservationRecord {
106
106
  ...EventLogRecordFragment
107
107
  }
108
+ lastPlannedMaterializationStorageId
109
+ lastPlannedMaterializationRunId
108
110
  }
109
111
  """
110
112
  )
@@ -140,11 +142,28 @@ ASSET_CHECK_STATE_FRAGMENT = (
140
142
  }
141
143
  createTimestamp
142
144
  }
143
- lastRunId
144
- }
145
+ lastCompletedCheckExecutionRecord {
146
+ id
147
+ runId
148
+ status
149
+ event {
150
+ ...EventLogEntryFragment
151
+ }
152
+ createTimestamp
153
+ }
154
+ lastRunId
155
+ }
145
156
  """
146
157
  )
147
158
 
159
+ GET_MAXIMUM_RECORD_ID = """
160
+ query getMaximumRecordId {
161
+ eventLogs {
162
+ getMaximumRecordId
163
+ }
164
+ }
165
+ """
166
+
148
167
  GET_RECORDS_FOR_RUN_QUERY = (
149
168
  EVENT_RECORD_FRAGMENT
150
169
  + """
@@ -321,9 +340,9 @@ GET_MATERIALIZATION_COUNT_BY_PARTITION = """
321
340
  """
322
341
 
323
342
  GET_LATEST_STORAGE_ID_BY_PARTITION = """
324
- query getLatestStorageIdByPartition($assetKey: String!, $eventType: String!) {
343
+ query getLatestStorageIdByPartition($assetKey: String!, $eventType: String!, $partitions: [String!]) {
325
344
  eventLogs {
326
- getLatestStorageIdByPartition(assetKey: $assetKey, eventType: $eventType) {
345
+ getLatestStorageIdByPartition(assetKey: $assetKey, eventType: $eventType, partitions: $partitions) {
327
346
  ... on StorageIdByPartition {
328
347
  partition
329
348
  storageId
@@ -465,6 +484,16 @@ WIPE_ASSET_MUTATION = """
465
484
  }
466
485
  """
467
486
 
487
+ WIPE_ASSET_PARTITIONS_MUTATION = """
488
+ mutation WipeAssetPartitions($assetKey: String!, $partitionKeys: [String!]!) {
489
+ eventLogs {
490
+ WipeAssetPartitions(assetKey: $assetKey, partitionKeys: $partitionKeys) {
491
+ ok
492
+ }
493
+ }
494
+ }
495
+ """
496
+
468
497
  WIPE_ASSET_CACHED_STATUS_DATA_MUTATION = """
469
498
  mutation WipeAssetCachedStatusData($assetKey: String!) {
470
499
  eventLogs {
@@ -483,6 +512,19 @@ query getDynamicPartitions($partitionsDefName: String!) {
483
512
  }
484
513
  """
485
514
 
515
+ GET_PAGINATED_DYNAMIC_PARTITIONS_QUERY = """
516
+ query getPaginatedDynamicPartitions($partitionsDefName: String!, $limit: Int!, $ascending: Boolean!, $cursor: String) {
517
+ eventLogs {
518
+ getPaginatedDynamicPartitions(partitionsDefName: $partitionsDefName, limit: $limit, ascending: $ascending, cursor: $cursor) {
519
+ results
520
+ hasMore
521
+ cursor
522
+ }
523
+ }
524
+ }
525
+ """
526
+
527
+
486
528
  HAS_DYNAMIC_PARTITION_QUERY = """
487
529
  query hasDynamicPartition($partitionsDefName: String!, $partitionKey: String!) {
488
530
  eventLogs {
@@ -566,6 +608,30 @@ query getConcurrencyKeys {
566
608
  }
567
609
  """
568
610
 
611
+ GET_POOL_LIMITS_QUERY = """
612
+ query getPoolLimits {
613
+ eventLogs {
614
+ getPoolLimits {
615
+ name
616
+ limit
617
+ fromDefault
618
+ }
619
+ }
620
+ }
621
+ """
622
+
623
+ GET_POOL_CONFIG_QUERY = """
624
+ query getPoolConfig {
625
+ eventLogs {
626
+ getPoolConfig {
627
+ poolGranularity
628
+ defaultPoolLimit
629
+ opGranularityRunBuffer
630
+ }
631
+ }
632
+ }
633
+ """
634
+
569
635
  GET_CONCURRENCY_INFO_QUERY = """
570
636
  query getConcurrencyInfo($concurrencyKey: String!) {
571
637
  eventLogs {
@@ -583,6 +649,8 @@ query getConcurrencyInfo($concurrencyKey: String!) {
583
649
  assignedTimestamp
584
650
  priority
585
651
  }
652
+ limit
653
+ usingDefaultLimit
586
654
  }
587
655
  }
588
656
  }
@@ -705,6 +773,23 @@ query getRunStatusChangeEventRecordsQuery($recordsFilter: RunStatusChangeRecords
705
773
  """
706
774
  )
707
775
 
776
+ FETCH_FAILED_MATERIALIZATIONS_QUERY = (
777
+ EVENT_RECORD_FRAGMENT
778
+ + """
779
+ query fetchFailedMaterializationsQuery($recordsFilter: AssetRecordsFilter!, $limit: Int, $cursor: String, $ascending: Boolean) {
780
+ eventLogs {
781
+ fetchFailedMaterializations(recordsFilter: $recordsFilter, limit: $limit, cursor: $cursor, ascending: $ascending) {
782
+ records {
783
+ ...EventLogRecordFragment
784
+ }
785
+ cursor
786
+ hasMore
787
+ }
788
+ }
789
+ }
790
+ """
791
+ )
792
+
708
793
  GET_LATEST_PLANNED_MATERIALIZATION_INFO = """
709
794
  query getLatestPlannedMaterializationInfo($assetKey: String!, $partition: String) {
710
795
  eventLogs {
@@ -724,3 +809,16 @@ GET_UPDATED_DATA_VERSION_PARTITIONS = """
724
809
  }
725
810
  }
726
811
  """
812
+
813
+ GET_ASSET_STATUS_CACHE_VALUES = (
814
+ CACHED_STATUS_DATA_FRAGMENT
815
+ + """
816
+ query getAssetStatusCacheValues($assetKeys: [AssetKeyInput!]!) {
817
+ eventLogs {
818
+ getAssetStatusCacheValues(assetKeys: $assetKeys) {
819
+ ...CachedStatusDataFragment
820
+ }
821
+ }
822
+ }
823
+ """
824
+ )