mage-ai 0.9.66__py3-none-any.whl → 0.9.67__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mage-ai might be problematic. Click here for more details.

Files changed (153) hide show
  1. mage_ai/data_preparation/decorators.py +15 -0
  2. mage_ai/data_preparation/executors/streaming_pipeline_executor.py +22 -12
  3. mage_ai/data_preparation/git/api.py +3 -0
  4. mage_ai/data_preparation/git/clients/gitlab.py +3 -2
  5. mage_ai/data_preparation/models/block/__init__.py +24 -18
  6. mage_ai/data_preparation/storage/gcs_storage.py +27 -2
  7. mage_ai/data_preparation/storage/local_storage.py +18 -3
  8. mage_ai/data_preparation/storage/s3_storage.py +7 -2
  9. mage_ai/data_preparation/templates/data_loaders/streaming/generic_python.py +23 -0
  10. mage_ai/data_preparation/templates/template.py +6 -2
  11. mage_ai/data_preparation/variable_manager.py +2 -1
  12. mage_ai/io/bigquery.py +2 -0
  13. mage_ai/io/oracledb.py +14 -2
  14. mage_ai/orchestration/job_manager.py +6 -1
  15. mage_ai/server/constants.py +1 -1
  16. mage_ai/server/frontend_dist/404.html +2 -2
  17. mage_ai/server/frontend_dist/_next/static/chunks/pages/{_app-913116aa749f7ca6.js → _app-1c1ffd928f5a00f7.js} +1 -1
  18. mage_ai/server/frontend_dist/_next/static/chunks/pages/index-b7b8695a7f9efde2.js +1 -0
  19. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/{edit-d84556cefbe31a1a.js → edit-36377e679da2cd91.js} +1 -1
  20. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/logs-3f5c14076ddde20e.js +1 -0
  21. mage_ai/server/frontend_dist/_next/static/{rtugsJoijF2SsCCB5_yKS → vPsMu6Fi2zrHaf2fRXKRO}/_buildManifest.js +1 -1
  22. mage_ai/server/frontend_dist/block-layout.html +2 -2
  23. mage_ai/server/frontend_dist/compute.html +2 -2
  24. mage_ai/server/frontend_dist/files.html +2 -2
  25. mage_ai/server/frontend_dist/global-data-products/[...slug].html +2 -2
  26. mage_ai/server/frontend_dist/global-data-products.html +2 -2
  27. mage_ai/server/frontend_dist/global-hooks/[...slug].html +2 -2
  28. mage_ai/server/frontend_dist/global-hooks.html +2 -2
  29. mage_ai/server/frontend_dist/index.html +2 -2
  30. mage_ai/server/frontend_dist/manage/files.html +2 -2
  31. mage_ai/server/frontend_dist/manage/settings.html +2 -2
  32. mage_ai/server/frontend_dist/manage/users/[user].html +2 -2
  33. mage_ai/server/frontend_dist/manage/users/new.html +2 -2
  34. mage_ai/server/frontend_dist/manage/users.html +2 -2
  35. mage_ai/server/frontend_dist/manage.html +2 -2
  36. mage_ai/server/frontend_dist/oauth.html +2 -2
  37. mage_ai/server/frontend_dist/overview.html +2 -2
  38. mage_ai/server/frontend_dist/pipeline-runs.html +2 -2
  39. mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills/[...slug].html +2 -2
  40. mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills.html +2 -2
  41. mage_ai/server/frontend_dist/pipelines/[pipeline]/dashboard.html +2 -2
  42. mage_ai/server/frontend_dist/pipelines/[pipeline]/edit.html +2 -2
  43. mage_ai/server/frontend_dist/pipelines/[pipeline]/logs.html +2 -2
  44. mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runs.html +2 -2
  45. mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runtime.html +2 -2
  46. mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors.html +2 -2
  47. mage_ai/server/frontend_dist/pipelines/[pipeline]/runs/[run].html +2 -2
  48. mage_ai/server/frontend_dist/pipelines/[pipeline]/runs.html +2 -2
  49. mage_ai/server/frontend_dist/pipelines/[pipeline]/settings.html +2 -2
  50. mage_ai/server/frontend_dist/pipelines/[pipeline]/syncs.html +2 -2
  51. mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers/[...slug].html +2 -2
  52. mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers.html +2 -2
  53. mage_ai/server/frontend_dist/pipelines/[pipeline].html +2 -2
  54. mage_ai/server/frontend_dist/pipelines.html +2 -2
  55. mage_ai/server/frontend_dist/platform/global-hooks/[...slug].html +2 -2
  56. mage_ai/server/frontend_dist/platform/global-hooks.html +2 -2
  57. mage_ai/server/frontend_dist/settings/account/profile.html +2 -2
  58. mage_ai/server/frontend_dist/settings/platform/preferences.html +2 -2
  59. mage_ai/server/frontend_dist/settings/platform/settings.html +2 -2
  60. mage_ai/server/frontend_dist/settings/workspace/permissions/[...slug].html +2 -2
  61. mage_ai/server/frontend_dist/settings/workspace/permissions.html +2 -2
  62. mage_ai/server/frontend_dist/settings/workspace/preferences.html +2 -2
  63. mage_ai/server/frontend_dist/settings/workspace/roles/[...slug].html +2 -2
  64. mage_ai/server/frontend_dist/settings/workspace/roles.html +2 -2
  65. mage_ai/server/frontend_dist/settings/workspace/sync-data.html +2 -2
  66. mage_ai/server/frontend_dist/settings/workspace/users/[...slug].html +2 -2
  67. mage_ai/server/frontend_dist/settings/workspace/users.html +2 -2
  68. mage_ai/server/frontend_dist/settings.html +2 -2
  69. mage_ai/server/frontend_dist/sign-in.html +2 -2
  70. mage_ai/server/frontend_dist/templates/[...slug].html +2 -2
  71. mage_ai/server/frontend_dist/templates.html +2 -2
  72. mage_ai/server/frontend_dist/terminal.html +2 -2
  73. mage_ai/server/frontend_dist/test.html +2 -2
  74. mage_ai/server/frontend_dist/triggers.html +2 -2
  75. mage_ai/server/frontend_dist/version-control.html +2 -2
  76. mage_ai/server/frontend_dist_base_path_template/404.html +2 -2
  77. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{_app-913116aa749f7ca6.js → _app-1c1ffd928f5a00f7.js} +1 -1
  78. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/index-b7b8695a7f9efde2.js +1 -0
  79. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/{edit-d84556cefbe31a1a.js → edit-36377e679da2cd91.js} +1 -1
  80. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/logs-3f5c14076ddde20e.js +1 -0
  81. mage_ai/server/frontend_dist_base_path_template/_next/static/{wxqkF4diPqRxUfP6Ac6ai → khKiaJtwrslgMmp4YSa1f}/_buildManifest.js +1 -1
  82. mage_ai/server/frontend_dist_base_path_template/block-layout.html +2 -2
  83. mage_ai/server/frontend_dist_base_path_template/compute.html +2 -2
  84. mage_ai/server/frontend_dist_base_path_template/files.html +2 -2
  85. mage_ai/server/frontend_dist_base_path_template/global-data-products/[...slug].html +2 -2
  86. mage_ai/server/frontend_dist_base_path_template/global-data-products.html +2 -2
  87. mage_ai/server/frontend_dist_base_path_template/global-hooks/[...slug].html +2 -2
  88. mage_ai/server/frontend_dist_base_path_template/global-hooks.html +2 -2
  89. mage_ai/server/frontend_dist_base_path_template/index.html +2 -2
  90. mage_ai/server/frontend_dist_base_path_template/manage/files.html +2 -2
  91. mage_ai/server/frontend_dist_base_path_template/manage/settings.html +2 -2
  92. mage_ai/server/frontend_dist_base_path_template/manage/users/[user].html +2 -2
  93. mage_ai/server/frontend_dist_base_path_template/manage/users/new.html +2 -2
  94. mage_ai/server/frontend_dist_base_path_template/manage/users.html +2 -2
  95. mage_ai/server/frontend_dist_base_path_template/manage.html +2 -2
  96. mage_ai/server/frontend_dist_base_path_template/oauth.html +2 -2
  97. mage_ai/server/frontend_dist_base_path_template/overview.html +2 -2
  98. mage_ai/server/frontend_dist_base_path_template/pipeline-runs.html +2 -2
  99. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills/[...slug].html +2 -2
  100. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills.html +2 -2
  101. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/dashboard.html +2 -2
  102. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/edit.html +2 -2
  103. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/logs.html +2 -2
  104. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runs.html +2 -2
  105. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runtime.html +2 -2
  106. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors.html +2 -2
  107. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs/[run].html +2 -2
  108. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs.html +2 -2
  109. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/settings.html +2 -2
  110. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/syncs.html +2 -2
  111. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers/[...slug].html +2 -2
  112. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers.html +2 -2
  113. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline].html +2 -2
  114. mage_ai/server/frontend_dist_base_path_template/pipelines.html +2 -2
  115. mage_ai/server/frontend_dist_base_path_template/platform/global-hooks/[...slug].html +2 -2
  116. mage_ai/server/frontend_dist_base_path_template/platform/global-hooks.html +2 -2
  117. mage_ai/server/frontend_dist_base_path_template/settings/account/profile.html +2 -2
  118. mage_ai/server/frontend_dist_base_path_template/settings/platform/preferences.html +2 -2
  119. mage_ai/server/frontend_dist_base_path_template/settings/platform/settings.html +2 -2
  120. mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions/[...slug].html +2 -2
  121. mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions.html +2 -2
  122. mage_ai/server/frontend_dist_base_path_template/settings/workspace/preferences.html +2 -2
  123. mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles/[...slug].html +2 -2
  124. mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles.html +2 -2
  125. mage_ai/server/frontend_dist_base_path_template/settings/workspace/sync-data.html +2 -2
  126. mage_ai/server/frontend_dist_base_path_template/settings/workspace/users/[...slug].html +2 -2
  127. mage_ai/server/frontend_dist_base_path_template/settings/workspace/users.html +2 -2
  128. mage_ai/server/frontend_dist_base_path_template/settings.html +2 -2
  129. mage_ai/server/frontend_dist_base_path_template/sign-in.html +2 -2
  130. mage_ai/server/frontend_dist_base_path_template/templates/[...slug].html +2 -2
  131. mage_ai/server/frontend_dist_base_path_template/templates.html +2 -2
  132. mage_ai/server/frontend_dist_base_path_template/terminal.html +2 -2
  133. mage_ai/server/frontend_dist_base_path_template/test.html +2 -2
  134. mage_ai/server/frontend_dist_base_path_template/triggers.html +2 -2
  135. mage_ai/server/frontend_dist_base_path_template/version-control.html +2 -2
  136. mage_ai/services/aws/s3/s3.py +8 -2
  137. mage_ai/settings/server.py +6 -1
  138. mage_ai/shared/logger.py +12 -6
  139. mage_ai/streaming/sources/base_python.py +30 -0
  140. mage_ai/streaming/sources/source_factory.py +25 -0
  141. mage_ai/tests/data_preparation/models/test_block.py +39 -2
  142. {mage_ai-0.9.66.dist-info → mage_ai-0.9.67.dist-info}/METADATA +1 -1
  143. {mage_ai-0.9.66.dist-info → mage_ai-0.9.67.dist-info}/RECORD +149 -147
  144. mage_ai/server/frontend_dist/_next/static/chunks/pages/index-64851458dde54ad9.js +0 -1
  145. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/logs-cf656cbe37ecaacc.js +0 -1
  146. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/index-64851458dde54ad9.js +0 -1
  147. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/logs-cf656cbe37ecaacc.js +0 -1
  148. /mage_ai/server/frontend_dist/_next/static/{rtugsJoijF2SsCCB5_yKS → vPsMu6Fi2zrHaf2fRXKRO}/_ssgManifest.js +0 -0
  149. /mage_ai/server/frontend_dist_base_path_template/_next/static/{wxqkF4diPqRxUfP6Ac6ai → khKiaJtwrslgMmp4YSa1f}/_ssgManifest.js +0 -0
  150. {mage_ai-0.9.66.dist-info → mage_ai-0.9.67.dist-info}/LICENSE +0 -0
  151. {mage_ai-0.9.66.dist-info → mage_ai-0.9.67.dist-info}/WHEEL +0 -0
  152. {mage_ai-0.9.66.dist-info → mage_ai-0.9.67.dist-info}/entry_points.txt +0 -0
  153. {mage_ai-0.9.66.dist-info → mage_ai-0.9.67.dist-info}/top_level.txt +0 -0
@@ -103,3 +103,18 @@ def data_integration_query(function):
103
103
 
104
104
  def preprocesser_functions(function):
105
105
  return function
106
+
107
+
108
+ def streaming_source(cls):
109
+ return cls
110
+
111
+
112
+ def collect_decorated_objs(decorated_objs):
113
+ """
114
+ Method to collect the decorated objects (function or class)
115
+ """
116
+ def custom_code(obj):
117
+ decorated_objs.append(obj)
118
+ return obj
119
+
120
+ return custom_code
@@ -10,7 +10,7 @@ from jinja2 import Template
10
10
 
11
11
  from mage_ai.data_preparation.executors.pipeline_executor import PipelineExecutor
12
12
  from mage_ai.data_preparation.logging.logger import DictLogger
13
- from mage_ai.data_preparation.models.constants import BlockType
13
+ from mage_ai.data_preparation.models.constants import BlockLanguage, BlockType
14
14
  from mage_ai.data_preparation.models.pipeline import Pipeline
15
15
  from mage_ai.data_preparation.shared.stream import StreamToLogger
16
16
  from mage_ai.data_preparation.shared.utils import get_template_vars
@@ -107,18 +107,28 @@ class StreamingPipelineExecutor(PipelineExecutor):
107
107
 
108
108
  if global_vars is None:
109
109
  global_vars = dict()
110
- source_config = self.__interpolate_vars(
111
- self.source_block.content,
112
- global_vars=global_vars,
113
- )
114
- source = SourceFactory.get_source(
115
- source_config,
116
- checkpoint_path=os.path.join(
117
- self.pipeline.pipeline_variables_dir,
118
- 'streaming_checkpoint',
119
- ),
120
- )
121
110
 
111
+ # Initialize source block
112
+ if self.source_block.language == BlockLanguage.PYTHON:
113
+ source = SourceFactory.get_python_source(
114
+ self.source_block.content,
115
+ global_vars=global_vars,
116
+ )
117
+ else:
118
+ # Default to YAML config
119
+ source_config = self.__interpolate_vars(
120
+ self.source_block.content,
121
+ global_vars=global_vars,
122
+ )
123
+ source = SourceFactory.get_source(
124
+ source_config,
125
+ checkpoint_path=os.path.join(
126
+ self.pipeline.pipeline_variables_dir,
127
+ 'streaming_checkpoint',
128
+ ),
129
+ )
130
+
131
+ # Initialize destination blocks
122
132
  sinks_by_uuid = dict()
123
133
  for sink_block in self.sink_blocks:
124
134
  sinks_by_uuid[sink_block.uuid] = SinkFactory.get_sink(
@@ -15,8 +15,10 @@ from mage_ai.data_preparation.git.utils import (
15
15
  )
16
16
  from mage_ai.data_preparation.repo_manager import get_project_uuid
17
17
  from mage_ai.orchestration.db.models.oauth import Oauth2AccessToken, User
18
+ from mage_ai.server.logger import Logger
18
19
 
19
20
  API_ENDPOINT = 'https://api.github.com'
21
+ logger = Logger().new_server_logger(__name__)
20
22
 
21
23
 
22
24
  def get_oauth_client_id(provider: str) -> str:
@@ -262,4 +264,5 @@ def get_user(token: str, provider: str = ProviderName.GITHUB) -> Dict:
262
264
  try:
263
265
  return GitClient.get_client_for_provider(provider)(token).get_user()
264
266
  except Exception:
267
+ logger.exception('Error fetching user from git provider.')
265
268
  return dict()
@@ -3,14 +3,15 @@ from urllib.parse import quote_plus
3
3
 
4
4
  import requests
5
5
 
6
- from mage_ai.authentication.oauth.constants import GITLAB_HOST
7
6
  from mage_ai.data_preparation.git.clients.base import Client
7
+ from mage_ai.settings import get_settings_value
8
+ from mage_ai.settings.keys import GITLAB_HOST
8
9
 
9
10
 
10
11
  class GitlabClient(Client):
11
12
  def __init__(self, access_token: str):
12
13
  super().__init__(access_token)
13
- self.hostname = GITLAB_HOST or 'https://gitlab.com'
14
+ self.hostname = get_settings_value(GITLAB_HOST, 'https://gitlab.com')
14
15
 
15
16
  def get_user(self):
16
17
  # For GitLab, we don't need to make a request to the API to get the username
@@ -339,7 +339,6 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
339
339
  self.upstream_blocks = []
340
340
  self.downstream_blocks = []
341
341
  self.test_functions = []
342
- self.global_vars = {}
343
342
  self.template_runtime_configuration = {}
344
343
 
345
344
  self.dynamic_block_index = None
@@ -1198,7 +1197,7 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
1198
1197
  f'Please run upstream blocks {upstream_block_uuids} '
1199
1198
  'before running the current block.'
1200
1199
  )
1201
- global_vars = self.__enrich_global_vars(
1200
+ global_vars = self.enrich_global_vars(
1202
1201
  global_vars,
1203
1202
  dynamic_block_index=dynamic_block_index,
1204
1203
  )
@@ -2204,6 +2203,7 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
2204
2203
  self.pipeline.uuid,
2205
2204
  block_uuid,
2206
2205
  partition=execution_partition,
2206
+ max_results=DATAFRAME_SAMPLE_COUNT_PREVIEW,
2207
2207
  )
2208
2208
 
2209
2209
  if not include_print_outputs:
@@ -3037,7 +3037,7 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
3037
3037
  variable_mapping = merge_dict(output_variables, consolidated_print_variables)
3038
3038
  return variable_mapping
3039
3039
 
3040
- def __enrich_global_vars(
3040
+ def enrich_global_vars(
3041
3041
  self,
3042
3042
  global_vars: Dict = None,
3043
3043
  dynamic_block_index: int = None,
@@ -3658,23 +3658,29 @@ class AddonBlock(Block):
3658
3658
  global_vars['dynamic_block_index'] = dynamic_block_index
3659
3659
 
3660
3660
  if parent_block:
3661
+ if parent_block.global_vars is None:
3662
+ global_vars_copy = global_vars.copy()
3663
+ parent_block.enrich_global_vars(
3664
+ global_vars=global_vars_copy,
3665
+ dynamic_block_index=dynamic_block_index,
3666
+ )
3667
+ global_vars = merge_dict(parent_block.global_vars, global_vars)
3661
3668
  global_vars['parent_block_uuid'] = parent_block.uuid
3662
3669
 
3663
- if parent_block and \
3664
- parent_block.pipeline and \
3665
- PipelineType.INTEGRATION == parent_block.pipeline.type:
3666
-
3667
- template_runtime_configuration = parent_block.template_runtime_configuration
3668
- index = template_runtime_configuration.get('index', None)
3669
- is_last_block_run = template_runtime_configuration.get('is_last_block_run', False)
3670
- selected_streams = template_runtime_configuration.get('selected_streams', [])
3671
- stream = selected_streams[0] if len(selected_streams) >= 1 else None
3672
- destination_table = template_runtime_configuration.get('destination_table', stream)
3673
-
3674
- global_vars['index'] = index
3675
- global_vars['is_last_block_run'] = is_last_block_run
3676
- global_vars['stream'] = stream
3677
- global_vars['destination_table'] = destination_table
3670
+ if parent_block.pipeline and \
3671
+ PipelineType.INTEGRATION == parent_block.pipeline.type:
3672
+
3673
+ template_runtime_configuration = parent_block.template_runtime_configuration
3674
+ index = template_runtime_configuration.get('index', None)
3675
+ is_last_block_run = template_runtime_configuration.get('is_last_block_run', False)
3676
+ selected_streams = template_runtime_configuration.get('selected_streams', [])
3677
+ stream = selected_streams[0] if len(selected_streams) >= 1 else None
3678
+ destination_table = template_runtime_configuration.get('destination_table', stream)
3679
+
3680
+ global_vars['index'] = index
3681
+ global_vars['is_last_block_run'] = is_last_block_run
3682
+ global_vars['stream'] = stream
3683
+ global_vars['destination_table'] = destination_table
3678
3684
 
3679
3685
  return global_vars
3680
3686
 
@@ -29,11 +29,36 @@ class GCSStorage(BaseStorage):
29
29
  path += '/'
30
30
  return self.path_exists(path)
31
31
 
32
- def listdir(self, path: str, suffix: str = None) -> List[str]:
32
+ def listdir(
33
+ self,
34
+ path: str,
35
+ suffix: str = None,
36
+ max_results: int = None,
37
+ ) -> List[str]:
33
38
  if not path.endswith('/'):
34
39
  path += '/'
35
40
  path = gcs_url_path(path)
36
- blobs = self.bucket.list_blobs(prefix=path)
41
+
42
+ """
43
+ NOTE: The number of results returned by the GCS bucket list_blobs method may
44
+ be lower than the number entered for the max_results argument, since this
45
+ method recursively fetches files inside of subdirectories at the path
46
+ specified, not just the directories themselves at the top level of the path.
47
+
48
+ As a result, when fetching block output folders in a remote variables directory,
49
+ the json or parquet files inside of the block output folders (e.g. the "type.json"
50
+ file in folder "output_0") are counted towards the "max_results" limit. If max_results
51
+ is set to 10, less than 10 output folders may be returned due to there being multiple
52
+ json or other files in each output folder. We do not increase the max_results or
53
+ include a "delimiter" argument because it would increase load times to unacceptable levels.
54
+
55
+ Example block output path in GCS:
56
+ gs://mage_demo/pipelines/example_pipeline/.variables/example_block/output_0/
57
+ """
58
+ blobs = self.bucket.list_blobs(
59
+ prefix=path,
60
+ max_results=max_results,
61
+ )
37
62
  keys = []
38
63
  for blob in blobs:
39
64
  # Avoid finding files recursevively in the dir path.
@@ -19,12 +19,27 @@ class LocalStorage(BaseStorage):
19
19
  def isdir(self, path: str) -> bool:
20
20
  return os.path.isdir(path)
21
21
 
22
- def listdir(self, path: str, suffix: str = None) -> List[str]:
22
+ def listdir(
23
+ self,
24
+ path: str,
25
+ suffix: str = None,
26
+ max_results: int = None,
27
+ ) -> List[str]:
28
+ paths = []
23
29
  if not os.path.exists(path):
24
- return []
25
- paths = os.listdir(path)
30
+ return paths
31
+
32
+ if max_results is not None:
33
+ with os.scandir(path) as it:
34
+ for idx, entry in enumerate(it):
35
+ paths.append(entry.name)
36
+ if idx >= max_results - 1:
37
+ break
38
+ else:
39
+ paths = os.listdir(path)
26
40
  if suffix is not None:
27
41
  paths = [p for p in paths if p.endswith(suffix)]
42
+
28
43
  return paths
29
44
 
30
45
  def makedirs(self, path: str, **kwargs) -> None:
@@ -30,11 +30,16 @@ class S3Storage(BaseStorage):
30
30
  path += '/'
31
31
  return self.path_exists(path)
32
32
 
33
- def listdir(self, path: str, suffix: str = None) -> List[str]:
33
+ def listdir(
34
+ self,
35
+ path: str,
36
+ suffix: str = None,
37
+ max_results: int = None,
38
+ ) -> List[str]:
34
39
  if not path.endswith('/'):
35
40
  path += '/'
36
41
  path = s3_url_path(path)
37
- keys = self.client.listdir(path, suffix=suffix)
42
+ keys = self.client.listdir(path, suffix=suffix, max_results=max_results)
38
43
  return [k[len(path):].rstrip('/') for k in keys]
39
44
 
40
45
  def makedirs(self, path: str, **kwargs) -> None:
@@ -0,0 +1,23 @@
1
+ from mage_ai.streaming.sources.base_python import BasePythonSource
2
+ from typing import Callable
3
+
4
+ if 'streaming_source' not in globals():
5
+ from mage_ai.data_preparation.decorators import streaming_source
6
+
7
+
8
+ @streaming_source
9
+ class CustomSource(BasePythonSource):
10
+ def init_client(self):
11
+ """
12
+ Implement the logic of initializing the client.
13
+ """
14
+
15
+ def batch_read(self, handler: Callable):
16
+ """
17
+ Batch read the messages from the source and use handler to process the messages.
18
+ """
19
+ while True:
20
+ records = []
21
+ # Implement the logic of fetching the records
22
+ if len(records) > 0:
23
+ handler(records)
@@ -140,19 +140,23 @@ def __fetch_data_loader_templates(
140
140
  pipeline_type: PipelineType = PipelineType.PYTHON,
141
141
  ) -> str:
142
142
  data_source = config.get('data_source')
143
+ default_template_name = None
143
144
  file_extension = 'py'
144
145
  if pipeline_type == PipelineType.PYSPARK:
145
146
  template_folder = 'data_loaders/pyspark'
146
147
  elif pipeline_type == PipelineType.STREAMING:
147
148
  template_folder = 'data_loaders/streaming'
148
- file_extension = 'yaml'
149
+ if language == BlockLanguage.YAML:
150
+ file_extension = 'yaml'
151
+ else:
152
+ default_template_name = 'generic_python.py'
149
153
  elif language == BlockLanguage.R:
150
154
  template_folder = 'data_loaders/r'
151
155
  file_extension = 'r'
152
156
  else:
153
157
  template_folder = 'data_loaders'
154
158
 
155
- default_template = template_folder + '/' + 'default.jinja'
159
+ default_template = template_folder + '/' + (default_template_name or 'default.jinja')
156
160
  if data_source is None:
157
161
  template_path = default_template
158
162
  else:
@@ -271,6 +271,7 @@ class VariableManager:
271
271
  block_uuid: str,
272
272
  partition: str = None,
273
273
  clean_block_uuid: bool = True,
274
+ max_results: int = None,
274
275
  ) -> List[str]:
275
276
  variable_dir_path = os.path.join(
276
277
  self.pipeline_path(pipeline_uuid),
@@ -280,7 +281,7 @@ class VariableManager:
280
281
  )
281
282
  if not self.storage.path_exists(variable_dir_path):
282
283
  return []
283
- variables = self.storage.listdir(variable_dir_path)
284
+ variables = self.storage.listdir(variable_dir_path, max_results=max_results)
284
285
  return sorted([v.split('.')[0] for v in variables])
285
286
 
286
287
  def pipeline_path(self, pipeline_uuid: str) -> str:
mage_ai/io/bigquery.py CHANGED
@@ -1,5 +1,6 @@
1
1
  from typing import Dict, List, Mapping, Union
2
2
 
3
+ import numpy as np
3
4
  from google.cloud.bigquery import (
4
5
  Client,
5
6
  LoadJobConfig,
@@ -295,6 +296,7 @@ WHERE table_id = '{table_name}'
295
296
  column_types = self.get_column_types(schema, table_name)
296
297
 
297
298
  if df is not None:
299
+ df.fillna(value=np.NaN, inplace=True)
298
300
  for col in df.columns:
299
301
  col_type = column_types.get(col)
300
302
  if not col_type:
mage_ai/io/oracledb.py CHANGED
@@ -7,6 +7,9 @@ from pandas import DataFrame, read_sql
7
7
  from mage_ai.io.base import QUERY_ROW_LIMIT
8
8
  from mage_ai.io.config import BaseConfigLoader, ConfigKey
9
9
  from mage_ai.io.sql import BaseSQL
10
+ from mage_ai.server.logger import Logger
11
+
12
+ logger = Logger().new_server_logger(__name__)
10
13
 
11
14
 
12
15
  class OracleDB(BaseSQL):
@@ -40,8 +43,17 @@ class OracleDB(BaseSQL):
40
43
  )
41
44
 
42
45
  def open(self) -> None:
43
- with self.printer.print_msg('Opening connection to OracleDB database (thin mode)'):
44
- self._ctx = oracledb.connect(**self.settings)
46
+ if self.settings['mode'] and self.settings['mode'].lower() == 'thick':
47
+ logger.info('Initializing Oracle thick mode.')
48
+ oracledb.init_oracle_client()
49
+ with self.printer.print_msg(f'Opening connection to OracleDB database \
50
+ ({self.settings["mode"]} mode)'):
51
+ connection_dsn = "{}:{}/{}".format(
52
+ self.settings['host'],
53
+ self.settings['port'],
54
+ self.settings['service_name'])
55
+ self._ctx = oracledb.connect(
56
+ user=self.settings['user'], password=self.settings['password'], dsn=connection_dsn)
45
57
 
46
58
  def load(
47
59
  self,
@@ -1,3 +1,4 @@
1
+ import traceback
1
2
  from enum import Enum
2
3
  from typing import Callable, Dict, Union
3
4
 
@@ -89,4 +90,8 @@ class JobManager:
89
90
  return f'{job_type}_{uid}'
90
91
 
91
92
 
92
- job_manager = JobManager()
93
+ try:
94
+ job_manager = JobManager()
95
+ except Exception:
96
+ traceback.print_exc()
97
+ job_manager = None
@@ -12,4 +12,4 @@ DATAFRAME_OUTPUT_SAMPLE_COUNT = 10
12
12
  # Dockerfile depends on it because it runs ./scripts/install_mage.sh and uses
13
13
  # the last line to determine the version to install.
14
14
  VERSION = \
15
- '0.9.66'
15
+ '0.9.67'
@@ -1,4 +1,4 @@
1
- <!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><title>Mage</title><meta content="width=device-width, initial-scale=1.0, maximum-scale=5.0, user-scalable=0" name="viewport"/><meta name="next-head-count" content="3"/><link href="/favicon.ico" rel="icon"/><link rel="preload" href="/_next/static/css/b59541b123fd7191.css" as="style"/><link rel="stylesheet" href="/_next/static/css/b59541b123fd7191.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js"></script><script src="/_next/static/chunks/webpack-d079359c241db804.js" defer=""></script><script src="/_next/static/chunks/framework-22b71764bf44ede4.js" defer=""></script><script src="/_next/static/chunks/main-77fe248a6fbd12d8.js" defer=""></script><script src="/_next/static/chunks/pages/_app-913116aa749f7ca6.js" defer=""></script><script src="/_next/static/chunks/pages/404-8ecd93274c427b76.js" defer=""></script><script src="/_next/static/rtugsJoijF2SsCCB5_yKS/_buildManifest.js" defer=""></script><script src="/_next/static/rtugsJoijF2SsCCB5_yKS/_ssgManifest.js" defer=""></script><style data-styled="" data-styled-version="5.3.6">html{-webkit-box-sizing:border-box;box-sizing:border-box;-ms-overflow-style:scrollbar;}/*!sc*/
1
+ <!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><title>Mage</title><meta content="width=device-width, initial-scale=1.0, maximum-scale=5.0, user-scalable=0" name="viewport"/><meta name="next-head-count" content="3"/><link href="/favicon.ico" rel="icon"/><link rel="preload" href="/_next/static/css/b59541b123fd7191.css" as="style"/><link rel="stylesheet" href="/_next/static/css/b59541b123fd7191.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js"></script><script src="/_next/static/chunks/webpack-d079359c241db804.js" defer=""></script><script src="/_next/static/chunks/framework-22b71764bf44ede4.js" defer=""></script><script src="/_next/static/chunks/main-77fe248a6fbd12d8.js" defer=""></script><script src="/_next/static/chunks/pages/_app-1c1ffd928f5a00f7.js" defer=""></script><script src="/_next/static/chunks/pages/404-8ecd93274c427b76.js" defer=""></script><script src="/_next/static/vPsMu6Fi2zrHaf2fRXKRO/_buildManifest.js" defer=""></script><script src="/_next/static/vPsMu6Fi2zrHaf2fRXKRO/_ssgManifest.js" defer=""></script><style data-styled="" data-styled-version="5.3.6">html{-webkit-box-sizing:border-box;box-sizing:border-box;-ms-overflow-style:scrollbar;}/*!sc*/
2
2
  *,*::before,*::after{-webkit-box-sizing:inherit;box-sizing:inherit;}/*!sc*/
3
3
  data-styled.g4[id="sc-global-czSCUT1"]{content:"sc-global-czSCUT1,"}/*!sc*/
4
4
  .cAAVhx{margin-top:16px;margin-bottom:16px;}/*!sc*/
@@ -17,4 +17,4 @@ data-styled.g79[id="Headline__H1Style-sc-12jzt2e-2"]{content:"hkcwCQ,"}/*!sc*/
17
17
  .kOVcuR .Toastify__toast--success{background:#00A81A !important;color:#FFFFFF !important;}/*!sc*/
18
18
  .kOVcuR .Toastify__toast--warning{background:#DD9900 !important;color:#FFFFFF !important;}/*!sc*/
19
19
  data-styled.g275[id="ToastWrapper-sc-1a33ph1-0"]{content:"kOVcuR,"}/*!sc*/
20
- </style></head><body><div id="__next"><div class="" style="position:fixed;top:0;left:0;height:2px;background:transparent;z-index:99999999999;width:100%"><div class="" style="height:100%;background:#FF144D;transition:all 500ms ease;width:0%"><div style="box-shadow:0 0 10px #FF144D, 0 0 10px #FF144D;width:5%;opacity:1;position:absolute;height:100%;transition:all 500ms ease;transform:rotate(3deg) translate(0px, -4px);left:-10rem"></div></div></div><main style="align-items:center;display:flex;flex-direction:column;height:100vh;margin-top:16rem"><svg width="84.70857142857143" height="64" viewBox="0 0 20 14" fill="none" xmlns="http://www.w3.org/2000/svg"><path opacity="0.4" d="M15.3266 0L19.2641 1.82961e-06L11.9687 14L8.03125 14L15.3266 0Z" fill="url(#paint0_linear_1303_5)"></path><path d="M11.9692 1.82961e-06L8.03164 0L0.736328 14L4.67383 14L8.03164 7.55626V14H11.9691V8.38194e-05L11.9692 1.82961e-06Z" fill="url(#paint1_linear_1303_5)"></path><path d="M15.3269 2.57679e-06H19.2644V14H15.3269V2.57679e-06Z" fill="url(#paint2_linear_1303_5)"></path><defs><linearGradient id="paint0_linear_1303_5" x1="8.03125" y1="7" x2="19.2641" y2="7" gradientUnits="userSpaceOnUse"><stop offset="0.28125" stop-color="#7D55EC"></stop><stop offset="1" stop-color="#2AB2FE"></stop></linearGradient><linearGradient id="paint1_linear_1303_5" x1="0.736328" y1="7" x2="19.2644" y2="7" gradientUnits="userSpaceOnUse"><stop offset="0.28125" stop-color="#7D55EC"></stop><stop offset="1" stop-color="#2AB2FE"></stop></linearGradient><linearGradient id="paint2_linear_1303_5" x1="0.736328" y1="7" x2="19.2644" y2="7" gradientUnits="userSpaceOnUse"><stop offset="0.28125" stop-color="#7D55EC"></stop><stop offset="1" stop-color="#2AB2FE"></stop></linearGradient></defs></svg><div class="Spacing__SpacingStyle-sc-1mpmtgj-0 cAAVhx"><div class="Headline__HeadlineContainerStyle-sc-12jzt2e-0 dDWSWx"><h1 class="Headline__H1Style-sc-12jzt2e-2 hkcwCQ">404 - Page Not Found<!-- --></h1></div></div><p class="Text__TextStyle-sc-sl5nur-0 fmHTvj">You will be redirected to the Pipelines dashboard in <!-- -->5<!-- --> seconds.<!-- --></p></main><div id="command-center-root"></div><div></div><div></div><div></div><div class="ToastWrapper-sc-1a33ph1-0 kOVcuR"><div class="Toastify"></div></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{},"currentTheme":{"accent":{"alert":"#F6540B","blue":"#4877FF","blueLight":"rgba(72, 119, 255, 0.5)","contentDefaultTransparent":"rgba(174, 174, 174, 0.5)","cyan":"#65E3FF","cyanLight":"rgba(101, 227, 255, 0.3)","cyanTransparent":"rgba(101, 227, 255, 0.12)","dbt":"#fc6949","dbtDark":"rgba(252, 105, 73, 0.3)","dbtLight":"rgba(252, 105, 73, 0.5)","info":"#00ABFF","infoTransparent":"rgba(0, 171, 255, 0.5)","negative":"#FF1E59","negativeTransparent":"rgba(255, 30, 89, 0.3)","pink":"#FF4FF8","pinkLight":"rgb(255, 79, 248, 0.5)","positive":"#00A81A","primaryTransparent":"rgba(155, 108, 167, 0.5)","purple":"#7D55EC","purpleLight":"rgba(125, 85, 236, 0.5)","rose":"#D1A2AB","roseLight":"rgba(209, 162, 171, 0.5)","sky":"#6AA1E0","skyLight":"rgba(106, 161, 224, 0.5)","teal":"#00B4CC","tealLight":"rgba(0, 180, 204, 0.5)","warning":"#DD9900","warningTransparent":"rgba(221, 153, 0, 0.5)","yellow":"#FFCC19","yellowLight":"rgba(255, 204, 25, 0.5)"},"background":{"blackTransparent":"rgba(0, 0, 0, 0.6)","blackTransparentDark":"rgba(0, 0, 0, 0.8)","chartBlock":"#2E3036","codeArea":"#1E1F24","codeTextarea":"#000000","content":"#1B1C20","danger":"#FFD0DB","dark":"#B1B8C3","dashboard":"#18181C","dashboardTransparent":"rgba(24, 24, 28, 0.1)","header":"#1B1B1B","menu":"#0F4CFF","muted":"#F9FAFC","navigation":"#EDEDED","output":"#2E3036","page":"#1E1F24","panel":"#232429","panelTransparent":"rgba(35, 36, 41, 0.7)","popup":"#27292E","row":"#2C2C2C","row2":"#51535C","scrollbarThumb":"rgba(100, 100, 100, 0.5)","scrollbarThumbHover":"rgba(255, 255, 255, 0.3)","scrollbarTrack":"#2E3036","success":"#8ADE00","successLight":"rgb(138, 222, 0, 0.3)","table":"#292A2F","tag":"#3A4550"},"borders":{"button":"#454850","contrast":"#FFFFFF","danger":"#FF144D","dark":"#000000","darkLight":"#2E3036","info":"#FFCC19","light":"#2F3034","medium":"#1C1C1C","medium2":"#141414","success":"#2FCB52"},"brand":{"earth100":"#C6EEDB","earth200":"#9DDFBF","earth300":"#6BBF96","earth400":"#37A46F","earth400Transparent":"rgba(55, 164, 111, 0.4)","earth500":"#00954C","energy100":"#FFF4BA","energy200":"#FFED92","energy300":"#FFE662","energy400":"#FFDA19","energy400Transparent":"rgba(255, 218, 25, 0.04)","energy500":"#F6C000","fire100":"#FFD7E0","fire200":"#FFA3B9","fire300":"#FF547D","fire400":"#FF144D","fire400Transparent":"rgba(255, 20, 77, 0.4)","fire500":"#EB0032","stone100":"#F3E6D7","stone200":"#E3D4C2","stone400":"#BFA78B","stone500":"#AF8859","water100":"#BDCEFF","water200":"#81A1FF","water300":"#517DFF","water400":"#2A60FE","water400Transparent":"rgba(42, 96, 254, 0.4)","water500":"#0F4CFF","wind100":"#EEEAFF","wind200":"#CCC1F4","wind300":"#A698DD","wind400":"#6B50D7","wind400SuperTransparent":"rgba(107, 80, 215, 0.12)","wind400Transparent":"rgba(107, 80, 215, 0.4)","wind500":"#4E32BC"},"chart":{"backgroundPrimary":"#7D55EC","backgroundSecondary":"#FF144D","backgroundTertiary":"#86E2FF","button1":"#4877FF","button2":"#FFCC19","button3":"#8ADE00","button4":"#FF4FF8","button5":"#B98D95","lines":"#9B6CA7","primary":"#6B50D7","secondary":"#FF144D","tertiary":"#2A60FE"},"content":{"active":"#FFFFFF","default":"#AEAEAE","disabled":"rgba(255, 255, 255, 0.3)","inverted":"#2C2C2C","muted":"#787A85"},"elevation":{"visualizationAccent":"#996CFF","visualizationAccentAlt":"#C1ACF7"},"feature":{"active":"rgba(250, 248, 254, 0.14)","disabled":"rgba(201, 206, 218, 0.12)"},"icons":{"neutral":"#787878"},"interactive":{"activeBorder":"#060606","blackBackgroundTransparent":"rgba(0, 0, 0, 0.5)","checked":"#060606","dangerBorder":"#FF144D","defaultBackground":"#36383F","defaultBackgroundTransparent":"rgba(54, 56, 63, 0.5)","defaultBorder":"#ffffff1a","disabledBorder":"#B1B8C3","focusBackground":"#B1B8C3","focusBorder":"#86E2FF","hoverBackground":"#4E4E4E","hoverBackgroundTransparent":"rgba(78, 78, 78, 0.5)","hoverBlackBackgroundTransparent":"rgba(0, 0, 0, 0.7)","hoverBorder":"#B9BFCA","hoverOverlay":"rgba(255, 255, 255, 0.1)","linkPrimary":"#1752FF","linkPrimaryHover":"#4877FF","linkPrimaryLight":"#5982ff","linkSecondary":"#6B50D7","linkSecondaryDisabled":"#C4B9EF","linkText":"#6AA1E0","linkTextLight":"#9ECBFF","purple":"#885EFF","rowHoverBackground":"rgba(0, 0, 0, 0.1)","transparent":"rgba(255, 255, 255, 0)"},"loader":{"color":"#EB0032","colorInverted":"#8ADE00"},"logo":{"color":"#FFFFFF"},"monotone":{"black":"#060606","blackTransparent":"rgba(0, 0, 0, 0.6)","gray":"#B1B8C3","grey100":"#F2F2F2","grey200":"#D5D7DC","grey300":"#B4B8C0","grey400":"#70747C","grey500":"#51535C","purple":"#6B50D7","white":"#FFFFFF","whiteTransparent":"rgba(255, 255, 255, 0.6)"},"neutral":{"n100":"#E7E8EA","n200":"#D8DADE","n300":"#CBCCD0","n400":"#BCBEC4","n500":"#AEB0B6"},"progress":{"negative":"#FF144D","positive":"#6B50D7"},"shadow":{"base":"12px 40px 120px rgba(0, 0, 0, 0.3)","frame":"0px 10px 40px rgba(0, 0, 0, 0.26)","menu":"4px 10px 20px rgba(6, 6, 6, 0.12)","popup":"10px 20px 40px rgba(0, 0, 0, 0.2)","small":"0px, 4px, rgba(0, 0, 0, 0.25)","window":"0px 10px 60px rgba(0, 0, 0, 0.7)"},"status":{"negative":"#FF144D","positive":"#24B400"},"text":{"fileBrowser":"#787A85"}}},"page":"/404","query":{},"buildId":"rtugsJoijF2SsCCB5_yKS","nextExport":true,"isFallback":false,"appGip":true,"scriptLoader":[]}</script></body></html>
20
+ </style></head><body><div id="__next"><div class="" style="position:fixed;top:0;left:0;height:2px;background:transparent;z-index:99999999999;width:100%"><div class="" style="height:100%;background:#FF144D;transition:all 500ms ease;width:0%"><div style="box-shadow:0 0 10px #FF144D, 0 0 10px #FF144D;width:5%;opacity:1;position:absolute;height:100%;transition:all 500ms ease;transform:rotate(3deg) translate(0px, -4px);left:-10rem"></div></div></div><main style="align-items:center;display:flex;flex-direction:column;height:100vh;margin-top:16rem"><svg width="84.70857142857143" height="64" viewBox="0 0 20 14" fill="none" xmlns="http://www.w3.org/2000/svg"><path opacity="0.4" d="M15.3266 0L19.2641 1.82961e-06L11.9687 14L8.03125 14L15.3266 0Z" fill="url(#paint0_linear_1303_5)"></path><path d="M11.9692 1.82961e-06L8.03164 0L0.736328 14L4.67383 14L8.03164 7.55626V14H11.9691V8.38194e-05L11.9692 1.82961e-06Z" fill="url(#paint1_linear_1303_5)"></path><path d="M15.3269 2.57679e-06H19.2644V14H15.3269V2.57679e-06Z" fill="url(#paint2_linear_1303_5)"></path><defs><linearGradient id="paint0_linear_1303_5" x1="8.03125" y1="7" x2="19.2641" y2="7" gradientUnits="userSpaceOnUse"><stop offset="0.28125" stop-color="#7D55EC"></stop><stop offset="1" stop-color="#2AB2FE"></stop></linearGradient><linearGradient id="paint1_linear_1303_5" x1="0.736328" y1="7" x2="19.2644" y2="7" gradientUnits="userSpaceOnUse"><stop offset="0.28125" stop-color="#7D55EC"></stop><stop offset="1" stop-color="#2AB2FE"></stop></linearGradient><linearGradient id="paint2_linear_1303_5" x1="0.736328" y1="7" x2="19.2644" y2="7" gradientUnits="userSpaceOnUse"><stop offset="0.28125" stop-color="#7D55EC"></stop><stop offset="1" stop-color="#2AB2FE"></stop></linearGradient></defs></svg><div class="Spacing__SpacingStyle-sc-1mpmtgj-0 cAAVhx"><div class="Headline__HeadlineContainerStyle-sc-12jzt2e-0 dDWSWx"><h1 class="Headline__H1Style-sc-12jzt2e-2 hkcwCQ">404 - Page Not Found<!-- --></h1></div></div><p class="Text__TextStyle-sc-sl5nur-0 fmHTvj">You will be redirected to the Pipelines dashboard in <!-- -->5<!-- --> seconds.<!-- --></p></main><div id="command-center-root"></div><div></div><div></div><div></div><div class="ToastWrapper-sc-1a33ph1-0 kOVcuR"><div class="Toastify"></div></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{},"currentTheme":{"accent":{"alert":"#F6540B","blue":"#4877FF","blueLight":"rgba(72, 119, 255, 0.5)","contentDefaultTransparent":"rgba(174, 174, 174, 0.5)","cyan":"#65E3FF","cyanLight":"rgba(101, 227, 255, 0.3)","cyanTransparent":"rgba(101, 227, 255, 0.12)","dbt":"#fc6949","dbtDark":"rgba(252, 105, 73, 0.3)","dbtLight":"rgba(252, 105, 73, 0.5)","info":"#00ABFF","infoTransparent":"rgba(0, 171, 255, 0.5)","negative":"#FF1E59","negativeTransparent":"rgba(255, 30, 89, 0.3)","pink":"#FF4FF8","pinkLight":"rgb(255, 79, 248, 0.5)","positive":"#00A81A","primaryTransparent":"rgba(155, 108, 167, 0.5)","purple":"#7D55EC","purpleLight":"rgba(125, 85, 236, 0.5)","rose":"#D1A2AB","roseLight":"rgba(209, 162, 171, 0.5)","sky":"#6AA1E0","skyLight":"rgba(106, 161, 224, 0.5)","teal":"#00B4CC","tealLight":"rgba(0, 180, 204, 0.5)","warning":"#DD9900","warningTransparent":"rgba(221, 153, 0, 0.5)","yellow":"#FFCC19","yellowLight":"rgba(255, 204, 25, 0.5)"},"background":{"blackTransparent":"rgba(0, 0, 0, 0.6)","blackTransparentDark":"rgba(0, 0, 0, 0.8)","chartBlock":"#2E3036","codeArea":"#1E1F24","codeTextarea":"#000000","content":"#1B1C20","danger":"#FFD0DB","dark":"#B1B8C3","dashboard":"#18181C","dashboardTransparent":"rgba(24, 24, 28, 0.1)","header":"#1B1B1B","menu":"#0F4CFF","muted":"#F9FAFC","navigation":"#EDEDED","output":"#2E3036","page":"#1E1F24","panel":"#232429","panelTransparent":"rgba(35, 36, 41, 0.7)","popup":"#27292E","row":"#2C2C2C","row2":"#51535C","scrollbarThumb":"rgba(100, 100, 100, 0.5)","scrollbarThumbHover":"rgba(255, 255, 255, 0.3)","scrollbarTrack":"#2E3036","success":"#8ADE00","successLight":"rgb(138, 222, 0, 0.3)","table":"#292A2F","tag":"#3A4550"},"borders":{"button":"#454850","contrast":"#FFFFFF","danger":"#FF144D","dark":"#000000","darkLight":"#2E3036","info":"#FFCC19","light":"#2F3034","medium":"#1C1C1C","medium2":"#141414","success":"#2FCB52"},"brand":{"earth100":"#C6EEDB","earth200":"#9DDFBF","earth300":"#6BBF96","earth400":"#37A46F","earth400Transparent":"rgba(55, 164, 111, 0.4)","earth500":"#00954C","energy100":"#FFF4BA","energy200":"#FFED92","energy300":"#FFE662","energy400":"#FFDA19","energy400Transparent":"rgba(255, 218, 25, 0.04)","energy500":"#F6C000","fire100":"#FFD7E0","fire200":"#FFA3B9","fire300":"#FF547D","fire400":"#FF144D","fire400Transparent":"rgba(255, 20, 77, 0.4)","fire500":"#EB0032","stone100":"#F3E6D7","stone200":"#E3D4C2","stone400":"#BFA78B","stone500":"#AF8859","water100":"#BDCEFF","water200":"#81A1FF","water300":"#517DFF","water400":"#2A60FE","water400Transparent":"rgba(42, 96, 254, 0.4)","water500":"#0F4CFF","wind100":"#EEEAFF","wind200":"#CCC1F4","wind300":"#A698DD","wind400":"#6B50D7","wind400SuperTransparent":"rgba(107, 80, 215, 0.12)","wind400Transparent":"rgba(107, 80, 215, 0.4)","wind500":"#4E32BC"},"chart":{"backgroundPrimary":"#7D55EC","backgroundSecondary":"#FF144D","backgroundTertiary":"#86E2FF","button1":"#4877FF","button2":"#FFCC19","button3":"#8ADE00","button4":"#FF4FF8","button5":"#B98D95","lines":"#9B6CA7","primary":"#6B50D7","secondary":"#FF144D","tertiary":"#2A60FE"},"content":{"active":"#FFFFFF","default":"#AEAEAE","disabled":"rgba(255, 255, 255, 0.3)","inverted":"#2C2C2C","muted":"#787A85"},"elevation":{"visualizationAccent":"#996CFF","visualizationAccentAlt":"#C1ACF7"},"feature":{"active":"rgba(250, 248, 254, 0.14)","disabled":"rgba(201, 206, 218, 0.12)"},"icons":{"neutral":"#787878"},"interactive":{"activeBorder":"#060606","blackBackgroundTransparent":"rgba(0, 0, 0, 0.5)","checked":"#060606","dangerBorder":"#FF144D","defaultBackground":"#36383F","defaultBackgroundTransparent":"rgba(54, 56, 63, 0.5)","defaultBorder":"#ffffff1a","disabledBorder":"#B1B8C3","focusBackground":"#B1B8C3","focusBorder":"#86E2FF","hoverBackground":"#4E4E4E","hoverBackgroundTransparent":"rgba(78, 78, 78, 0.5)","hoverBlackBackgroundTransparent":"rgba(0, 0, 0, 0.7)","hoverBorder":"#B9BFCA","hoverOverlay":"rgba(255, 255, 255, 0.1)","linkPrimary":"#1752FF","linkPrimaryHover":"#4877FF","linkPrimaryLight":"#5982ff","linkSecondary":"#6B50D7","linkSecondaryDisabled":"#C4B9EF","linkText":"#6AA1E0","linkTextLight":"#9ECBFF","purple":"#885EFF","rowHoverBackground":"rgba(0, 0, 0, 0.1)","transparent":"rgba(255, 255, 255, 0)"},"loader":{"color":"#EB0032","colorInverted":"#8ADE00"},"logo":{"color":"#FFFFFF"},"monotone":{"black":"#060606","blackTransparent":"rgba(0, 0, 0, 0.6)","gray":"#B1B8C3","grey100":"#F2F2F2","grey200":"#D5D7DC","grey300":"#B4B8C0","grey400":"#70747C","grey500":"#51535C","purple":"#6B50D7","white":"#FFFFFF","whiteTransparent":"rgba(255, 255, 255, 0.6)"},"neutral":{"n100":"#E7E8EA","n200":"#D8DADE","n300":"#CBCCD0","n400":"#BCBEC4","n500":"#AEB0B6"},"progress":{"negative":"#FF144D","positive":"#6B50D7"},"shadow":{"base":"12px 40px 120px rgba(0, 0, 0, 0.3)","frame":"0px 10px 40px rgba(0, 0, 0, 0.26)","menu":"4px 10px 20px rgba(6, 6, 6, 0.12)","popup":"10px 20px 40px rgba(0, 0, 0, 0.2)","small":"0px, 4px, rgba(0, 0, 0, 0.25)","window":"0px 10px 60px rgba(0, 0, 0, 0.7)"},"status":{"negative":"#FF144D","positive":"#24B400"},"text":{"fileBrowser":"#787A85"}}},"page":"/404","query":{},"buildId":"vPsMu6Fi2zrHaf2fRXKRO","nextExport":true,"isFallback":false,"appGip":true,"scriptLoader":[]}</script></body></html>