mage-ai 0.9.8__py3-none-any.whl → 0.9.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mage-ai might be problematic. Click here for more details.
- mage_ai/ai/constants.py +1 -0
- mage_ai/ai/generator.py +6 -0
- mage_ai/ai/{gen_doc_for_pipeline.py → generator_cmds.py} +5 -0
- mage_ai/ai/llm_pipeline_wizard.py +108 -3
- mage_ai/api/policies/BlockPolicy.py +1 -0
- mage_ai/api/policies/BlockTemplatePolicy.py +9 -0
- mage_ai/api/policies/LlmPolicy.py +3 -2
- mage_ai/api/policies/OauthAccessTokenPolicy.py +31 -0
- mage_ai/api/policies/PipelinePolicy.py +1 -0
- mage_ai/api/policies/ProjectPolicy.py +9 -0
- mage_ai/api/policies/SearchResultPolicy.py +32 -0
- mage_ai/api/policies/VariablePolicy.py +8 -0
- mage_ai/api/presenters/BlockTemplatePresenter.py +1 -0
- mage_ai/api/presenters/OauthAccessTokenPresenter.py +9 -0
- mage_ai/api/presenters/ProjectPresenter.py +2 -0
- mage_ai/api/presenters/SearchResultPresenter.py +73 -0
- mage_ai/api/resources/BlockResource.py +43 -4
- mage_ai/api/resources/BlockTemplateResource.py +15 -2
- mage_ai/api/resources/CustomTemplateResource.py +15 -3
- mage_ai/api/resources/FileContentResource.py +8 -1
- mage_ai/api/resources/FileResource.py +22 -2
- mage_ai/api/resources/OauthAccessTokenResource.py +24 -0
- mage_ai/api/resources/PipelineResource.py +70 -2
- mage_ai/api/resources/ProjectResource.py +15 -1
- mage_ai/api/resources/SearchResultResource.py +68 -0
- mage_ai/api/resources/VariableResource.py +25 -14
- mage_ai/cache/base.py +3 -0
- mage_ai/cache/block_action_object/__init__.py +282 -0
- mage_ai/cache/block_action_object/constants.py +4 -0
- mage_ai/cache/constants.py +1 -0
- mage_ai/data_integrations/destinations/constants.py +2 -0
- mage_ai/data_preparation/executors/k8s_pipeline_executor.py +1 -1
- mage_ai/data_preparation/models/block/__init__.py +20 -1
- mage_ai/data_preparation/models/block/dbt/utils/__init__.py +1 -0
- mage_ai/data_preparation/models/custom_templates/custom_block_template.py +8 -3
- mage_ai/data_preparation/models/custom_templates/utils.py +6 -3
- mage_ai/data_preparation/models/file.py +3 -0
- mage_ai/data_preparation/models/pipeline.py +10 -1
- mage_ai/data_preparation/models/{project.py → project/__init__.py} +17 -0
- mage_ai/data_preparation/models/project/constants.py +5 -0
- mage_ai/data_preparation/repo_manager.py +9 -6
- mage_ai/data_preparation/templates/constants.py +478 -1
- mage_ai/data_preparation/templates/template.py +9 -1
- mage_ai/orchestration/db/__init__.py +25 -0
- mage_ai/orchestration/pipeline_scheduler.py +23 -0
- mage_ai/server/constants.py +1 -1
- mage_ai/server/frontend_dist/404.html +2 -2
- mage_ai/server/frontend_dist/404.html.html +2 -2
- mage_ai/server/frontend_dist/_next/static/OwvTROmputQCA1Z9HEGDv/_buildManifest.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/{1005-ce36fe7ed460d006.js → 1005-16c1a6678de5311e.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/1424-e22139f33196c5c1.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/1484-a76b2c31e4808379.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/2485-b569baad92049d6b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/{2786-d8b593e5d7c42665.js → 2786-acce6ea0d9b4898e.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/3654-573457b62efff6e2.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/3881-0eb04f7f7a244347.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/{4317-b112fe7bb532b998.js → 4317-3fe63fc9401685d6.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/{4822-52d9d661b6bd1ae9.js → 4822-045cc7cdd7c95163.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/{547-14029bdacc0ed267.js → 547-4ad2cdae967055b6.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/6299-cf188c1b7a1bc33c.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/{7815-6dc55b5fd5d1f48a.js → 7815-fbd99c8f259d9e8b.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/{8312-cdc4d6c3b9f53099.js → 8312-c88a3b1012e5448b.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/{8952-492bf0e29d215ddd.js → 8952-e94a7334fcd5c68b.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/9055-d0d25298cb5f4e45.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/{_app-c7557dc9e725825e.js → _app-fa38a24707c76411.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/files-b9dac983c75798f2.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/{settings-17bb362b9683079c.js → settings-453aaa7bf78f8101.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/users/{[user]-bfa53c0ee79920a0.js → [user]-3b502aa6d1a04cad.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/users/{new-2f7e6bf8e2a37380.js → new-dd41d718cbb5990c.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/users-ec748f838b7b8fad.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage-bcc24120f5911f76.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/{overview-8ffbc4b0e8a43266.js → overview-6f20b9e9cbec91d6.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills/{[...slug]-373f6b8c9161e819.js → [...slug]-55212dfc15881ef2.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/{backfills-226bf0c5c1bfe004.js → backfills-6b4a82cd0516e121.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-0705c99df236ec8a.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/{logs-4ade07836fd577c9.js → logs-c294f5d74bd2d50f.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/{monitors-fd9eb711e89425e9.js → monitors-1b9e4abc54842cdf.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/{runs-027666ed18d3151e.js → runs-17ebe87e94242ae5.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-48b60ca236a26818.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/{syncs-de58e6b91e5e4552.js → syncs-5153f4665aaf8b17.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers/{[...slug]-432843576e1ed09e.js → [...slug]-1306e85191f6b47c.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/{triggers-ae31b1d0d8a4634d.js → triggers-2e731d343a26d46a.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines-0fab737dbc9074fc.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-b2354688508b0755.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/{users-a4f01b5c429757cc.js → users-6522c963087f18e8.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/{sign-in-1140f8ee7e89381a.js → sign-in-7dcd1257ca9660bb.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/templates/{[...slug]-f59b3f04c5a49351.js → [...slug]-78a07e7fe8973811.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/{templates-dbb209ce131390a9.js → templates-19df643f52679d5a.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-a1cf445565b68809.js +1 -0
- mage_ai/server/frontend_dist/files.html +2 -2
- mage_ai/server/frontend_dist/index.html +2 -2
- mage_ai/server/frontend_dist/manage/settings.html +2 -2
- mage_ai/server/frontend_dist/manage/users/[user].html +2 -2
- mage_ai/server/frontend_dist/manage/users/new.html +2 -2
- mage_ai/server/frontend_dist/manage/users.html +2 -2
- mage_ai/server/frontend_dist/manage.html +2 -2
- mage_ai/server/frontend_dist/overview.html +2 -2
- mage_ai/server/frontend_dist/pipeline-runs.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills/[...slug].html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/edit.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/logs.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runs.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runtime.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/runs/[run].html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/runs.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/settings.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/syncs.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers/[...slug].html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline].html +2 -2
- mage_ai/server/frontend_dist/pipelines.html +2 -2
- mage_ai/server/frontend_dist/settings/account/profile.html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/preferences.html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/sync-data.html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/users.html +2 -2
- mage_ai/server/frontend_dist/settings.html +2 -2
- mage_ai/server/frontend_dist/sign-in.html +16 -16
- mage_ai/server/frontend_dist/templates/[...slug].html +2 -2
- mage_ai/server/frontend_dist/templates.html +2 -2
- mage_ai/server/frontend_dist/terminal.html +2 -2
- mage_ai/server/frontend_dist/test.html +2 -2
- mage_ai/server/frontend_dist/triggers.html +2 -2
- mage_ai/server/frontend_dist/version-control.html +2 -2
- mage_ai/server/server.py +5 -0
- mage_ai/services/search/__init__.py +0 -0
- mage_ai/services/search/block_action_objects.py +33 -0
- mage_ai/services/search/constants.py +1 -0
- mage_ai/shared/array.py +7 -0
- mage_ai/shared/parsers.py +11 -2
- mage_ai/tests/shared/test_parsers.py +73 -0
- mage_ai/usage_statistics/constants.py +1 -0
- mage_ai/usage_statistics/logger.py +19 -1
- {mage_ai-0.9.8.dist-info → mage_ai-0.9.9.dist-info}/METADATA +9 -7
- {mage_ai-0.9.8.dist-info → mage_ai-0.9.9.dist-info}/RECORD +142 -129
- mage_ai/server/frontend_dist/_next/static/RcIgkp9pBXE1Rb4tOGR1L/_buildManifest.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/1424-de97c5aa0477894b.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/1484-c9d6bc33227070dd.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/2485-86c97f516c1abb9f.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/3215-c2f5f19b650f9a50.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/3654-bff4740b63cf280c.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/3881-7f121dbbe6e273cd.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/6299-97f283be01adf6a8.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/files-0df5a20d6f1aa606.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/users-1f6cb7656299cbd1.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage-4894eabb442a039c.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-a9bd86bee73b5762.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-cf51793c2112915c.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines-f1131ffe0d2a4a06.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-170943ca00445486.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-2ecda1f1437013f2.js +0 -1
- /mage_ai/server/frontend_dist/_next/static/{RcIgkp9pBXE1Rb4tOGR1L → OwvTROmputQCA1Z9HEGDv}/_middlewareManifest.js +0 -0
- /mage_ai/server/frontend_dist/_next/static/{RcIgkp9pBXE1Rb4tOGR1L → OwvTROmputQCA1Z9HEGDv}/_ssgManifest.js +0 -0
- {mage_ai-0.9.8.dist-info → mage_ai-0.9.9.dist-info}/LICENSE +0 -0
- {mage_ai-0.9.8.dist-info → mage_ai-0.9.9.dist-info}/WHEEL +0 -0
- {mage_ai-0.9.8.dist-info → mage_ai-0.9.9.dist-info}/entry_points.txt +0 -0
- {mage_ai-0.9.8.dist-info → mage_ai-0.9.9.dist-info}/top_level.txt +0 -0
|
@@ -1,8 +1,22 @@
|
|
|
1
1
|
from mage_ai.data_preparation.models.constants import BlockLanguage, BlockType
|
|
2
|
+
from mage_ai.io.base import DataSource
|
|
2
3
|
from mage_ai.shared.hash import index_by
|
|
3
4
|
|
|
5
|
+
GROUP_AGGREGATE = 'Aggregate'
|
|
6
|
+
GROUP_COLUMN_ACTIONS = 'Column actions'
|
|
7
|
+
GROUP_COLUMN_REMOVAL = 'Column removal'
|
|
8
|
+
GROUP_DATABASES = 'Databases'
|
|
9
|
+
GROUP_DATABASES_NO_SQL = 'Databases (NoSQL)'
|
|
10
|
+
GROUP_DATA_CLEANING = 'Data cleaning'
|
|
11
|
+
GROUP_DATA_LAKES = 'Data lakes'
|
|
12
|
+
GROUP_DATA_WAREHOUSES = 'Data warehouses'
|
|
4
13
|
GROUP_DELTA_LAKE = 'Delta Lake'
|
|
14
|
+
GROUP_FEATURE_EXTRACTION = 'Feature extraction'
|
|
15
|
+
GROUP_FEATURE_SCALING = 'Feature scaling'
|
|
16
|
+
GROUP_FORMATTING = 'Formatting'
|
|
5
17
|
GROUP_ORCHESTRATION = 'Orchestration'
|
|
18
|
+
GROUP_ROW_ACTIONS = 'Row actions'
|
|
19
|
+
GROUP_SHIFT = 'Shift'
|
|
6
20
|
|
|
7
21
|
TEMPLATES = [
|
|
8
22
|
dict(
|
|
@@ -32,6 +46,7 @@ TEMPLATES = [
|
|
|
32
46
|
dict(
|
|
33
47
|
block_type=BlockType.DATA_LOADER,
|
|
34
48
|
description='Load data from MongoDB.',
|
|
49
|
+
groups=[GROUP_DATABASES_NO_SQL],
|
|
35
50
|
language=BlockLanguage.PYTHON,
|
|
36
51
|
name='MongoDB',
|
|
37
52
|
path='data_loaders/mongodb.py',
|
|
@@ -107,4 +122,466 @@ TEMPLATES = [
|
|
|
107
122
|
),
|
|
108
123
|
]
|
|
109
124
|
|
|
110
|
-
|
|
125
|
+
TEMPLATES_ONLY_FOR_V2 = [
|
|
126
|
+
# Data loaders
|
|
127
|
+
dict(
|
|
128
|
+
block_type=BlockType.DATA_LOADER,
|
|
129
|
+
language=BlockLanguage.PYTHON,
|
|
130
|
+
name='Base template (generic)',
|
|
131
|
+
path='data_loaders/default.jinja',
|
|
132
|
+
),
|
|
133
|
+
# Data lakes
|
|
134
|
+
dict(
|
|
135
|
+
block_type=BlockType.DATA_LOADER,
|
|
136
|
+
groups=[GROUP_DATA_LAKES],
|
|
137
|
+
language=BlockLanguage.PYTHON,
|
|
138
|
+
name='Amazon S3',
|
|
139
|
+
path='data_loaders/s3.py',
|
|
140
|
+
),
|
|
141
|
+
dict(
|
|
142
|
+
block_type=BlockType.DATA_LOADER,
|
|
143
|
+
groups=[GROUP_DATA_LAKES],
|
|
144
|
+
language=BlockLanguage.PYTHON,
|
|
145
|
+
name='Azure Blob Storage',
|
|
146
|
+
path='data_loaders/azure_blob_storage.py',
|
|
147
|
+
),
|
|
148
|
+
dict(
|
|
149
|
+
block_type=BlockType.DATA_LOADER,
|
|
150
|
+
groups=[GROUP_DATA_LAKES],
|
|
151
|
+
language=BlockLanguage.PYTHON,
|
|
152
|
+
name='Google Cloud Storage',
|
|
153
|
+
path='data_loaders/google_cloud_storage.py',
|
|
154
|
+
),
|
|
155
|
+
# Data warehouses
|
|
156
|
+
dict(
|
|
157
|
+
block_type=BlockType.DATA_LOADER,
|
|
158
|
+
groups=[GROUP_DATA_WAREHOUSES],
|
|
159
|
+
language=BlockLanguage.PYTHON,
|
|
160
|
+
name='Amazon Redshift',
|
|
161
|
+
path='data_loaders/redshift.py',
|
|
162
|
+
),
|
|
163
|
+
dict(
|
|
164
|
+
block_type=BlockType.DATA_LOADER,
|
|
165
|
+
description='Load data from Google BigQuery.',
|
|
166
|
+
groups=[GROUP_DATA_WAREHOUSES],
|
|
167
|
+
language=BlockLanguage.PYTHON,
|
|
168
|
+
name='Google BigQuery',
|
|
169
|
+
path='data_loaders/bigquery.py',
|
|
170
|
+
),
|
|
171
|
+
dict(
|
|
172
|
+
block_type=BlockType.DATA_LOADER,
|
|
173
|
+
groups=[GROUP_DATA_WAREHOUSES],
|
|
174
|
+
language=BlockLanguage.PYTHON,
|
|
175
|
+
name='Snowflake',
|
|
176
|
+
path='data_loaders/snowflake.py',
|
|
177
|
+
),
|
|
178
|
+
# Databases
|
|
179
|
+
dict(
|
|
180
|
+
block_type=BlockType.DATA_LOADER,
|
|
181
|
+
groups=[GROUP_DATABASES],
|
|
182
|
+
language=BlockLanguage.PYTHON,
|
|
183
|
+
name='MySQL',
|
|
184
|
+
path='data_loaders/mysql.py',
|
|
185
|
+
),
|
|
186
|
+
dict(
|
|
187
|
+
block_type=BlockType.DATA_LOADER,
|
|
188
|
+
groups=[GROUP_DATABASES],
|
|
189
|
+
language=BlockLanguage.PYTHON,
|
|
190
|
+
name='Oracle DB',
|
|
191
|
+
path='data_loaders/oracledb.py',
|
|
192
|
+
),
|
|
193
|
+
dict(
|
|
194
|
+
block_type=BlockType.DATA_LOADER,
|
|
195
|
+
groups=[GROUP_DATABASES],
|
|
196
|
+
language=BlockLanguage.PYTHON,
|
|
197
|
+
name='PostgreSQL',
|
|
198
|
+
path='data_loaders/postgres.py',
|
|
199
|
+
),
|
|
200
|
+
dict(
|
|
201
|
+
block_type=BlockType.DATA_LOADER,
|
|
202
|
+
description='Fetch data from an API request.',
|
|
203
|
+
language=BlockLanguage.PYTHON,
|
|
204
|
+
name='API',
|
|
205
|
+
path='data_loaders/api.py',
|
|
206
|
+
),
|
|
207
|
+
dict(
|
|
208
|
+
block_type=BlockType.DATA_LOADER,
|
|
209
|
+
description='Load data from a file on your machine.',
|
|
210
|
+
language=BlockLanguage.PYTHON,
|
|
211
|
+
name='Local file',
|
|
212
|
+
path='data_loaders/file.py',
|
|
213
|
+
),
|
|
214
|
+
dict(
|
|
215
|
+
block_type=BlockType.DATA_LOADER,
|
|
216
|
+
language=BlockLanguage.PYTHON,
|
|
217
|
+
name='Druid',
|
|
218
|
+
path='data_loaders/druid.py',
|
|
219
|
+
),
|
|
220
|
+
# Transformers
|
|
221
|
+
dict(
|
|
222
|
+
block_type=BlockType.TRANSFORMER,
|
|
223
|
+
language=BlockLanguage.PYTHON,
|
|
224
|
+
name='Base template (generic)',
|
|
225
|
+
path='transformers/default.jinja',
|
|
226
|
+
),
|
|
227
|
+
# Data warehouses
|
|
228
|
+
dict(
|
|
229
|
+
block_type=BlockType.TRANSFORMER,
|
|
230
|
+
groups=[GROUP_DATA_WAREHOUSES],
|
|
231
|
+
language=BlockLanguage.PYTHON,
|
|
232
|
+
name='Amazon Redshift',
|
|
233
|
+
path='transformers/data_warehouse_transformer.jinja',
|
|
234
|
+
template_variables=dict(
|
|
235
|
+
additional_args='\n loader.commit() # Permanently apply database changes',
|
|
236
|
+
data_source=DataSource.REDSHIFT.value,
|
|
237
|
+
data_source_handler='Redshift',
|
|
238
|
+
),
|
|
239
|
+
),
|
|
240
|
+
dict(
|
|
241
|
+
block_type=BlockType.TRANSFORMER,
|
|
242
|
+
groups=[GROUP_DATA_WAREHOUSES],
|
|
243
|
+
language=BlockLanguage.PYTHON,
|
|
244
|
+
name='Google BigQuery',
|
|
245
|
+
path='transformers/data_warehouse_transformer.jinja',
|
|
246
|
+
template_variables=dict(
|
|
247
|
+
additional_args='',
|
|
248
|
+
data_source=DataSource.BIGQUERY.value,
|
|
249
|
+
data_source_handler='BigQuery',
|
|
250
|
+
),
|
|
251
|
+
),
|
|
252
|
+
dict(
|
|
253
|
+
block_type=BlockType.TRANSFORMER,
|
|
254
|
+
groups=[GROUP_DATA_WAREHOUSES],
|
|
255
|
+
language=BlockLanguage.PYTHON,
|
|
256
|
+
name='Snowflake',
|
|
257
|
+
path='transformers/data_warehouse_transformer.jinja',
|
|
258
|
+
template_variables=dict(
|
|
259
|
+
additional_args='\n loader.commit() # Permanently apply database changes',
|
|
260
|
+
data_source=DataSource.SNOWFLAKE.value,
|
|
261
|
+
data_source_handler='Snowflake',
|
|
262
|
+
),
|
|
263
|
+
),
|
|
264
|
+
# Databases
|
|
265
|
+
dict(
|
|
266
|
+
block_type=BlockType.TRANSFORMER,
|
|
267
|
+
groups=[GROUP_DATABASES],
|
|
268
|
+
language=BlockLanguage.PYTHON,
|
|
269
|
+
name='PostgreSQL',
|
|
270
|
+
path='transformers/data_warehouse_transformer.jinja',
|
|
271
|
+
template_variables=dict(
|
|
272
|
+
additional_args='\n loader.commit() # Permanently apply database changes',
|
|
273
|
+
data_source=DataSource.POSTGRES.value,
|
|
274
|
+
data_source_handler='Postgres',
|
|
275
|
+
),
|
|
276
|
+
),
|
|
277
|
+
# Row actions
|
|
278
|
+
dict(
|
|
279
|
+
block_type=BlockType.TRANSFORMER,
|
|
280
|
+
groups=[GROUP_ROW_ACTIONS],
|
|
281
|
+
language=BlockLanguage.PYTHON,
|
|
282
|
+
name='Drop duplicate rows',
|
|
283
|
+
path='transformers/transformer_actions/row/drop_duplicate.py',
|
|
284
|
+
),
|
|
285
|
+
dict(
|
|
286
|
+
block_type=BlockType.TRANSFORMER,
|
|
287
|
+
groups=[GROUP_ROW_ACTIONS],
|
|
288
|
+
language=BlockLanguage.PYTHON,
|
|
289
|
+
name='Filter rows',
|
|
290
|
+
path='transformers/transformer_actions/row/filter.py',
|
|
291
|
+
),
|
|
292
|
+
dict(
|
|
293
|
+
block_type=BlockType.TRANSFORMER,
|
|
294
|
+
groups=[GROUP_ROW_ACTIONS],
|
|
295
|
+
language=BlockLanguage.PYTHON,
|
|
296
|
+
name='Remove rows',
|
|
297
|
+
path='transformers/transformer_actions/row/remove.py',
|
|
298
|
+
),
|
|
299
|
+
dict(
|
|
300
|
+
block_type=BlockType.TRANSFORMER,
|
|
301
|
+
groups=[GROUP_ROW_ACTIONS],
|
|
302
|
+
language=BlockLanguage.PYTHON,
|
|
303
|
+
name='Sort rows',
|
|
304
|
+
path='transformers/transformer_actions/row/sort.py',
|
|
305
|
+
),
|
|
306
|
+
# Column actions
|
|
307
|
+
# Aggregate
|
|
308
|
+
dict(
|
|
309
|
+
block_type=BlockType.TRANSFORMER,
|
|
310
|
+
groups=[GROUP_COLUMN_ACTIONS, GROUP_AGGREGATE],
|
|
311
|
+
language=BlockLanguage.PYTHON,
|
|
312
|
+
name='Average value of column',
|
|
313
|
+
path='transformers/transformer_actions/column/average.py',
|
|
314
|
+
),
|
|
315
|
+
dict(
|
|
316
|
+
block_type=BlockType.TRANSFORMER,
|
|
317
|
+
groups=[GROUP_COLUMN_ACTIONS, GROUP_AGGREGATE],
|
|
318
|
+
language=BlockLanguage.PYTHON,
|
|
319
|
+
name='Count unique values in column',
|
|
320
|
+
path='transformers/transformer_actions/column/count_distinct.py',
|
|
321
|
+
),
|
|
322
|
+
dict(
|
|
323
|
+
block_type=BlockType.TRANSFORMER,
|
|
324
|
+
groups=[GROUP_COLUMN_ACTIONS, GROUP_AGGREGATE],
|
|
325
|
+
language=BlockLanguage.PYTHON,
|
|
326
|
+
name='First value in column',
|
|
327
|
+
path='transformers/transformer_actions/column/first.py',
|
|
328
|
+
),
|
|
329
|
+
dict(
|
|
330
|
+
block_type=BlockType.TRANSFORMER,
|
|
331
|
+
groups=[GROUP_COLUMN_ACTIONS, GROUP_AGGREGATE],
|
|
332
|
+
language=BlockLanguage.PYTHON,
|
|
333
|
+
name='Last value in column',
|
|
334
|
+
path='transformers/transformer_actions/column/last.py',
|
|
335
|
+
),
|
|
336
|
+
dict(
|
|
337
|
+
block_type=BlockType.TRANSFORMER,
|
|
338
|
+
groups=[GROUP_COLUMN_ACTIONS, GROUP_AGGREGATE],
|
|
339
|
+
language=BlockLanguage.PYTHON,
|
|
340
|
+
name='Maximum value in column',
|
|
341
|
+
path='transformers/transformer_actions/column/max.py',
|
|
342
|
+
),
|
|
343
|
+
dict(
|
|
344
|
+
block_type=BlockType.TRANSFORMER,
|
|
345
|
+
groups=[GROUP_COLUMN_ACTIONS, GROUP_AGGREGATE],
|
|
346
|
+
language=BlockLanguage.PYTHON,
|
|
347
|
+
name='Median value in column',
|
|
348
|
+
path='transformers/transformer_actions/column/median.py',
|
|
349
|
+
),
|
|
350
|
+
dict(
|
|
351
|
+
block_type=BlockType.TRANSFORMER,
|
|
352
|
+
groups=[GROUP_COLUMN_ACTIONS, GROUP_AGGREGATE],
|
|
353
|
+
language=BlockLanguage.PYTHON,
|
|
354
|
+
name='Min value in column',
|
|
355
|
+
path='transformers/transformer_actions/column/min.py',
|
|
356
|
+
),
|
|
357
|
+
dict(
|
|
358
|
+
block_type=BlockType.TRANSFORMER,
|
|
359
|
+
groups=[GROUP_COLUMN_ACTIONS, GROUP_AGGREGATE],
|
|
360
|
+
language=BlockLanguage.PYTHON,
|
|
361
|
+
name='Sum of all values in column',
|
|
362
|
+
path='transformers/transformer_actions/column/sum.py',
|
|
363
|
+
),
|
|
364
|
+
dict(
|
|
365
|
+
block_type=BlockType.TRANSFORMER,
|
|
366
|
+
groups=[GROUP_COLUMN_ACTIONS, GROUP_AGGREGATE],
|
|
367
|
+
language=BlockLanguage.PYTHON,
|
|
368
|
+
name='Total count of values in column',
|
|
369
|
+
path='transformers/transformer_actions/column/count.py',
|
|
370
|
+
),
|
|
371
|
+
# Formatting
|
|
372
|
+
dict(
|
|
373
|
+
block_type=BlockType.TRANSFORMER,
|
|
374
|
+
groups=[GROUP_COLUMN_ACTIONS, GROUP_FORMATTING],
|
|
375
|
+
language=BlockLanguage.PYTHON,
|
|
376
|
+
name='Clean column name',
|
|
377
|
+
path='transformers/transformer_actions/column/clean_column_name.py',
|
|
378
|
+
),
|
|
379
|
+
dict(
|
|
380
|
+
block_type=BlockType.TRANSFORMER,
|
|
381
|
+
groups=[GROUP_COLUMN_ACTIONS, GROUP_FORMATTING],
|
|
382
|
+
language=BlockLanguage.PYTHON,
|
|
383
|
+
name='Fix syntax errors',
|
|
384
|
+
path='transformers/transformer_actions/column/fix_syntax_errors.py',
|
|
385
|
+
),
|
|
386
|
+
dict(
|
|
387
|
+
block_type=BlockType.TRANSFORMER,
|
|
388
|
+
groups=[GROUP_COLUMN_ACTIONS, GROUP_FORMATTING],
|
|
389
|
+
language=BlockLanguage.PYTHON,
|
|
390
|
+
name='Reformat values in column',
|
|
391
|
+
path='transformers/transformer_actions/column/reformat.py',
|
|
392
|
+
),
|
|
393
|
+
# Column removal
|
|
394
|
+
dict(
|
|
395
|
+
block_type=BlockType.TRANSFORMER,
|
|
396
|
+
groups=[GROUP_COLUMN_ACTIONS, GROUP_COLUMN_REMOVAL],
|
|
397
|
+
language=BlockLanguage.PYTHON,
|
|
398
|
+
name='Keep column(s)',
|
|
399
|
+
path='transformers/transformer_actions/column/select.py',
|
|
400
|
+
),
|
|
401
|
+
dict(
|
|
402
|
+
block_type=BlockType.TRANSFORMER,
|
|
403
|
+
groups=[GROUP_COLUMN_ACTIONS, GROUP_COLUMN_REMOVAL],
|
|
404
|
+
language=BlockLanguage.PYTHON,
|
|
405
|
+
name='Remove column(s)',
|
|
406
|
+
path='transformers/transformer_actions/column/remove.py',
|
|
407
|
+
),
|
|
408
|
+
# Shift
|
|
409
|
+
dict(
|
|
410
|
+
block_type=BlockType.TRANSFORMER,
|
|
411
|
+
groups=[GROUP_COLUMN_ACTIONS, GROUP_SHIFT],
|
|
412
|
+
language=BlockLanguage.PYTHON,
|
|
413
|
+
name='Shift row values down',
|
|
414
|
+
path='transformers/transformer_actions/column/shift_down.py',
|
|
415
|
+
),
|
|
416
|
+
dict(
|
|
417
|
+
block_type=BlockType.TRANSFORMER,
|
|
418
|
+
groups=[GROUP_COLUMN_ACTIONS, GROUP_SHIFT],
|
|
419
|
+
language=BlockLanguage.PYTHON,
|
|
420
|
+
name='Shift row values up',
|
|
421
|
+
path='transformers/transformer_actions/column/shift_up.py',
|
|
422
|
+
),
|
|
423
|
+
# Feature scaling
|
|
424
|
+
dict(
|
|
425
|
+
block_type=BlockType.TRANSFORMER,
|
|
426
|
+
groups=[GROUP_COLUMN_ACTIONS, GROUP_FEATURE_SCALING],
|
|
427
|
+
language=BlockLanguage.PYTHON,
|
|
428
|
+
name='Normalize data',
|
|
429
|
+
path='transformers/transformer_actions/column/normalize.py',
|
|
430
|
+
),
|
|
431
|
+
dict(
|
|
432
|
+
block_type=BlockType.TRANSFORMER,
|
|
433
|
+
groups=[GROUP_COLUMN_ACTIONS, GROUP_FEATURE_SCALING],
|
|
434
|
+
language=BlockLanguage.PYTHON,
|
|
435
|
+
name='Standardize data',
|
|
436
|
+
path='transformers/transformer_actions/column/standardize.py',
|
|
437
|
+
),
|
|
438
|
+
# Data cleaning
|
|
439
|
+
dict(
|
|
440
|
+
block_type=BlockType.TRANSFORMER,
|
|
441
|
+
groups=[GROUP_COLUMN_ACTIONS, GROUP_DATA_CLEANING],
|
|
442
|
+
language=BlockLanguage.PYTHON,
|
|
443
|
+
name='Fill in missing values',
|
|
444
|
+
path='transformers/transformer_actions/column/impute.py',
|
|
445
|
+
),
|
|
446
|
+
dict(
|
|
447
|
+
block_type=BlockType.TRANSFORMER,
|
|
448
|
+
groups=[GROUP_COLUMN_ACTIONS, GROUP_DATA_CLEANING],
|
|
449
|
+
language=BlockLanguage.PYTHON,
|
|
450
|
+
name='Remove outliers',
|
|
451
|
+
path='transformers/transformer_actions/column/remove_outliers.py',
|
|
452
|
+
),
|
|
453
|
+
# Feature extraction
|
|
454
|
+
dict(
|
|
455
|
+
block_type=BlockType.TRANSFORMER,
|
|
456
|
+
groups=[GROUP_COLUMN_ACTIONS, GROUP_FEATURE_EXTRACTION],
|
|
457
|
+
language=BlockLanguage.PYTHON,
|
|
458
|
+
name='Calculate difference between values',
|
|
459
|
+
path='transformers/transformer_actions/column/diff.py',
|
|
460
|
+
),
|
|
461
|
+
# Data exporters
|
|
462
|
+
dict(
|
|
463
|
+
block_type=BlockType.DATA_EXPORTER,
|
|
464
|
+
language=BlockLanguage.PYTHON,
|
|
465
|
+
name='Base template (generic)',
|
|
466
|
+
path='data_exporters/default.jinja',
|
|
467
|
+
),
|
|
468
|
+
dict(
|
|
469
|
+
block_type=BlockType.DATA_EXPORTER,
|
|
470
|
+
language=BlockLanguage.PYTHON,
|
|
471
|
+
name='Local file',
|
|
472
|
+
path='data_exporters/file.py',
|
|
473
|
+
),
|
|
474
|
+
# Data lakes
|
|
475
|
+
dict(
|
|
476
|
+
block_type=BlockType.DATA_EXPORTER,
|
|
477
|
+
groups=[GROUP_DATA_LAKES],
|
|
478
|
+
language=BlockLanguage.PYTHON,
|
|
479
|
+
name='Amazon S3',
|
|
480
|
+
path='data_exporters/s3.py',
|
|
481
|
+
),
|
|
482
|
+
dict(
|
|
483
|
+
block_type=BlockType.DATA_EXPORTER,
|
|
484
|
+
groups=[GROUP_DATA_LAKES],
|
|
485
|
+
language=BlockLanguage.PYTHON,
|
|
486
|
+
name='Azure Blob Storage',
|
|
487
|
+
path='data_exporters/azure_blob_storage.py',
|
|
488
|
+
),
|
|
489
|
+
dict(
|
|
490
|
+
block_type=BlockType.DATA_EXPORTER,
|
|
491
|
+
groups=[GROUP_DATA_LAKES],
|
|
492
|
+
language=BlockLanguage.PYTHON,
|
|
493
|
+
name='Google Cloud Storage',
|
|
494
|
+
path='data_exporters/google_cloud_storage.py',
|
|
495
|
+
),
|
|
496
|
+
# Data warehouses
|
|
497
|
+
dict(
|
|
498
|
+
block_type=BlockType.DATA_EXPORTER,
|
|
499
|
+
groups=[GROUP_DATA_WAREHOUSES],
|
|
500
|
+
language=BlockLanguage.PYTHON,
|
|
501
|
+
name='Amazon Redshift',
|
|
502
|
+
path='data_exporters/redshift.py',
|
|
503
|
+
),
|
|
504
|
+
dict(
|
|
505
|
+
block_type=BlockType.DATA_EXPORTER,
|
|
506
|
+
groups=[GROUP_DATA_WAREHOUSES],
|
|
507
|
+
language=BlockLanguage.PYTHON,
|
|
508
|
+
name='Google BigQuery',
|
|
509
|
+
path='data_exporters/bigquery.py',
|
|
510
|
+
),
|
|
511
|
+
dict(
|
|
512
|
+
block_type=BlockType.DATA_EXPORTER,
|
|
513
|
+
groups=[GROUP_DATA_WAREHOUSES],
|
|
514
|
+
language=BlockLanguage.PYTHON,
|
|
515
|
+
name='Snowflake',
|
|
516
|
+
path='data_exporters/snowflake.py',
|
|
517
|
+
),
|
|
518
|
+
# Databases
|
|
519
|
+
dict(
|
|
520
|
+
block_type=BlockType.DATA_EXPORTER,
|
|
521
|
+
groups=[GROUP_DATABASES],
|
|
522
|
+
language=BlockLanguage.PYTHON,
|
|
523
|
+
name='MySQL',
|
|
524
|
+
path='data_exporters/mysql.py',
|
|
525
|
+
),
|
|
526
|
+
dict(
|
|
527
|
+
block_type=BlockType.DATA_EXPORTER,
|
|
528
|
+
groups=[GROUP_DATABASES],
|
|
529
|
+
language=BlockLanguage.PYTHON,
|
|
530
|
+
name='PostgreSQL',
|
|
531
|
+
path='data_exporters/postgres.py',
|
|
532
|
+
),
|
|
533
|
+
# Sensors
|
|
534
|
+
dict(
|
|
535
|
+
block_type=BlockType.SENSOR,
|
|
536
|
+
language=BlockLanguage.PYTHON,
|
|
537
|
+
name='Base template (generic)',
|
|
538
|
+
path='sensors/default.py',
|
|
539
|
+
),
|
|
540
|
+
# Data lakes
|
|
541
|
+
dict(
|
|
542
|
+
block_type=BlockType.SENSOR,
|
|
543
|
+
groups=[GROUP_DATA_LAKES],
|
|
544
|
+
language=BlockLanguage.PYTHON,
|
|
545
|
+
name='Amazon S3',
|
|
546
|
+
path='sensors/s3.py',
|
|
547
|
+
),
|
|
548
|
+
# Data warehouses
|
|
549
|
+
dict(
|
|
550
|
+
block_type=BlockType.SENSOR,
|
|
551
|
+
groups=[GROUP_DATA_WAREHOUSES],
|
|
552
|
+
language=BlockLanguage.PYTHON,
|
|
553
|
+
name='Amazon Redshift',
|
|
554
|
+
path='sensors/redshift.py',
|
|
555
|
+
),
|
|
556
|
+
dict(
|
|
557
|
+
block_type=BlockType.SENSOR,
|
|
558
|
+
groups=[GROUP_DATA_WAREHOUSES],
|
|
559
|
+
language=BlockLanguage.PYTHON,
|
|
560
|
+
name='Google BigQuery',
|
|
561
|
+
path='sensors/bigquery.py',
|
|
562
|
+
),
|
|
563
|
+
dict(
|
|
564
|
+
block_type=BlockType.SENSOR,
|
|
565
|
+
groups=[GROUP_DATA_WAREHOUSES],
|
|
566
|
+
language=BlockLanguage.PYTHON,
|
|
567
|
+
name='Snowflake',
|
|
568
|
+
path='sensors/snowflake.py',
|
|
569
|
+
),
|
|
570
|
+
# Databases
|
|
571
|
+
dict(
|
|
572
|
+
block_type=BlockType.SENSOR,
|
|
573
|
+
groups=[GROUP_DATABASES],
|
|
574
|
+
language=BlockLanguage.PYTHON,
|
|
575
|
+
name='MySQL',
|
|
576
|
+
path='sensors/mysql.py',
|
|
577
|
+
),
|
|
578
|
+
dict(
|
|
579
|
+
block_type=BlockType.SENSOR,
|
|
580
|
+
groups=[GROUP_DATABASES],
|
|
581
|
+
language=BlockLanguage.PYTHON,
|
|
582
|
+
name='PostgreSQL',
|
|
583
|
+
path='sensors/postgres.py',
|
|
584
|
+
),
|
|
585
|
+
]
|
|
586
|
+
|
|
587
|
+
TEMPLATES_BY_UUID = index_by(lambda x: x['name'], TEMPLATES + TEMPLATES_ONLY_FOR_V2)
|
|
@@ -59,9 +59,17 @@ def fetch_template_source(
|
|
|
59
59
|
return template_source
|
|
60
60
|
|
|
61
61
|
if 'template_path' in config:
|
|
62
|
+
template_variables_to_render = dict(
|
|
63
|
+
code=config.get('existing_code', ''),
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
template_variables = config.get('template_variables')
|
|
67
|
+
if template_variables:
|
|
68
|
+
template_variables_to_render.update(template_variables)
|
|
69
|
+
|
|
62
70
|
return (
|
|
63
71
|
template_env.get_template(config['template_path']).render(
|
|
64
|
-
|
|
72
|
+
**template_variables_to_render,
|
|
65
73
|
)
|
|
66
74
|
)
|
|
67
75
|
elif block_type == BlockType.DATA_LOADER:
|
|
@@ -4,6 +4,7 @@ import os
|
|
|
4
4
|
import sqlalchemy
|
|
5
5
|
from sqlalchemy import create_engine
|
|
6
6
|
from sqlalchemy.orm import scoped_session, sessionmaker
|
|
7
|
+
from urllib.parse import parse_qs, urlparse
|
|
7
8
|
|
|
8
9
|
from mage_ai.data_preparation.repo_manager import get_variables_dir
|
|
9
10
|
from mage_ai.orchestration.constants import (
|
|
@@ -71,8 +72,32 @@ class DBConnection:
|
|
|
71
72
|
self.session = None
|
|
72
73
|
|
|
73
74
|
|
|
75
|
+
def get_postgresql_schema(url):
|
|
76
|
+
parse_result = urlparse(url)
|
|
77
|
+
if parse_result.scheme == 'postgresql+psycopg2':
|
|
78
|
+
q = parse_qs(
|
|
79
|
+
parse_result.query.replace('%%', '%')
|
|
80
|
+
)
|
|
81
|
+
options = q.get('options')
|
|
82
|
+
if options and len(options) >= 1:
|
|
83
|
+
params = options[0].replace('-c ', '').split(' ')
|
|
84
|
+
kvs = dict(p.split('=') for p in params)
|
|
85
|
+
return kvs.get('search_path')
|
|
86
|
+
|
|
87
|
+
|
|
74
88
|
db_connection = DBConnection()
|
|
75
89
|
|
|
90
|
+
if db_connection_url.startswith('postgresql'):
|
|
91
|
+
db_schema = get_postgresql_schema(db_connection_url)
|
|
92
|
+
if db_schema:
|
|
93
|
+
db_connection.start_session()
|
|
94
|
+
db_connection.session.execute(f'CREATE SCHEMA IF NOT EXISTS {db_schema};')
|
|
95
|
+
db_connection.session.commit()
|
|
96
|
+
db_connection.close_session()
|
|
97
|
+
print(f'Set the default PostgreSQL schema to {db_schema}')
|
|
98
|
+
else:
|
|
99
|
+
print('No schema in PostgreSQL connection URL: use the default "public" schema')
|
|
100
|
+
|
|
76
101
|
|
|
77
102
|
def safe_db_query(func):
|
|
78
103
|
def func_with_rollback(*args, **kwargs):
|
|
@@ -214,6 +214,8 @@ class PipelineScheduler:
|
|
|
214
214
|
pipeline=self.pipeline,
|
|
215
215
|
pipeline_run=self.pipeline_run,
|
|
216
216
|
)
|
|
217
|
+
# Cancel block runs that are still in progress for the pipeline run.
|
|
218
|
+
cancel_block_runs_and_jobs(self.pipeline_run, self.pipeline)
|
|
217
219
|
elif PipelineType.INTEGRATION == self.pipeline.type:
|
|
218
220
|
self.__schedule_integration_streams(block_runs)
|
|
219
221
|
else:
|
|
@@ -1159,6 +1161,27 @@ def stop_pipeline_run(
|
|
|
1159
1161
|
pipeline_run.update(status=PipelineRun.PipelineRunStatus.CANCELLED)
|
|
1160
1162
|
|
|
1161
1163
|
# Cancel all the block runs
|
|
1164
|
+
cancel_block_runs_and_jobs(pipeline_run, pipeline)
|
|
1165
|
+
|
|
1166
|
+
|
|
1167
|
+
def cancel_block_runs_and_jobs(
|
|
1168
|
+
pipeline_run: PipelineRun,
|
|
1169
|
+
pipeline: Pipeline = None,
|
|
1170
|
+
) -> None:
|
|
1171
|
+
"""Cancel in progress block runs and jobs for a pipeline run.
|
|
1172
|
+
|
|
1173
|
+
This function cancels blocks runs for the pipeline run. If a pipeline object
|
|
1174
|
+
is provided, it also kills the jobs associated with the pipeline run and its
|
|
1175
|
+
integration streams if applicable.
|
|
1176
|
+
|
|
1177
|
+
Args:
|
|
1178
|
+
pipeline_run (PipelineRun): The pipeline run to stop.
|
|
1179
|
+
pipeline (Pipeline, optional): The pipeline associated with the pipeline run.
|
|
1180
|
+
Defaults to None.
|
|
1181
|
+
|
|
1182
|
+
Returns:
|
|
1183
|
+
None
|
|
1184
|
+
"""
|
|
1162
1185
|
block_runs_to_cancel = []
|
|
1163
1186
|
running_blocks = []
|
|
1164
1187
|
for b in pipeline_run.block_runs:
|
mage_ai/server/constants.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta content="width=device-width, initial-scale=1.0, maximum-scale=5.0, user-scalable=0" name="viewport"/><title>404: This page could not be found</title><meta name="next-head-count" content="3"/><link href="/favicon.ico" rel="icon"/><link rel="preload" href="/_next/static/css/d1e8e64d0b07af2f.css" as="style"/><link rel="stylesheet" href="/_next/static/css/d1e8e64d0b07af2f.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-5cd94c89d3acac5f.js"></script><script src="/_next/static/chunks/webpack-8f51011a10b384a1.js" defer=""></script><script src="/_next/static/chunks/framework-7c365855dab1bf41.js" defer=""></script><script src="/_next/static/chunks/main-bb0dd5375146d7fd.js" defer=""></script><script src="/_next/static/chunks/pages/_app-
|
|
1
|
+
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta content="width=device-width, initial-scale=1.0, maximum-scale=5.0, user-scalable=0" name="viewport"/><title>404: This page could not be found</title><meta name="next-head-count" content="3"/><link href="/favicon.ico" rel="icon"/><link rel="preload" href="/_next/static/css/d1e8e64d0b07af2f.css" as="style"/><link rel="stylesheet" href="/_next/static/css/d1e8e64d0b07af2f.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-5cd94c89d3acac5f.js"></script><script src="/_next/static/chunks/webpack-8f51011a10b384a1.js" defer=""></script><script src="/_next/static/chunks/framework-7c365855dab1bf41.js" defer=""></script><script src="/_next/static/chunks/main-bb0dd5375146d7fd.js" defer=""></script><script src="/_next/static/chunks/pages/_app-fa38a24707c76411.js" defer=""></script><script src="/_next/static/chunks/pages/_error-235304e5badb19eb.js" defer=""></script><script src="/_next/static/OwvTROmputQCA1Z9HEGDv/_buildManifest.js" defer=""></script><script src="/_next/static/OwvTROmputQCA1Z9HEGDv/_ssgManifest.js" defer=""></script><script src="/_next/static/OwvTROmputQCA1Z9HEGDv/_middlewareManifest.js" defer=""></script><style data-styled="" data-styled-version="5.3.6">html{-webkit-box-sizing:border-box;box-sizing:border-box;-ms-overflow-style:scrollbar;}/*!sc*/
|
|
2
2
|
*,*::before,*::after{-webkit-box-sizing:inherit;box-sizing:inherit;}/*!sc*/
|
|
3
3
|
data-styled.g4[id="sc-global-czSCUT1"]{content:"sc-global-czSCUT1,"}/*!sc*/
|
|
4
4
|
.kOVcuR .Toastify__toast-container{margin-top:24px;padding:0 !important;width:500px !important;}/*!sc*/
|
|
@@ -19,4 +19,4 @@ data-styled.g5[id="ToastWrapper-sc-1a33ph1-0"]{content:"kOVcuR,"}/*!sc*/
|
|
|
19
19
|
.next-error-h1 {
|
|
20
20
|
border-right: 1px solid rgba(255, 255, 255, .3);
|
|
21
21
|
}
|
|
22
|
-
}</style><h1 class="next-error-h1" style="display:inline-block;margin:0;margin-right:20px;padding:10px 23px 10px 0;font-size:24px;font-weight:500;vertical-align:top">404<!-- --></h1><div style="display:inline-block;text-align:left;line-height:49px;height:49px;vertical-align:middle"><h2 style="font-size:14px;font-weight:normal;line-height:inherit;margin:0;padding:0">This page could not be found<!-- -->.<!-- --></h2></div></div></div><div></div><div></div><div></div><div class="ToastWrapper-sc-1a33ph1-0 kOVcuR"><div class="Toastify"></div></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"statusCode":404},"currentTheme":{"accent":{"alert":"#F6540B","blue":"#4877FF","blueLight":"rgba(72, 119, 255, 0.5)","contentDefaultTransparent":"rgba(174, 174, 174, 0.5)","cyan":"#65E3FF","cyanLight":"rgba(101, 227, 255, 0.3)","cyanTransparent":"rgba(101, 227, 255, 0.12)","dbt":"#fc6949","dbtLight":"rgba(252, 105, 73, 0.5)","info":"#00ABFF","infoTransparent":"rgba(0, 171, 255, 0.5)","negative":"#FF1E59","negativeTransparent":"rgba(255, 30, 89, 0.3)","pink":"#FF4FF8","pinkLight":"rgb(255, 79, 248, 0.5)","positive":"#00A81A","primaryTransparent":"rgba(155, 108, 167, 0.5)","purple":"#7D55EC","purpleLight":"rgba(125, 85, 236, 0.5)","rose":"#D1A2AB","roseLight":"rgba(209, 162, 171, 0.5)","sky":"#6AA1E0","skyLight":"rgba(106, 161, 224, 0.05)","teal":"#00B4CC","tealLight":"rgba(0, 180, 204, 0.5)","warning":"#DD9900","warningTransparent":"rgba(221, 153, 0, 0.5)","yellow":"#FFCC19","yellowLight":"rgba(255, 204, 25, 0.5)"},"background":{"chartBlock":"#2E3036","codeArea":"#1E1F24","codeTextarea":"#000000","content":"#1B1C20","danger":"#FFD0DB","dark":"#B1B8C3","dashboard":"#18181C","header":"#1B1B1B","menu":"#0F4CFF","muted":"#F9FAFC","navigation":"#EDEDED","output":"#2E3036","page":"#1E1F24","panel":"#232429","popup":"#27292E","row":"#2C2C2C","row2":"#51535C","scrollbarThumb":"rgba(100, 100, 100, 0.5)","scrollbarThumbHover":"rgba(255, 255, 255, 0.3)","scrollbarTrack":"#2E3036","success":"#8ADE00","successLight":"rgb(138, 222, 0, 0.3)","table":"#292A2F","tag":"#3A4550"},"borders":{"button":"#454850","contrast":"#FFFFFF","danger":"#FF144D","dark":"#000000","info":"#FFCC19","light":"#2F3034","medium":"#1C1C1C","medium2":"#141414","success":"#2FCB52"},"brand":{"earth100":"#C6EEDB","earth200":"#9DDFBF","earth300":"#6BBF96","earth400":"#37A46F","earth400Transparent":"rgba(55, 164, 111, 0.4)","earth500":"#00954C","energy100":"#FFF4BA","energy200":"#FFED92","energy300":"#FFE662","energy400":"#FFDA19","energy400Transparent":"rgba(255, 218, 25, 0.04)","energy500":"#F6C000","fire100":"#FFD7E0","fire200":"#FFA3B9","fire300":"#FF547D","fire400":"#FF144D","fire400Transparent":"rgba(255, 20, 77, 0.4)","fire500":"#EB0032","stone100":"#F3E6D7","stone200":"#E3D4C2","stone400":"#BFA78B","stone500":"#AF8859","water100":"#BDCEFF","water200":"#81A1FF","water300":"#517DFF","water400":"#2A60FE","water400Transparent":"rgba(42, 96, 254, 0.4)","water500":"#0F4CFF","wind100":"#EEEAFF","wind200":"#CCC1F4","wind300":"#A698DD","wind400":"#6B50D7","wind400SuperTransparent":"rgba(107, 80, 215, 0.12)","wind400Transparent":"rgba(107, 80, 215, 0.4)","wind500":"#4E32BC"},"chart":{"backgroundPrimary":"#7D55EC","backgroundSecondary":"#FF144D","backgroundTertiary":"#86E2FF","button1":"#4877FF","button2":"#FFCC19","button3":"#8ADE00","button4":"#FF4FF8","button5":"#B98D95","lines":"#9B6CA7","primary":"#6B50D7","secondary":"#FF144D","tertiary":"#2A60FE"},"content":{"active":"#FFFFFF","default":"#AEAEAE","disabled":"rgba(255, 255, 255, 0.3)","inverted":"#2C2C2C","muted":"#787A85"},"elevation":{"visualizationAccent":"#996CFF","visualizationAccentAlt":"#C1ACF7"},"feature":{"active":"rgba(250, 248, 254, 0.14)","disabled":"rgba(201, 206, 218, 0.12)"},"icons":{"neutral":"#787878"},"interactive":{"activeBorder":"#060606","checked":"#060606","dangerBorder":"#FF144D","defaultBackground":"#36383F","defaultBorder":"#2E3036","disabledBorder":"#B1B8C3","focusBackground":"#B1B8C3","focusBorder":"#86E2FF","hoverBackground":"#4E4E4E","hoverBorder":"#B9BFCA","hoverOverlay":"rgba(255, 255, 255, 0.1)","linkPrimary":"#1752FF","linkPrimaryHover":"#4877FF","linkPrimaryLight":"#5982ff","linkSecondary":"#6B50D7","linkSecondaryDisabled":"#C4B9EF","linkText":"#6AA1E0","rowHoverBackground":"rgba(0, 0, 0, 0.1)","transparent":"rgba(255, 255, 255, 0)"},"loader":{"color":"#EB0032","colorInverted":"#8ADE00"},"logo":{"color":"#FFFFFF"},"monotone":{"black":"#060606","blackTransparent":"rgba(0, 0, 0, 0.6)","gray":"#B1B8C3","grey100":"#F2F2F2","grey200":"#D5D7DC","grey300":"#B4B8C0","grey400":"#70747C","grey500":"#51535C","purple":"#6B50D7","white":"#FFFFFF"},"neutral":{"n100":"#E7E8EA","n200":"#D8DADE","n300":"#CBCCD0","n400":"#BCBEC4","n500":"#AEB0B6"},"progress":{"negative":"#FF144D","positive":"#6B50D7"},"shadow":{"base":"12px 40px 120px rgba(106, 117, 139, 0.4)","frame":"0px 10px 40px rgba(0, 0, 0, 0.26)","menu":"4px 10px 20px rgba(6, 6, 6, 0.12)","popup":"10px 20px 40px rgba(0, 0, 0, 0.2)","small":"0px, 4px, rgba(0, 0, 0, 0.25)","window":"0px 10px 60px rgba(0, 0, 0, 0.7)"},"status":{"negative":"#FF144D","positive":"#24B400"},"text":{"fileBrowser":"#787A85"}}},"page":"/_error","query":{},"buildId":"
|
|
22
|
+
}</style><h1 class="next-error-h1" style="display:inline-block;margin:0;margin-right:20px;padding:10px 23px 10px 0;font-size:24px;font-weight:500;vertical-align:top">404<!-- --></h1><div style="display:inline-block;text-align:left;line-height:49px;height:49px;vertical-align:middle"><h2 style="font-size:14px;font-weight:normal;line-height:inherit;margin:0;padding:0">This page could not be found<!-- -->.<!-- --></h2></div></div></div><div></div><div></div><div></div><div class="ToastWrapper-sc-1a33ph1-0 kOVcuR"><div class="Toastify"></div></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"statusCode":404},"currentTheme":{"accent":{"alert":"#F6540B","blue":"#4877FF","blueLight":"rgba(72, 119, 255, 0.5)","contentDefaultTransparent":"rgba(174, 174, 174, 0.5)","cyan":"#65E3FF","cyanLight":"rgba(101, 227, 255, 0.3)","cyanTransparent":"rgba(101, 227, 255, 0.12)","dbt":"#fc6949","dbtLight":"rgba(252, 105, 73, 0.5)","info":"#00ABFF","infoTransparent":"rgba(0, 171, 255, 0.5)","negative":"#FF1E59","negativeTransparent":"rgba(255, 30, 89, 0.3)","pink":"#FF4FF8","pinkLight":"rgb(255, 79, 248, 0.5)","positive":"#00A81A","primaryTransparent":"rgba(155, 108, 167, 0.5)","purple":"#7D55EC","purpleLight":"rgba(125, 85, 236, 0.5)","rose":"#D1A2AB","roseLight":"rgba(209, 162, 171, 0.5)","sky":"#6AA1E0","skyLight":"rgba(106, 161, 224, 0.05)","teal":"#00B4CC","tealLight":"rgba(0, 180, 204, 0.5)","warning":"#DD9900","warningTransparent":"rgba(221, 153, 0, 0.5)","yellow":"#FFCC19","yellowLight":"rgba(255, 204, 25, 0.5)"},"background":{"chartBlock":"#2E3036","codeArea":"#1E1F24","codeTextarea":"#000000","content":"#1B1C20","danger":"#FFD0DB","dark":"#B1B8C3","dashboard":"#18181C","header":"#1B1B1B","menu":"#0F4CFF","muted":"#F9FAFC","navigation":"#EDEDED","output":"#2E3036","page":"#1E1F24","panel":"#232429","popup":"#27292E","row":"#2C2C2C","row2":"#51535C","scrollbarThumb":"rgba(100, 100, 100, 0.5)","scrollbarThumbHover":"rgba(255, 255, 255, 0.3)","scrollbarTrack":"#2E3036","success":"#8ADE00","successLight":"rgb(138, 222, 0, 0.3)","table":"#292A2F","tag":"#3A4550"},"borders":{"button":"#454850","contrast":"#FFFFFF","danger":"#FF144D","dark":"#000000","info":"#FFCC19","light":"#2F3034","medium":"#1C1C1C","medium2":"#141414","success":"#2FCB52"},"brand":{"earth100":"#C6EEDB","earth200":"#9DDFBF","earth300":"#6BBF96","earth400":"#37A46F","earth400Transparent":"rgba(55, 164, 111, 0.4)","earth500":"#00954C","energy100":"#FFF4BA","energy200":"#FFED92","energy300":"#FFE662","energy400":"#FFDA19","energy400Transparent":"rgba(255, 218, 25, 0.04)","energy500":"#F6C000","fire100":"#FFD7E0","fire200":"#FFA3B9","fire300":"#FF547D","fire400":"#FF144D","fire400Transparent":"rgba(255, 20, 77, 0.4)","fire500":"#EB0032","stone100":"#F3E6D7","stone200":"#E3D4C2","stone400":"#BFA78B","stone500":"#AF8859","water100":"#BDCEFF","water200":"#81A1FF","water300":"#517DFF","water400":"#2A60FE","water400Transparent":"rgba(42, 96, 254, 0.4)","water500":"#0F4CFF","wind100":"#EEEAFF","wind200":"#CCC1F4","wind300":"#A698DD","wind400":"#6B50D7","wind400SuperTransparent":"rgba(107, 80, 215, 0.12)","wind400Transparent":"rgba(107, 80, 215, 0.4)","wind500":"#4E32BC"},"chart":{"backgroundPrimary":"#7D55EC","backgroundSecondary":"#FF144D","backgroundTertiary":"#86E2FF","button1":"#4877FF","button2":"#FFCC19","button3":"#8ADE00","button4":"#FF4FF8","button5":"#B98D95","lines":"#9B6CA7","primary":"#6B50D7","secondary":"#FF144D","tertiary":"#2A60FE"},"content":{"active":"#FFFFFF","default":"#AEAEAE","disabled":"rgba(255, 255, 255, 0.3)","inverted":"#2C2C2C","muted":"#787A85"},"elevation":{"visualizationAccent":"#996CFF","visualizationAccentAlt":"#C1ACF7"},"feature":{"active":"rgba(250, 248, 254, 0.14)","disabled":"rgba(201, 206, 218, 0.12)"},"icons":{"neutral":"#787878"},"interactive":{"activeBorder":"#060606","checked":"#060606","dangerBorder":"#FF144D","defaultBackground":"#36383F","defaultBorder":"#2E3036","disabledBorder":"#B1B8C3","focusBackground":"#B1B8C3","focusBorder":"#86E2FF","hoverBackground":"#4E4E4E","hoverBorder":"#B9BFCA","hoverOverlay":"rgba(255, 255, 255, 0.1)","linkPrimary":"#1752FF","linkPrimaryHover":"#4877FF","linkPrimaryLight":"#5982ff","linkSecondary":"#6B50D7","linkSecondaryDisabled":"#C4B9EF","linkText":"#6AA1E0","rowHoverBackground":"rgba(0, 0, 0, 0.1)","transparent":"rgba(255, 255, 255, 0)"},"loader":{"color":"#EB0032","colorInverted":"#8ADE00"},"logo":{"color":"#FFFFFF"},"monotone":{"black":"#060606","blackTransparent":"rgba(0, 0, 0, 0.6)","gray":"#B1B8C3","grey100":"#F2F2F2","grey200":"#D5D7DC","grey300":"#B4B8C0","grey400":"#70747C","grey500":"#51535C","purple":"#6B50D7","white":"#FFFFFF"},"neutral":{"n100":"#E7E8EA","n200":"#D8DADE","n300":"#CBCCD0","n400":"#BCBEC4","n500":"#AEB0B6"},"progress":{"negative":"#FF144D","positive":"#6B50D7"},"shadow":{"base":"12px 40px 120px rgba(106, 117, 139, 0.4)","frame":"0px 10px 40px rgba(0, 0, 0, 0.26)","menu":"4px 10px 20px rgba(6, 6, 6, 0.12)","popup":"10px 20px 40px rgba(0, 0, 0, 0.2)","small":"0px, 4px, rgba(0, 0, 0, 0.25)","window":"0px 10px 60px rgba(0, 0, 0, 0.7)"},"status":{"negative":"#FF144D","positive":"#24B400"},"text":{"fileBrowser":"#787A85"}}},"page":"/_error","query":{},"buildId":"OwvTROmputQCA1Z9HEGDv","nextExport":true,"isFallback":false,"gip":true,"appGip":true,"scriptLoader":[]}</script></body></html>
|