mage-ai 0.8.97__py3-none-any.whl → 0.8.99__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mage-ai might be problematic. Click here for more details.

Files changed (152) hide show
  1. mage_ai/api/policies/AutocompleteItemPolicy.py +2 -1
  2. mage_ai/api/policies/BasePolicy.py +2 -2
  3. mage_ai/api/policies/BlockTemplatePolicy.py +2 -1
  4. mage_ai/api/policies/ClusterPolicy.py +2 -1
  5. mage_ai/api/policies/DataProviderPolicy.py +2 -1
  6. mage_ai/api/policies/EventRulePolicy.py +2 -1
  7. mage_ai/api/policies/ExtensionOptionPolicy.py +2 -1
  8. mage_ai/api/policies/FileVersionPolicy.py +2 -1
  9. mage_ai/api/policies/GitBranchPolicy.py +9 -0
  10. mage_ai/api/policies/KernelPolicy.py +2 -1
  11. mage_ai/api/policies/LogPolicy.py +2 -2
  12. mage_ai/api/policies/OauthPolicy.py +15 -0
  13. mage_ai/api/policies/OutputPolicy.py +2 -2
  14. mage_ai/api/policies/PipelinePolicy.py +2 -2
  15. mage_ai/api/policies/PipelineRunPolicy.py +2 -2
  16. mage_ai/api/policies/PipelineSchedulePolicy.py +2 -2
  17. mage_ai/api/policies/PullRequestPolicy.py +64 -0
  18. mage_ai/api/policies/SessionPolicy.py +4 -1
  19. mage_ai/api/policies/VariablePolicy.py +2 -2
  20. mage_ai/api/policies/WidgetPolicy.py +2 -2
  21. mage_ai/api/policies/WorkspacePolicy.py +3 -3
  22. mage_ai/api/presenters/PipelinePresenter.py +1 -0
  23. mage_ai/api/presenters/PullRequestPresenter.py +16 -0
  24. mage_ai/api/presenters/StatusPresenter.py +2 -0
  25. mage_ai/api/presenters/SyncPresenter.py +1 -0
  26. mage_ai/api/presenters/WorkspacePresenter.py +2 -0
  27. mage_ai/api/resources/GitBranchResource.py +81 -26
  28. mage_ai/api/resources/OauthResource.py +31 -4
  29. mage_ai/api/resources/PipelineResource.py +8 -1
  30. mage_ai/api/resources/PullRequestResource.py +87 -0
  31. mage_ai/api/resources/RoleResource.py +6 -3
  32. mage_ai/api/resources/SecretResource.py +2 -5
  33. mage_ai/api/resources/SessionResource.py +18 -0
  34. mage_ai/api/resources/StatusResource.py +7 -3
  35. mage_ai/api/resources/UserResource.py +11 -16
  36. mage_ai/api/resources/WorkspaceResource.py +83 -53
  37. mage_ai/authentication/oauth/active_directory.py +17 -0
  38. mage_ai/authentication/oauth/constants.py +9 -0
  39. mage_ai/authentication/oauth/utils.py +2 -1
  40. mage_ai/authentication/oauth2.py +9 -3
  41. mage_ai/cli/main.py +94 -51
  42. mage_ai/cluster_manager/kubernetes/workload_manager.py +141 -45
  43. mage_ai/data_preparation/git/__init__.py +86 -16
  44. mage_ai/data_preparation/git/api.py +175 -0
  45. mage_ai/data_preparation/models/block/dbt/utils/__init__.py +49 -14
  46. mage_ai/data_preparation/models/block/sql/__init__.py +3 -2
  47. mage_ai/data_preparation/models/pipeline.py +4 -1
  48. mage_ai/data_preparation/models/pipelines/integration_pipeline.py +7 -3
  49. mage_ai/data_preparation/preferences.py +4 -2
  50. mage_ai/data_preparation/repo_manager.py +41 -10
  51. mage_ai/data_preparation/shared/secrets.py +5 -6
  52. mage_ai/data_preparation/sync/__init__.py +2 -1
  53. mage_ai/data_preparation/sync/git_sync.py +2 -5
  54. mage_ai/data_preparation/templates/utils.py +2 -0
  55. mage_ai/orchestration/db/models/oauth.py +22 -4
  56. mage_ai/orchestration/pipeline_scheduler.py +19 -8
  57. mage_ai/orchestration/queue/process_queue.py +15 -12
  58. mage_ai/server/api/clusters.py +21 -11
  59. mage_ai/server/constants.py +1 -1
  60. mage_ai/server/frontend_dist/404.html +2 -2
  61. mage_ai/server/frontend_dist/404.html.html +2 -2
  62. mage_ai/server/frontend_dist/_next/static/WRxCTOtmZhTqQws_7OJZD/_buildManifest.js +1 -0
  63. mage_ai/server/frontend_dist/_next/static/chunks/{1286-993725c925c56a98.js → 1286-b90bd4b7f8abfc3a.js} +1 -1
  64. mage_ai/server/frontend_dist/_next/static/chunks/{1424-f475cae42f8a7fca.js → 1424-90c0f66ba2f86b88.js} +1 -1
  65. mage_ai/server/frontend_dist/_next/static/chunks/3883-c95563b9f60ae526.js +1 -0
  66. mage_ai/server/frontend_dist/_next/static/chunks/6694-c8f2a68074420906.js +1 -0
  67. mage_ai/server/frontend_dist/_next/static/chunks/{9350-1ff50f1d7b9ee754.js → 9350-5191c83a8d0cf454.js} +1 -1
  68. mage_ai/server/frontend_dist/_next/static/chunks/pages/{_app-3527178abd99bc87.js → _app-171846e16d26855a.js} +1 -1
  69. mage_ai/server/frontend_dist/_next/static/chunks/pages/files-e4e778f8f5e1bf2e.js +1 -0
  70. mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/settings-c788c1b127999825.js +1 -0
  71. mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/users/[user]-b4650224a19e8fe6.js +1 -0
  72. mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/users/new-931eb719e3fae29c.js +1 -0
  73. mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/users-d3724bde0b186dd9.js +1 -0
  74. mage_ai/server/frontend_dist/_next/static/chunks/pages/manage-af11f9cf94024ac0.js +1 -0
  75. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills/{[...slug]-3ec5eb9562e4bff4.js → [...slug]-34326db259f922d1.js} +1 -1
  76. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-503ecb7a72257b79.js +1 -0
  77. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/{[run]-7667080098731e30.js → [run]-2994b8ab7862c07b.js} +1 -1
  78. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs-7b31b851e2544b42.js +1 -0
  79. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers/{[...slug]-e18058e13882b20d.js → [...slug]-4445619d4eabe065.js} +1 -1
  80. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/{triggers-6854c10d5589d394.js → triggers-b7db0b682fadb840.js} +1 -1
  81. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/account/profile-ee0931af3abb55b3.js +1 -0
  82. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-f8a59d718751be9a.js +1 -0
  83. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-90f8830890036eb2.js +1 -0
  84. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/users-9f82673fc438ea83.js +1 -0
  85. mage_ai/server/frontend_dist/_next/static/chunks/pages/sign-in-a1871b8a537d823c.js +1 -0
  86. mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-48859b4e9c846212.js +1 -0
  87. mage_ai/server/frontend_dist/files.html +2 -2
  88. mage_ai/server/frontend_dist/index.html +2 -2
  89. mage_ai/server/frontend_dist/manage/settings.html +24 -0
  90. mage_ai/server/frontend_dist/manage/users/[user].html +2 -2
  91. mage_ai/server/frontend_dist/manage/users/new.html +24 -0
  92. mage_ai/server/frontend_dist/manage/users.html +2 -2
  93. mage_ai/server/frontend_dist/manage.html +2 -2
  94. mage_ai/server/frontend_dist/pipeline-runs.html +2 -2
  95. mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills/[...slug].html +2 -2
  96. mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills.html +2 -2
  97. mage_ai/server/frontend_dist/pipelines/[pipeline]/edit.html +2 -2
  98. mage_ai/server/frontend_dist/pipelines/[pipeline]/logs.html +2 -2
  99. mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runs.html +2 -2
  100. mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runtime.html +2 -2
  101. mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors.html +2 -2
  102. mage_ai/server/frontend_dist/pipelines/[pipeline]/runs/[run].html +2 -2
  103. mage_ai/server/frontend_dist/pipelines/[pipeline]/runs.html +2 -2
  104. mage_ai/server/frontend_dist/pipelines/[pipeline]/settings.html +2 -2
  105. mage_ai/server/frontend_dist/pipelines/[pipeline]/syncs.html +2 -2
  106. mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers/[...slug].html +2 -2
  107. mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers.html +2 -2
  108. mage_ai/server/frontend_dist/pipelines/[pipeline].html +2 -2
  109. mage_ai/server/frontend_dist/pipelines.html +2 -2
  110. mage_ai/server/frontend_dist/settings/account/profile.html +2 -2
  111. mage_ai/server/frontend_dist/settings/workspace/preferences.html +2 -2
  112. mage_ai/server/frontend_dist/settings/workspace/sync-data.html +2 -2
  113. mage_ai/server/frontend_dist/settings/workspace/users.html +2 -2
  114. mage_ai/server/frontend_dist/settings.html +2 -2
  115. mage_ai/server/frontend_dist/sign-in.html +2 -2
  116. mage_ai/server/frontend_dist/terminal.html +2 -2
  117. mage_ai/server/frontend_dist/test.html +2 -2
  118. mage_ai/server/frontend_dist/triggers.html +2 -2
  119. mage_ai/server/frontend_dist/version-control.html +2 -2
  120. mage_ai/server/scheduler_manager.py +7 -2
  121. mage_ai/server/server.py +37 -3
  122. mage_ai/server/terminal_server.py +2 -2
  123. mage_ai/server/websocket_server.py +6 -2
  124. mage_ai/services/newrelic/__init__.py +21 -0
  125. mage_ai/settings/__init__.py +32 -0
  126. mage_ai/shared/hash.py +2 -0
  127. mage_ai/tests/api/test_utils.py +29 -2
  128. mage_ai/tests/data_preparation/models/test_pipeline.py +5 -0
  129. {mage_ai-0.8.97.dist-info → mage_ai-0.8.99.dist-info}/METADATA +8 -3
  130. {mage_ai-0.8.97.dist-info → mage_ai-0.8.99.dist-info}/RECORD +136 -127
  131. mage_ai/data_preparation/templates/main/projects/__init__.py +0 -0
  132. mage_ai/server/frontend_dist/_next/static/YLZRSrQ0aqtl-GGePfsMB/_buildManifest.js +0 -1
  133. mage_ai/server/frontend_dist/_next/static/chunks/3077-d58f18ed770e5137.js +0 -1
  134. mage_ai/server/frontend_dist/_next/static/chunks/3714-b676173cd4d8d86c.js +0 -1
  135. mage_ai/server/frontend_dist/_next/static/chunks/pages/files-82b5409dac9564f4.js +0 -1
  136. mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/users/[user]-bb6aaa23e92a5add.js +0 -1
  137. mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/users-c91ee702a4cd7a6f.js +0 -1
  138. mage_ai/server/frontend_dist/_next/static/chunks/pages/manage-7961010cb0fb9abd.js +0 -1
  139. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-7b8ce89f0d717465.js +0 -1
  140. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs-5bd17a8f3f3d57ef.js +0 -1
  141. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/account/profile-7d75e42d5f4936bb.js +0 -1
  142. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-8220c1200472bf70.js +0 -1
  143. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-b602fa9b6ffabd12.js +0 -1
  144. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/users-3f9d5800f268a263.js +0 -1
  145. mage_ai/server/frontend_dist/_next/static/chunks/pages/sign-in-2925c2c1b0c5559a.js +0 -1
  146. mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-5ffc663cfb0ec81e.js +0 -1
  147. /mage_ai/server/frontend_dist/_next/static/{YLZRSrQ0aqtl-GGePfsMB → WRxCTOtmZhTqQws_7OJZD}/_middlewareManifest.js +0 -0
  148. /mage_ai/server/frontend_dist/_next/static/{YLZRSrQ0aqtl-GGePfsMB → WRxCTOtmZhTqQws_7OJZD}/_ssgManifest.js +0 -0
  149. {mage_ai-0.8.97.dist-info → mage_ai-0.8.99.dist-info}/LICENSE +0 -0
  150. {mage_ai-0.8.97.dist-info → mage_ai-0.8.99.dist-info}/WHEEL +0 -0
  151. {mage_ai-0.8.97.dist-info → mage_ai-0.8.99.dist-info}/entry_points.txt +0 -0
  152. {mage_ai-0.8.97.dist-info → mage_ai-0.8.99.dist-info}/top_level.txt +0 -0
@@ -1,16 +1,22 @@
1
+ import json
2
+ import urllib.parse
1
3
  from datetime import datetime, timedelta
4
+
2
5
  from mage_ai.api.errors import ApiError
3
6
  from mage_ai.api.resources.GenericResource import GenericResource
7
+ from mage_ai.authentication.oauth2 import generate_access_token
4
8
  from mage_ai.authentication.oauth.constants import (
9
+ ACTIVE_DIRECTORY_CLIENT_ID,
5
10
  GITHUB_CLIENT_ID,
6
11
  GITHUB_STATE,
12
+ OAUTH_PROVIDER_ACTIVE_DIRECTORY,
7
13
  OAUTH_PROVIDER_GITHUB,
14
+ VALID_OAUTH_PROVIDERS,
8
15
  )
9
16
  from mage_ai.authentication.oauth.utils import access_tokens_for_provider
10
- from mage_ai.authentication.oauth2 import generate_access_token
11
17
  from mage_ai.orchestration.db import safe_db_query
12
18
  from mage_ai.orchestration.db.models.oauth import Oauth2AccessToken, Oauth2Application
13
- import urllib.parse
19
+ from mage_ai.settings import ACTIVE_DIRECTORY_DIRECTORY_ID
14
20
 
15
21
 
16
22
  class OauthResource(GenericResource):
@@ -22,7 +28,7 @@ class OauthResource(GenericResource):
22
28
  provider = payload.get('provider')
23
29
  token = payload.get('token')
24
30
 
25
- if not provider or provider not in [OAUTH_PROVIDER_GITHUB]:
31
+ if not provider or provider not in VALID_OAUTH_PROVIDERS:
26
32
  error.update(dict(message='Invalid provider.'))
27
33
  raise ApiError(error)
28
34
 
@@ -65,7 +71,7 @@ class OauthResource(GenericResource):
65
71
  model = dict(provider=pk)
66
72
 
67
73
  error = ApiError.RESOURCE_INVALID.copy()
68
- if pk not in [OAUTH_PROVIDER_GITHUB]:
74
+ if pk not in VALID_OAUTH_PROVIDERS:
69
75
  error.update(dict(message='Invalid provider.'))
70
76
  raise ApiError(error)
71
77
 
@@ -94,5 +100,26 @@ class OauthResource(GenericResource):
94
100
  query_strings.append(f'{k}={v}')
95
101
 
96
102
  model['url'] = f"https://github.com/login/oauth/authorize?{'&'.join(query_strings)}"
103
+ elif OAUTH_PROVIDER_ACTIVE_DIRECTORY == pk:
104
+ ad_directory_id = ACTIVE_DIRECTORY_DIRECTORY_ID
105
+ if ad_directory_id:
106
+ mage_redirect_uri = '?'.join([
107
+ redirect_uri,
108
+ f'provider={pk}',
109
+ ])
110
+ query = dict(
111
+ client_id=ACTIVE_DIRECTORY_CLIENT_ID,
112
+ redirect_uri=f'https://api.mage.ai/v1/oauth/{pk}',
113
+ response_type='code',
114
+ scope='User.Read',
115
+ state=urllib.parse.quote_plus(json.dumps(dict(
116
+ redirect_uri=mage_redirect_uri,
117
+ tenant_id=ad_directory_id,
118
+ ))),
119
+ )
120
+ query_strings = []
121
+ for k, v in query.items():
122
+ query_strings.append(f'{k}={v}')
123
+ model['url'] = f"https://login.microsoftonline.com/{ad_directory_id}/oauth2/v2.0/authorize?{'&'.join(query_strings)}" # noqa: E501
97
124
 
98
125
  return self(model, user, **kwargs)
@@ -1,5 +1,6 @@
1
1
  import asyncio
2
2
 
3
+ from sqlalchemy import or_
3
4
  from sqlalchemy.orm import aliased
4
5
 
5
6
  from mage_ai.api.operations.constants import DELETE
@@ -79,7 +80,13 @@ class PipelineResource(BaseResource):
79
80
  a.status,
80
81
  a.updated_at,
81
82
  ]).
82
- filter(a.pipeline_uuid.in_(pipeline_uuids))
83
+ filter(
84
+ a.pipeline_uuid.in_(pipeline_uuids),
85
+ or_(
86
+ a.repo_path == get_repo_path(),
87
+ a.repo_path.is_(None),
88
+ )
89
+ )
83
90
  ).all()
84
91
  return group_by(lambda x: x.pipeline_uuid, result)
85
92
 
@@ -0,0 +1,87 @@
1
+ from github import Auth, Github
2
+ from mage_ai.api.errors import ApiError
3
+ from mage_ai.api.resources.GenericResource import GenericResource
4
+ from mage_ai.data_preparation.git import api
5
+ from typing import Dict
6
+
7
+
8
+ def pull_request_to_dict(pr) -> Dict:
9
+ return dict(
10
+ body=pr.body,
11
+ created_at=pr.created_at,
12
+ id=pr.id,
13
+ is_merged=pr.is_merged(),
14
+ last_modified=pr.last_modified,
15
+ merged=pr.merged,
16
+ state=pr.state,
17
+ title=pr.title,
18
+ url=pr.html_url,
19
+ user=pr.user.login,
20
+ )
21
+
22
+
23
+ class PullRequestResource(GenericResource):
24
+ @classmethod
25
+ def collection(self, query, meta, user, **kwargs):
26
+ arr = []
27
+
28
+ repository = query.get('repository', None)
29
+ if repository:
30
+ repository = repository[0]
31
+
32
+ access_token = api.get_access_token_for_user(user)
33
+ if access_token:
34
+ auth = Auth.Token(access_token.token)
35
+ g = Github(auth=auth)
36
+ repo = g.get_repo(repository)
37
+ pulls = repo.get_pulls(
38
+ direction='desc',
39
+ sort='created',
40
+ state='open',
41
+ ).get_page(0)
42
+
43
+ for pr in pulls:
44
+ arr.append(pull_request_to_dict(pr))
45
+
46
+ return self.build_result_set(arr, user, **kwargs)
47
+
48
+ @classmethod
49
+ def create(self, payload, user, **kwargs):
50
+ error = ApiError.RESOURCE_INVALID
51
+
52
+ for key in [
53
+ 'base_branch',
54
+ 'compare_branch',
55
+ 'title',
56
+ ]:
57
+ if key not in payload:
58
+ error.update(dict(message=f'Value for {key} is required but empty.'))
59
+ raise ApiError(error)
60
+
61
+ repository = payload.get('repository')
62
+ if not repository:
63
+ error.update(dict(
64
+ message='Repository is empty, ' +
65
+ 'please select a repository to create a pull request in.',
66
+ ))
67
+ raise ApiError(error)
68
+
69
+ access_token = api.get_access_token_for_user(user)
70
+ if not access_token:
71
+ error.update(dict(
72
+ message='Access token not found, please authenticate with GitHub.',
73
+ ))
74
+ raise ApiError(error)
75
+
76
+ auth = Auth.Token(access_token.token)
77
+ g = Github(auth=auth)
78
+ repo = g.get_repo(repository)
79
+
80
+ pr = repo.create_pull(
81
+ base=payload.get('base_branch'),
82
+ body=payload.get('body'),
83
+ head=payload.get('compare_branch'),
84
+ title=payload.get('title'),
85
+ )
86
+
87
+ return self(pull_request_to_dict(pr), user, **kwargs)
@@ -1,5 +1,5 @@
1
1
  from mage_ai.api.resources.GenericResource import GenericResource
2
- from mage_ai.data_preparation.repo_manager import get_repo_path
2
+ from mage_ai.data_preparation.repo_manager import get_project_uuid
3
3
  from mage_ai.orchestration.db import safe_db_query
4
4
  from mage_ai.orchestration.db.models.oauth import Role
5
5
 
@@ -41,14 +41,17 @@ class RoleResource(GenericResource):
41
41
  roles.append(permission.role)
42
42
  else:
43
43
  roles = Role.query.all()
44
- access = user.get_access(Permission.Entity.PROJECT, get_repo_path())
44
+
45
+ access = 0
46
+ if user:
47
+ access = user.get_access(Permission.Entity.PROJECT, get_project_uuid())
45
48
 
46
49
  if (access & Permission.Access.OWNER == 0) and limit_roles:
47
50
  role_access = Permission.Access.EDITOR | Permission.Access.VIEWER
48
51
  roles = list(filter(
49
52
  lambda role: role.get_access(
50
53
  Permission.Entity.PROJECT,
51
- get_repo_path(),
54
+ get_project_uuid(),
52
55
  ) | role_access == role_access, # Only editors and viewers
53
56
  roles,
54
57
  ))
@@ -2,15 +2,12 @@ from mage_ai.api.errors import ApiError
2
2
  from mage_ai.api.resources.DatabaseResource import DatabaseResource
3
3
  from mage_ai.data_preparation.preferences import get_preferences
4
4
  from mage_ai.data_preparation.repo_manager import get_repo_path
5
- from mage_ai.data_preparation.shared.secrets import (
6
- create_secret,
7
- get_valid_secrets,
8
- )
5
+ from mage_ai.data_preparation.shared.secrets import create_secret, get_valid_secrets
9
6
  from mage_ai.data_preparation.sync import (
10
- GitConfig,
11
7
  GIT_ACCESS_TOKEN_SECRET_NAME,
12
8
  GIT_SSH_PRIVATE_KEY_SECRET_NAME,
13
9
  GIT_SSH_PUBLIC_KEY_SECRET_NAME,
10
+ GitConfig,
14
11
  )
15
12
  from mage_ai.orchestration.db import safe_db_query
16
13
  from mage_ai.orchestration.db.models.secrets import Secret
@@ -4,6 +4,8 @@ from mage_ai.api.errors import ApiError
4
4
  from mage_ai.api.resources.BaseResource import BaseResource
5
5
  from mage_ai.authentication.ldap import new_ldap_connection
6
6
  from mage_ai.authentication.oauth2 import encode_token, generate_access_token
7
+ from mage_ai.authentication.oauth.active_directory import get_user_info
8
+ from mage_ai.authentication.oauth.constants import OAUTH_PROVIDER_ACTIVE_DIRECTORY
7
9
  from mage_ai.authentication.passwords import verify_password
8
10
  from mage_ai.orchestration.db import safe_db_query
9
11
  from mage_ai.orchestration.db.models.oauth import Role, User
@@ -18,6 +20,22 @@ class SessionResource(BaseResource):
18
20
  email = payload.get('email')
19
21
  password = payload.get('password')
20
22
  username = payload.get('username')
23
+ token = payload.get('token')
24
+ provider = payload.get('provider')
25
+
26
+ if token and provider:
27
+ if provider == OAUTH_PROVIDER_ACTIVE_DIRECTORY:
28
+ user_info = get_user_info(token)
29
+ principal_name = user_info.get('userPrincipalName')
30
+ user = User.query.filter(User.email == principal_name).first()
31
+ if not user: # noqa: E712
32
+ print('first user login, creating user.')
33
+ user = User.create(
34
+ username=principal_name,
35
+ email=principal_name,
36
+ )
37
+ oauth_token = generate_access_token(user, kwargs['oauth_client'])
38
+ return self(oauth_token, user, **kwargs)
21
39
 
22
40
  error = ApiError.RESOURCE_NOT_FOUND
23
41
  error.update({'message': 'Email/username and/or password invalid.'})
@@ -3,8 +3,8 @@ import os
3
3
  from mage_ai.api.resources.GenericResource import GenericResource
4
4
  from mage_ai.data_preparation.models.constants import MAX_PRINT_OUTPUT_LINES
5
5
  from mage_ai.data_preparation.repo_manager import (
6
- ProjectType,
7
6
  get_project_type,
7
+ get_project_uuid,
8
8
  get_repo_config,
9
9
  get_repo_path,
10
10
  )
@@ -32,8 +32,10 @@ class StatusResource(GenericResource):
32
32
  KUBE_NAMESPACE,
33
33
  )
34
34
  instance_type = None
35
- if get_project_type() == ProjectType.MAIN:
36
- instance_type = get_repo_config().cluster_type
35
+ project_type = get_project_type()
36
+ repo_config = get_repo_config()
37
+ if repo_config.cluster_type:
38
+ instance_type = repo_config.cluster_type
37
39
  elif os.getenv(ECS_CLUSTER_NAME):
38
40
  instance_type = ClusterType.ECS
39
41
  elif os.getenv(GCP_PROJECT_ID):
@@ -56,5 +58,7 @@ class StatusResource(GenericResource):
56
58
  'disable_pipeline_edit_access': is_disable_pipeline_edit_access(),
57
59
  'max_print_output_lines': MAX_PRINT_OUTPUT_LINES,
58
60
  'require_user_authentication': REQUIRE_USER_AUTHENTICATION,
61
+ 'project_type': project_type,
62
+ 'project_uuid': get_project_uuid(),
59
63
  }
60
64
  return self.build_result_set([status], user, **kwargs)
@@ -7,7 +7,7 @@ from mage_ai.authentication.passwords import (
7
7
  generate_salt,
8
8
  verify_password,
9
9
  )
10
- from mage_ai.data_preparation.repo_manager import get_repo_path
10
+ from mage_ai.data_preparation.repo_manager import get_project_uuid
11
11
  from mage_ai.orchestration.db import safe_db_query
12
12
  from mage_ai.orchestration.db.models.oauth import Permission, Role, User
13
13
  from mage_ai.shared.hash import extract, ignore_keys
@@ -30,7 +30,7 @@ class UserResource(DatabaseResource):
30
30
  order_by(User.username.asc())
31
31
  )
32
32
 
33
- if user.is_admin:
33
+ if user and user.is_admin:
34
34
  results = list(filter(lambda user: user.project_access & 3 == 0, results))
35
35
 
36
36
  return results
@@ -48,12 +48,9 @@ class UserResource(DatabaseResource):
48
48
  role_ids = payload.get('roles_new', [])
49
49
  roles_new = self.check_roles(role_ids)
50
50
 
51
- missing_values = []
52
- if len(roles_new) == 0:
53
- missing_values.append('roles')
54
-
55
51
  payload['roles_new'] = roles_new
56
52
 
53
+ missing_values = []
57
54
  for key in ['email', 'password']:
58
55
  if not payload.get(key):
59
56
  missing_values.append(key)
@@ -123,18 +120,13 @@ class UserResource(DatabaseResource):
123
120
  role_ids = payload.get('roles_new', [])
124
121
  roles_new = self.check_roles(role_ids)
125
122
 
126
- missing_values = []
127
- if len(roles_new) == 0:
128
- missing_values.append('roles')
129
-
130
123
  payload['roles_new'] = roles_new
131
124
 
132
- if len(missing_values) >= 1:
133
- error.update(
134
- {'message': 'Missing required values: {}.'.format(', '.join(missing_values))})
135
- raise ApiError(error)
136
-
137
- access = get_access_for_roles(roles_new, Permission.Entity.PROJECT, get_repo_path())
125
+ access = get_access_for_roles(
126
+ roles_new,
127
+ Permission.Entity.PROJECT,
128
+ get_project_uuid(),
129
+ )
138
130
 
139
131
  if self.current_user.is_admin:
140
132
  if self.owner:
@@ -187,6 +179,9 @@ class UserResource(DatabaseResource):
187
179
  'password',
188
180
  'password_confirmation',
189
181
  'password_current',
182
+ 'owner',
183
+ 'project_access',
184
+ 'roles_display',
190
185
  ]), **kwargs)
191
186
 
192
187
  @safe_db_query
@@ -1,7 +1,8 @@
1
1
  import os
2
- import shutil
2
+ import uuid
3
3
  from typing import Dict, List
4
4
 
5
+ import ruamel.yaml
5
6
  import yaml
6
7
 
7
8
  from mage_ai.api.errors import ApiError
@@ -14,21 +15,19 @@ from mage_ai.cluster_manager.constants import (
14
15
  GCP_PROJECT_ID,
15
16
  GCP_REGION,
16
17
  KUBE_NAMESPACE,
17
- KUBE_SERVICE_ACCOUNT_NAME,
18
- KUBE_STORAGE_CLASS_NAME,
19
18
  )
20
19
  from mage_ai.data_preparation.repo_manager import (
21
20
  ProjectType,
22
21
  get_project_type,
23
22
  get_repo_config,
24
23
  get_repo_path,
25
- init_repo,
26
24
  )
27
25
  from mage_ai.data_preparation.shared.constants import MANAGE_ENV_VAR
28
26
  from mage_ai.orchestration.db import safe_db_query
29
27
  from mage_ai.orchestration.db.models.oauth import Permission, Role, User
30
28
  from mage_ai.server.api.clusters import ClusterType
31
29
  from mage_ai.server.logger import Logger
30
+ from mage_ai.settings import REQUIRE_USER_AUTHENTICATION
32
31
 
33
32
  logger = Logger().new_server_logger(__name__)
34
33
 
@@ -47,7 +46,8 @@ class WorkspaceResource(GenericResource):
47
46
  query_user = None
48
47
  if user_id:
49
48
  user_id = user_id[0]
50
- query_user = User.query.get(user_id)
49
+ if user_id:
50
+ query_user = User.query.get(user_id)
51
51
 
52
52
  instances = self.get_instances(cluster_type)
53
53
  instance_map = {
@@ -60,7 +60,7 @@ class WorkspaceResource(GenericResource):
60
60
  repo_path = get_repo_path()
61
61
  projects_folder = os.path.join(repo_path, 'projects')
62
62
  if is_main_project:
63
- projects = [f.name for f in os.scandir(projects_folder) if f.is_dir()]
63
+ projects = [f.name.split('.')[0] for f in os.scandir(projects_folder) if not f.is_dir()]
64
64
  else:
65
65
  projects = [instance.get('name') for instance in instances]
66
66
 
@@ -68,19 +68,25 @@ class WorkspaceResource(GenericResource):
68
68
  for project in projects:
69
69
  if project in instance_map:
70
70
  try:
71
- repo_path = os.path.join(projects_folder, project)
72
71
  workspace = dict(
73
72
  name=project,
74
- repo_path=repo_path,
75
73
  cluster_type=cluster_type,
76
74
  instance=instance_map[project],
77
75
  )
78
- if is_main_project and query_user:
79
- workspace['access'] = query_user.get_access(
80
- Permission.Entity.PROJECT,
81
- repo_path,
82
- )
76
+ if is_main_project:
77
+ workspace_file = os.path.join(projects_folder, f'{project}.yaml')
78
+ config = {}
79
+ with open(workspace_file) as f:
80
+ config = yaml.full_load(f.read())
81
+ project_uuid = config['project_uuid']
82
+ workspace['project_uuid'] = project_uuid
83
+ if query_user:
84
+ workspace['access'] = query_user.get_access(
85
+ Permission.Entity.PROJECT,
86
+ project_uuid,
87
+ )
83
88
  workspaces.append(workspace)
89
+
84
90
  except Exception as e:
85
91
  print(f'Error fetching workspace: {str(e)}')
86
92
 
@@ -113,18 +119,33 @@ class WorkspaceResource(GenericResource):
113
119
  if not cluster_type:
114
120
  cluster_type = payload.get('cluster_type')
115
121
 
116
- workspace_name = payload.get('name')
117
-
118
122
  error = ApiError.RESOURCE_ERROR.copy()
123
+ workspace_name = payload.pop('name')
124
+ if not workspace_name:
125
+ error.update(message='Please enter a valid workspace name.')
126
+ raise ApiError(error)
119
127
 
120
- workspace_folder = None
121
- if get_project_type() == ProjectType.MAIN:
122
- workspace_folder = os.path.join(get_repo_path(), 'projects', workspace_name)
123
- if os.path.exists(workspace_folder):
128
+ workspace_file = None
129
+ project_uuid = None
130
+ project_type = get_project_type()
131
+ if project_type == ProjectType.MAIN:
132
+ project_folder = os.path.join(get_repo_path(), 'projects')
133
+ if not os.path.exists(project_folder):
134
+ os.makedirs(project_folder)
135
+ workspace_file = os.path.join(project_folder, f'{workspace_name}.yaml')
136
+ if os.path.exists(workspace_file):
124
137
  error.update(message=f'Project with name {workspace_name} already exists')
125
138
  raise ApiError(error)
126
139
  try:
127
- init_repo(workspace_folder, project_type=ProjectType.SUB)
140
+ yml = ruamel.yaml.YAML()
141
+ yml.preserve_quotes = True
142
+ yml.indent(mapping=2, sequence=2, offset=0)
143
+
144
+ project_uuid = uuid.uuid4().hex
145
+ data = dict(project_uuid=project_uuid)
146
+
147
+ with open(workspace_file, 'w') as f:
148
+ yml.dump(data, f)
128
149
  except Exception as e:
129
150
  error.update(message=f'Error creating project: {str(e)}')
130
151
  raise ApiError(error)
@@ -133,27 +154,19 @@ class WorkspaceResource(GenericResource):
133
154
  from mage_ai.cluster_manager.kubernetes.workload_manager import (
134
155
  WorkloadManager,
135
156
  )
136
- namespace = payload.get('namespace', os.getenv(KUBE_NAMESPACE))
137
- storage_class_name = payload.get(
138
- 'storage_class_name',
139
- os.getenv(KUBE_STORAGE_CLASS_NAME),
140
- )
141
- service_account_name = payload.get(
142
- 'service_account_name',
143
- os.getenv(KUBE_SERVICE_ACCOUNT_NAME),
144
- )
145
- container_config_yaml = payload.get('container_config')
146
- container_config = None
147
- if container_config_yaml:
148
- container_config = yaml.full_load(container_config_yaml)
157
+ namespace = payload.pop('namespace', os.getenv(KUBE_NAMESPACE))
149
158
 
150
159
  k8s_workload_manager = WorkloadManager(namespace)
151
- k8s_workload_manager.create_deployment(
160
+ extra_args = {}
161
+ if project_type == ProjectType.MAIN:
162
+ extra_args = {
163
+ 'project_type': ProjectType.SUB,
164
+ 'project_uuid': project_uuid,
165
+ }
166
+ k8s_workload_manager.create_workload(
152
167
  workspace_name,
153
- container_config=container_config,
154
- service_account_name=service_account_name,
155
- storage_class_name=storage_class_name,
156
- volume_host_path=workspace_folder,
168
+ **payload,
169
+ **extra_args,
157
170
  )
158
171
  elif cluster_type == ClusterType.ECS:
159
172
  from mage_ai.cluster_manager.aws.ecs_task_manager import EcsTaskManager
@@ -193,15 +206,17 @@ class WorkspaceResource(GenericResource):
193
206
 
194
207
  cloud_run_service_manager.create_service(workspace_name)
195
208
  except Exception as e:
196
- if workspace_folder and os.path.exists(workspace_folder):
197
- shutil.rmtree(workspace_folder)
209
+ if workspace_file and os.path.exists(workspace_file):
210
+ os.remove(workspace_file)
198
211
  error.update(message=str(e))
199
212
  raise ApiError(error)
200
213
 
201
- if get_project_type() == ProjectType.MAIN:
214
+ if project_type == ProjectType.MAIN and \
215
+ project_uuid is not None and \
216
+ REQUIRE_USER_AUTHENTICATION:
202
217
  Role.create_default_roles(
203
218
  entity=Permission.Entity.PROJECT,
204
- entity_id=workspace_folder,
219
+ entity_id=project_uuid,
205
220
  prefix=workspace_name,
206
221
  )
207
222
 
@@ -240,17 +255,32 @@ class WorkspaceResource(GenericResource):
240
255
  instance = self.model.get('instance')
241
256
 
242
257
  repo_path = get_repo_path()
243
- project_folder = os.path.join(repo_path, 'projects', workspace_name)
258
+ workspace_file = os.path.join(repo_path, 'projects', f'{workspace_name}.yaml')
244
259
 
245
- if get_project_type() == ProjectType.MAIN:
246
- shutil.rmtree(project_folder)
247
- if cluster_type == 'ecs':
248
- from mage_ai.cluster_manager.aws.ecs_task_manager import EcsTaskManager
249
- task_arn = instance.get('task_arn')
250
- cluster_name = os.getenv(ECS_CLUSTER_NAME)
260
+ error = ApiError.RESOURCE_ERROR.copy()
251
261
 
252
- ecs_instance_manager = EcsTaskManager(cluster_name)
253
- ecs_instance_manager.delete_task(workspace_name, task_arn)
262
+ try:
263
+ if cluster_type == ClusterType.ECS:
264
+ from mage_ai.cluster_manager.aws.ecs_task_manager import EcsTaskManager
265
+ task_arn = instance.get('task_arn')
266
+ cluster_name = os.getenv(ECS_CLUSTER_NAME)
267
+
268
+ ecs_instance_manager = EcsTaskManager(cluster_name)
269
+ ecs_instance_manager.delete_task(workspace_name, task_arn)
270
+ elif cluster_type == ClusterType.K8S:
271
+ from mage_ai.cluster_manager.kubernetes.workload_manager import (
272
+ WorkloadManager,
273
+ )
274
+ namespace = os.getenv(KUBE_NAMESPACE)
275
+
276
+ k8s_workload_manager = WorkloadManager(namespace)
277
+ k8s_workload_manager.delete_workload(workspace_name)
278
+ except Exception as e:
279
+ error.update(message=str(e))
280
+ raise ApiError(error)
281
+
282
+ if get_project_type() == ProjectType.MAIN:
283
+ os.remove(workspace_file)
254
284
 
255
285
  return self
256
286
 
@@ -265,7 +295,7 @@ class WorkspaceResource(GenericResource):
265
295
  if project_type == ProjectType.MAIN and subproject:
266
296
  repo_path = get_repo_path()
267
297
  projects_folder = os.path.join(repo_path, 'projects')
268
- projects = [name for name in os.listdir(projects_folder) if os.path.isdir(name)]
298
+ projects = [f.name.split('.')[0] for f in os.scandir(projects_folder) if not f.is_dir()]
269
299
  if subproject not in projects:
270
300
  error = ApiError.RESOURCE_NOT_FOUND.copy()
271
301
  error.update(message=f'Project {subproject} was not found.')
@@ -284,7 +314,7 @@ class WorkspaceResource(GenericResource):
284
314
  namespace = os.getenv(KUBE_NAMESPACE)
285
315
  workload_manager = WorkloadManager(namespace)
286
316
 
287
- instances = workload_manager.list_services()
317
+ instances = workload_manager.list_workloads()
288
318
  elif cluster_type == ClusterType.ECS:
289
319
  from mage_ai.cluster_manager.aws.ecs_task_manager import EcsTaskManager
290
320
 
@@ -0,0 +1,17 @@
1
+ from typing import Dict
2
+
3
+ import requests
4
+
5
+
6
+ def get_user_info(token: str) -> Dict:
7
+ url = 'https://graph.microsoft.com/v1.0/me'
8
+
9
+ headers = {
10
+ 'Content-Type': 'application\\json',
11
+ 'Authorization': 'Bearer {}'.format(token)
12
+ }
13
+
14
+ resp = requests.get(url, headers=headers)
15
+ result = resp.json()
16
+
17
+ return result
@@ -1,3 +1,12 @@
1
+ ACTIVE_DIRECTORY_CLIENT_ID = '51aec820-9d49-40a9-b046-17c1f28f620d'
2
+
1
3
  GITHUB_CLIENT_ID = '8577f13ddc81e2848b07'
2
4
  GITHUB_STATE = '1337'
5
+
6
+ OAUTH_PROVIDER_ACTIVE_DIRECTORY = 'active_directory'
3
7
  OAUTH_PROVIDER_GITHUB = 'github'
8
+
9
+ VALID_OAUTH_PROVIDERS = [
10
+ OAUTH_PROVIDER_ACTIVE_DIRECTORY,
11
+ OAUTH_PROVIDER_GITHUB,
12
+ ]
@@ -1,7 +1,8 @@
1
1
  from datetime import datetime
2
- from mage_ai.orchestration.db.models.oauth import Oauth2AccessToken, Oauth2Application
3
2
  from typing import List
4
3
 
4
+ from mage_ai.orchestration.db.models.oauth import Oauth2AccessToken, Oauth2Application
5
+
5
6
 
6
7
  def access_tokens_for_provider(provider: str) -> List[Oauth2AccessToken]:
7
8
  oauth_client = Oauth2Application.query.filter(
@@ -1,9 +1,15 @@
1
+ import os
2
+ import secrets
1
3
  from datetime import datetime, timedelta
2
- from mage_ai.orchestration.db.models.oauth import Oauth2AccessToken, Oauth2Application, User
3
4
  from typing import Dict
5
+
4
6
  import jwt
5
- import os
6
- import secrets
7
+
8
+ from mage_ai.orchestration.db.models.oauth import (
9
+ Oauth2AccessToken,
10
+ Oauth2Application,
11
+ User,
12
+ )
7
13
 
8
14
  JWT_ALGORITHM = 'HS256'
9
15
  JWT_SECRET = os.getenv('JWT_SECRET', 'materia')