mage-ai 0.9.70__py3-none-any.whl → 0.9.71__py3-none-any.whl

mage_ai/api/resources/PipelineRunResource.py

@@ -189,6 +189,7 @@ class PipelineRunResource(DatabaseResource):
     @classmethod
     @safe_db_query
     async def process_collection(self, query_arg, meta, user, **kwargs):
+        context_data = kwargs.get('context_data')
         total_results = await self.collection(query_arg, meta, user, **kwargs)
         total_count = total_results.count()
 
@@ -299,7 +300,11 @@ class PipelineRunResource(DatabaseResource):
         }
 
         if include_pipeline_uuids:
-            pipeline_uuids = Pipeline.get_all_pipelines_all_projects(get_repo_path())
+            repo_path = get_repo_path(context_data=context_data, user=user)
+            pipeline_uuids = Pipeline.get_all_pipelines_all_projects(
+                context_data=context_data,
+                repo_path=repo_path,
+            )
             result_set.metadata['pipeline_uuids'] = pipeline_uuids
 
         return result_set
@@ -309,7 +314,8 @@ class PipelineRunResource(DatabaseResource):
     def create(self, payload, user, **kwargs):
         pipeline_schedule = kwargs.get('parent_model')
 
-        pipeline = Pipeline.get(pipeline_schedule.pipeline_uuid)
+        repo_path = get_repo_path(user=user)
+        pipeline = Pipeline.get(pipeline_schedule.pipeline_uuid, repo_path=repo_path)
         configured_payload, _ = configure_pipeline_run_payload(
             pipeline_schedule,
             pipeline.type,
@@ -332,9 +338,10 @@ class PipelineRunResource(DatabaseResource):
 
     @safe_db_query
     def update(self, payload, **kwargs):
+        repo_path = get_repo_path(user=self.current_user)
         if 'retry_blocks' == payload.get('pipeline_run_action'):
             self.model.refresh()
-            pipeline = Pipeline.get(self.model.pipeline_uuid)
+            pipeline = Pipeline.get(self.model.pipeline_uuid, repo_path=repo_path)
             block_runs_to_retry = []
             from_block_uuid = payload.get('from_block_uuid')
             if from_block_uuid is not None:
@@ -385,6 +392,7 @@ class PipelineRunResource(DatabaseResource):
             pipeline = Pipeline.get(
                 self.model.pipeline_uuid,
                 check_if_exists=True,
+                repo_path=repo_path,
             )
 
             stop_pipeline_run(self.model, pipeline)

mage_ai/api/resources/PipelineScheduleResource.py

@@ -253,7 +253,10 @@ class PipelineScheduleResource(DatabaseResource):
         payload['pipeline_uuid'] = pipeline.uuid
 
         if 'repo_path' not in payload:
-            payload['repo_path'] = (pipeline.repo_path if pipeline else None) or get_repo_path()
+            payload['repo_path'] = (
+                (pipeline.repo_path if pipeline else None)
+                or get_repo_path(user=user)
+            )
         if 'token' not in payload:
             payload['token'] = uuid.uuid4().hex
         if payload.get('status') == ScheduleStatus.ACTIVE and \
@@ -408,7 +411,8 @@ class PipelineScheduleResource(DatabaseResource):
         resource = super().update(payload)
         updated_model = resource.model
 
-        pipeline = Pipeline.get(updated_model.pipeline_uuid)
+        repo_path = get_repo_path(user=self.current_user)
+        pipeline = Pipeline.get(updated_model.pipeline_uuid, repo_path=repo_path)
         if pipeline:
             trigger = Trigger(
                 last_enabled_at=updated_model.last_enabled_at,
@@ -420,6 +424,7 @@ class PipelineScheduleResource(DatabaseResource):
                 sla=updated_model.sla,
                 start_time=updated_model.start_time,
                 status=updated_model.status,
+                token=updated_model.token,
                 variables=updated_model.variables,
             )
 

mage_ai/api/resources/PipelineTriggerResource.py

@@ -7,6 +7,7 @@ from mage_ai.data_preparation.models.triggers import (
 )
 from mage_ai.orchestration.db import safe_db_query
 from mage_ai.orchestration.db.models.schedules import PipelineSchedule
+from mage_ai.settings.repo import get_repo_path
 
 
 class PipelineTriggerResource(GenericResource):
@@ -16,7 +17,10 @@ class PipelineTriggerResource(GenericResource):
         parent_model = kwargs['parent_model']
 
         return self.build_result_set(
-            get_triggers_by_pipeline(parent_model.uuid),
+            get_triggers_by_pipeline(
+                parent_model.uuid,
+                repo_path=get_repo_path(context_data=kwargs.get('context_data'), user=user),
+            ),
             user,
             **kwargs,
         )
@@ -46,6 +50,7 @@ class PipelineTriggerResource(GenericResource):
                 sla=pipeline_schedule.sla,
                 start_time=pipeline_schedule.start_time,
                 status=pipeline_schedule.status,
+                token=pipeline_schedule.token,
                 variables=pipeline_schedule.variables,
             )
         else:

mage_ai/api/resources/ProjectResource.py

@@ -26,8 +26,19 @@ from mage_ai.shared.hash import combine_into, merge_dict
 from mage_ai.usage_statistics.logger import UsageStatisticLogger
 
 
-async def build_project(repo_config=None, root_project: bool = False, **kwargs):
-    project = Project(repo_config=repo_config, root_project=root_project)
+async def build_project(
+    repo_config=None,
+    repo_path: str = None,
+    root_project: bool = False,
+    user=None,
+    **kwargs,
+):
+    project = Project(
+        repo_config=repo_config,
+        context_data=kwargs.get('context_data'),
+        root_project=root_project,
+        user=user,
+    )
 
     model = merge_dict(project.repo_config.to_dict(), dict(
         emr_config=project.emr_config,
@@ -36,7 +47,7 @@ async def build_project(repo_config=None, root_project: bool = False, **kwargs):
         features_override=project.features_override,
         latest_version=await project.latest_version(),
         name=project.name,
-        platform_settings=project.platform_settings,
+        platform_settings=project.platform_settings(),
         project_uuid=project.project_uuid,
         projects=project.projects(),
         remote_variables_dir=project.remote_variables_dir,
@@ -137,13 +148,13 @@ class ProjectResource(GenericResource):
     @classmethod
     @safe_db_query
     async def member(self, _, user, **kwargs):
-        model = await build_project(**kwargs)
+        model = await build_project(user=user, **kwargs)
         return self(model, user, **kwargs)
 
     @safe_db_query
     async def update(self, payload, **kwargs):
         if payload.get('activate_project'):
-            activate_project(payload.get('activate_project'))
+            activate_project(payload.get('activate_project'), user=self.current_user)
 
         platform_settings = payload.get('platform_settings')
         root_project = payload.get('root_project')
@@ -151,7 +162,7 @@ class ProjectResource(GenericResource):
 
         if root_project and platform_settings:
             update_settings(settings=platform_settings)
-            self.model = await build_project(repo_config)
+            self.model = await build_project(repo_config, user=self.current_user)
             return self
 
         data = {}
@@ -206,7 +217,7 @@ class ProjectResource(GenericResource):
         if len(data.keys()) >= 1:
             repo_config.save(**data)
 
-        self.model = await build_project(repo_config)
+        self.model = await build_project(repo_config, user=self.current_user)
 
         if should_log_project:
             await UsageStatisticLogger().project_impression()

mage_ai/api/resources/SecretResource.py

@@ -56,7 +56,7 @@ class SecretResource(DatabaseResource):
     @classmethod
     @safe_db_query
     def member(self, pk, user, **kwargs):
-        repo_path = get_repo_path()
+        repo_path = get_repo_path(user=user)
         model = Secret.query.filter(Secret.repo_name == repo_path, Secret.name == pk).first()
 
         if not model:

mage_ai/api/resources/SeedResource.py

@@ -1,3 +1,6 @@
+import asyncio
+import threading
+
 from mage_ai.api.resources.GenericResource import GenericResource
 from mage_ai.authentication.permissions.seed import bootstrap_permissions
 
@@ -7,6 +10,10 @@ class SeedResource(GenericResource):
     async def create(self, payload, user, **kwargs):
         if payload.get('roles') and payload.get('permissions'):
             policy_names = payload.get('policy_names')
-            await bootstrap_permissions(policy_names=policy_names)
+            t = threading.Thread(
+                target=asyncio.run,
+                args=(bootstrap_permissions(policy_names=policy_names),),
+            )
+            t.start()
 
         return self({}, user, **kwargs)

mage_ai/api/resources/StatusResource.py

@@ -31,9 +31,10 @@ class StatusResource(GenericResource):
             KUBE_NAMESPACE,
         )
 
+        context_data = kwargs.get('context_data')
         instance_type = None
-        project_type = get_project_type()
-        repo_config = get_repo_config()
+        repo_config = get_repo_config(context_data=context_data, user=user)
+        project_type = get_project_type(repo_config=repo_config)
         if repo_config.cluster_type:
             instance_type = repo_config.cluster_type
         elif os.getenv(ECS_CLUSTER_NAME):
@@ -53,10 +54,24 @@ class StatusResource(GenericResource):
 
         status = {
             'is_instance_manager': os.getenv(MANAGE_ENV_VAR) == '1',
-            'repo_path': get_repo_path(root_project=False),
-            'repo_path_relative': get_repo_path(root_project=False, absolute_path=False),
-            'repo_path_relative_root': get_repo_path(root_project=True, absolute_path=False),
-            'repo_path_root': get_repo_path(root_project=True),
+            'repo_path': get_repo_path(
+                context_data=context_data,
+                root_project=False,
+                user=user),
+            'repo_path_relative': get_repo_path(
+                context_data=context_data,
+                root_project=False,
+                absolute_path=False,
+                user=user),
+            'repo_path_relative_root': get_repo_path(
+                context_data=context_data,
+                root_project=True,
+                absolute_path=False),
+            'repo_path_root': get_repo_path(
+                context_data=context_data,
+                root_project=True),
+            'server_process_id': os.getpid(),
+            'scheduler_process_id': scheduler_manager.get_scheduler_pid(),
             'scheduler_status': scheduler_manager.get_status(),
             'instance_type': instance_type,
             'disable_pipeline_edit_access': is_disable_pipeline_edit_access(),

mage_ai/api/resources/SyncResource.py

@@ -40,6 +40,7 @@ class SyncResource(GenericResource):
     @classmethod
     def collection(self, query, meta, user, **kwargs):
         sync_config = self.get_project_sync_config(user)
+
         return self.build_result_set(
             [sync_config],
             user,
@@ -66,9 +67,9 @@ class SyncResource(GenericResource):
             user_payload = self.update_user_settings(user_settings, user=user, repo_name=repo_name)
             UserGitConfig.load(config=user_payload)
 
-            repo_path = kwargs.get('repo_path') or get_repo_path()
+            repo_path = kwargs.get('repo_path') or get_repo_path(user=user)
             user_preferences = user.preferences or {}
-            user_git_settings = user.git_settings or {}
+            user_git_settings = user.get_git_settings(repo_path) or {}
             user_preferences[repo_path] = {
                 **user_preferences.get(repo_path, {}),
                 'git_settings': {
@@ -146,16 +147,14 @@ class SyncResource(GenericResource):
 
         if ssh_public_key:
             secret_name = get_ssh_public_key_secret_name(user=user)
-            secret = Secret.query.filter(
-                Secret.name == secret_name).one_or_none()
+            secret = Secret.query.filter(Secret.name == secret_name).one_or_none()
             if secret:
                 secret.delete()
             create_secret(secret_name, ssh_public_key, repo_name=repo_name)
             user_payload['ssh_public_key_secret_name'] = secret_name
         if ssh_private_key:
             secret_name = get_ssh_private_key_secret_name(user=user)
-            secret = Secret.query.filter(
-                Secret.name == secret_name).one_or_none()
+            secret = Secret.query.filter(Secret.name == secret_name).one_or_none()
             if secret:
                 secret.delete()
             create_secret(secret_name, ssh_private_key, repo_name=repo_name)
@@ -164,8 +163,7 @@ class SyncResource(GenericResource):
         access_token = user_payload.pop('access_token', None)
         if access_token:
             secret_name = get_access_token_secret_name(user=user)
-            secret = Secret.query.filter(
-                Secret.name == secret_name).one_or_none()
+            secret = Secret.query.filter(Secret.name == secret_name).one_or_none()
             if secret:
                 secret.delete()
             create_secret(secret_name, access_token, repo_name=repo_name)

mage_ai/api/resources/VariableResource.py

@@ -2,7 +2,7 @@ from typing import Dict
 
 from mage_ai.api.errors import ApiError
 from mage_ai.api.resources.GenericResource import GenericResource
-from mage_ai.data_preparation.models.variable import VariableType
+from mage_ai.data_preparation.models.variables.constants import VariableType
 from mage_ai.data_preparation.variable_manager import (
     VariableManager,
     delete_global_variable,
@@ -10,7 +10,7 @@ from mage_ai.data_preparation.variable_manager import (
     set_global_variable,
 )
 from mage_ai.orchestration.db import safe_db_query
-from mage_ai.settings.repo import get_variables_dir
+from mage_ai.settings.repo import get_repo_path, get_variables_dir
 
 
 def get_variable_value(
@@ -41,6 +41,9 @@ class VariableResource(GenericResource):
     @classmethod
     @safe_db_query
     def collection(self, query, meta, user, **kwargs):
+        context_data = kwargs.get('context_data')
+        repo_path = get_repo_path(context_data=context_data, user=user)
+        variables_dir = get_variables_dir(repo_path=repo_path)
         pipeline_uuid = kwargs['parent_model'].uuid
 
         global_only = query.get('global_only', [False])
@@ -49,12 +52,13 @@ class VariableResource(GenericResource):
 
         # Get global variables from project's path
         global_variables = [
-            dict(
-                uuid=uuid,
-                type=str(type(value)),
-                value=value
-            )
-            for uuid, value in get_global_variables(pipeline_uuid).items()
+            dict(uuid=uuid, type=str(type(value)), value=value)
+            for uuid, value in get_global_variables(
+                pipeline_uuid,
+                context_data=context_data,
+                repo_path=repo_path,
+                variables_dir=variables_dir,
+            ).items()
         ]
         global_variables_arr = [
             dict(
@@ -65,7 +69,10 @@ class VariableResource(GenericResource):
         ]
         variables = global_variables_arr
         if not global_only:
-            variable_manager = VariableManager(variables_dir=get_variables_dir())
+            variable_manager = VariableManager(
+                repo_path=repo_path,
+                variables_dir=variables_dir,
+            )
             variables_dict = variable_manager.get_variables_by_pipeline(pipeline_uuid)
             variables = [
                 dict(
@@ -77,12 +84,14 @@ class VariableResource(GenericResource):
                             pipeline_uuid,
                             uuid,
                             var,
-                        ) for var in arr
+                        )
+                        for var in arr
                         # Not return printed outputs
                         if var == 'df' or var.startswith('output')
                     ],
                 )
-                for uuid, arr in variables_dict.items() if uuid != 'global'
+                for uuid, arr in variables_dict.items()
+                if uuid != 'global'
             ] + global_variables_arr
 
         return self.build_result_set(
@@ -100,7 +109,9 @@ class VariableResource(GenericResource):
 
         variable_uuid = payload.get('name')
         if not variable_uuid.isidentifier():
-            error.update(message=f'Invalid variable name syntax for variable name {variable_uuid}.')
+            error.update(
+                message=f'Invalid variable name syntax for variable name {variable_uuid}.'
+            )
             raise ApiError(error)
 
         variable_value = payload.get('value')
@@ -116,13 +127,17 @@ class VariableResource(GenericResource):
 
         global_variables = get_global_variables(pipeline_uuid)
 
-        return self(dict(
-            block=dict(uuid='global'),
-            name=variable_uuid,
-            pipeline=dict(uuid=pipeline_uuid),
-            value=variable_value,
-            variables=list(global_variables.keys()),
-        ), user, **kwargs)
+        return self(
+            dict(
+                block=dict(uuid='global'),
+                name=variable_uuid,
+                pipeline=dict(uuid=pipeline_uuid),
+                value=variable_value,
+                variables=list(global_variables.keys()),
+            ),
+            user,
+            **kwargs,
+        )
 
     @classmethod
     @safe_db_query
@@ -138,6 +153,9 @@ class VariableResource(GenericResource):
         variable_uuid = payload.get('name')
         if not variable_uuid.isidentifier():
             error.update(message=f'Invalid variable name syntax for variable name {self.name}.')
+            error.update(
+                message=f'Invalid variable name syntax for variable name {self.name}.'
+            )
             raise ApiError(error)
 
         variable_value = payload.get('value')
@@ -153,13 +171,15 @@ class VariableResource(GenericResource):
 
         global_variables = get_global_variables(pipeline_uuid)
 
-        self.model.update(dict(
-            block=dict(uuid='global'),
-            name=variable_uuid,
-            pipeline=dict(uuid=pipeline_uuid),
-            value=variable_value,
-            variables=list(global_variables.keys()),
-        ))
+        self.model.update(
+            dict(
+                block=dict(uuid='global'),
+                name=variable_uuid,
+                pipeline=dict(uuid=pipeline_uuid),
+                value=variable_value,
+                variables=list(global_variables.keys()),
+            )
+        )
 
         return self
 

mage_ai/api/resources/VersionControlProjectResource.py

@@ -1,3 +1,4 @@
+import inspect
 import urllib.parse
 from typing import Dict
 
@@ -41,11 +42,14 @@ class VersionControlProjectResource(VersionControlErrors, AsyncBaseResource):
         model = self.get_model(pk)
 
         sync_resource = SyncResource.member(None, user, repo_path=model.repo_path)
-        model.sync_config = await SyncResource.presenter_class()(
+        result = SyncResource.presenter_class()(
             sync_resource,
             user,
             **kwargs,
         ).present()
+        if result and inspect.isawaitable(result):
+            result = await result
+        model.sync_config = result
 
         res = self(model, user, **kwargs)
         res.validate_output()
@@ -71,11 +75,14 @@ class VersionControlProjectResource(VersionControlErrors, AsyncBaseResource):
                 self.current_user,
                 repo_path=self.model.repo_path,
             )
-            self.model.sync_config = await SyncResource.presenter_class()(
+            result = SyncResource.presenter_class()(
                 sync_resource,
                 self.current_user,
                 **kwargs,
             ).present()
+            if result and inspect.isawaitable(result):
+                result = await result
+            self.model.sync_config = result
 
         self.model.update_attributes()
         self.validate_output()

mage_ai/api/resources/WidgetResource.py

@@ -39,7 +39,7 @@ class WidgetResource(GenericResource):
         resource = Widget.create(
             payload.get('name') or payload.get('uuid'),
             payload.get('type'),
-            get_repo_path(),
+            get_repo_path(user=user),
             config=payload.get('config'),
             language=payload.get('language'),
             pipeline=pipeline,

mage_ai/api/resources/WorkspaceResource.py

@@ -163,7 +163,7 @@ class WorkspaceResource(GenericResource):
             raise ApiError(error)
 
         if project_type == ProjectType.MAIN and subproject:
-            repo_path = get_repo_path()
+            repo_path = get_repo_path(user=self.current_user)
             projects_folder = os.path.join(repo_path, 'projects')
             projects = [
                 f.name.split('.')[0]

mage_ai/api/views.py

@@ -34,8 +34,7 @@ async def execute_operation(
     if request.error:
         return __render_error(handler, request.error, **tags)
 
-    action, options = __determine_action(
-        request, child=child, child_pk=child_pk, pk=pk)
+    action, options = __determine_action(request, child=child, child_pk=child_pk, pk=pk)
     try:
         base_operation = BaseOperation(
             action=action,
@@ -68,13 +67,15 @@ async def execute_operation(
     if error_response:
         return __render_error(handler, error_response, **tags)
 
-    info('Action: {} {} {} {} {}'.format(
-        action,
-        child or resource,
-        child_pk or pk,
-        resource if child else '',
-        pk if child else '',
-    ))
+    info(
+        'Action: {} {} {} {} {}'.format(
+            action,
+            child or resource,
+            child_pk or pk,
+            resource if child else '',
+            pk if child else '',
+        )
+    )
     api_time = end_time.timestamp() - start_time.timestamp()
     info(f'Latency: {api_time:.4f} seconds')
     tags = dict(
@@ -87,11 +88,13 @@ async def execute_operation(
     timing('api.time', api_time, tags)
     timing('sql.time', api_time, tags)
 
-    handler.write(simplejson.dumps(
-        response,
-        default=encode_complex,
-        ignore_nan=True,
-    ))
+    handler.write(
+        simplejson.dumps(
+            response,
+            default=encode_complex,
+            ignore_nan=True,
+        )
+    )
 
 
 def __determine_action(
@@ -148,19 +151,15 @@ def __meta(request) -> Dict:
 
 
 def __meta_keys(request) -> List[str]:
-    return list(
-        filter(
-            lambda x: x[0] == '_', [
-                k for k in request.query_arguments.keys()]))
+    return list(filter(lambda x: x[0] == '_', [k for k in request.query_arguments.keys()]))
 
 
 def __payload(request) -> Dict:
-    if 'Content-Type' in request.headers and \
-       'multipart/form-data' in request.headers.get('Content-Type'):
-
+    if 'Content-Type' in request.headers and 'multipart/form-data' in request.headers.get(
+        'Content-Type'
+    ):
         parts = request.body.decode('utf-8', 'ignore').split('\r\n')
-        idx = parts.index(
-            'Content-Disposition: form-data; name="json_root_body"')
+        idx = parts.index('Content-Disposition: form-data; name="json_root_body"')
         json_root_body = parts[idx + 2]
 
         return json.loads(json_root_body)
@@ -193,15 +192,18 @@ def __query(request) -> Dict:
         else:
             obj[key] = value
         return obj
+
     return reduce(_build, request.query_arguments.keys() - meta_keys, {})
 
 
 def __render_error(handler, error: Dict, **kwargs):
     __log_error(handler.request, error, **kwargs)
-    handler.write(dict(
-        error=error,
-        status=200,
-    ))
+    handler.write(
+        dict(
+            error=error,
+            status=200,
+        )
+    )
 
 
 def __log_error(
@@ -225,16 +227,21 @@ def __log_error(
             if child_pk:
                 endpoint += '/{}'.format(child_pk)
 
-    error('[{}] [ERROR] {} /{} [user: {}]: {}'.format(
-        datetime.utcnow(),
-        request.method,
-        endpoint,
-        user_id,
-        err,
-    ))
-
-    increment('api.error', tags=dict(
-        endpoint=endpoint,
-        error=err['type'] if isinstance(err, dict) else type(err).__name__,
-        resource=resource,
-    ))
+    error(
+        '[{}] [ERROR] {} /{} [user: {}]: {}'.format(
+            datetime.utcnow(),
+            request.method,
+            endpoint,
+            user_id,
+            err,
+        )
+    )
+
+    increment(
+        'api.error',
+        tags=dict(
+            endpoint=endpoint,
+            error=err['type'] if isinstance(err, dict) else type(err).__name__,
+            resource=resource,
+        ),
+    )