mage-ai 0.9.45__py3-none-any.whl → 0.9.46__py3-none-any.whl

mage_ai/api/resources/ComputeConnectionResource.py

@@ -0,0 +1,103 @@
+from mage_ai.api.errors import ApiError
+from mage_ai.api.resources.GenericResource import GenericResource
+from mage_ai.services.compute.constants import (
+    ComputeConnectionActionUUID,
+    ComputeConnectionState,
+)
+from mage_ai.services.compute.models import ComputeConnection, ComputeService
+from mage_ai.services.spark.constants import ComputeServiceUUID
+from mage_ai.services.ssh.aws.emr.models import SSHTunnel, create_tunnel
+from mage_ai.services.ssh.aws.emr.utils import should_tunnel, tunnel
+from mage_ai.shared.array import find
+
+
+class ComputeConnectionResource(GenericResource):
+    @classmethod
+    async def collection(self, query_arg, _meta, user, **kwargs):
+        parent_model = kwargs.get('parent_model')
+
+        arr = []
+
+        if parent_model and isinstance(parent_model, ComputeService):
+            if ComputeServiceUUID.AWS_EMR == parent_model.uuid:
+                arr.extend(parent_model.compute_connections())
+
+        return self.build_result_set(
+            arr,
+            user,
+            **kwargs,
+        )
+
+    @classmethod
+    async def member(self, pk, user, **kwargs):
+        parent_model = kwargs.get('parent_model')
+
+        model = ComputeConnection.load(name=pk, uuid=pk)
+        if parent_model and isinstance(parent_model, ComputeService):
+            if ComputeServiceUUID.AWS_EMR == parent_model.uuid:
+                model = find(lambda x: x.uuid == pk, parent_model.compute_connections())
+
+        return self(model, user, **kwargs)
+
+    async def update(self, payload, **kwargs):
+        parent_model = kwargs.get('parent_model')
+
+        action_uuid = payload.get('action')
+
+        if not action_uuid:
+            return
+
+        if parent_model and isinstance(parent_model, ComputeService):
+            if ComputeServiceUUID.AWS_EMR == parent_model.uuid:
+                from mage_ai.services.compute.aws.steps import SetupStepUUID
+
+                if SetupStepUUID.OBSERVABILITY == self.model.uuid:
+                    action_valid = action_uuid in [
+                        ComputeConnectionActionUUID.CREATE,
+                        ComputeConnectionActionUUID.DELETE,
+                        ComputeConnectionActionUUID.DESELECT,
+                        ComputeConnectionActionUUID.UPDATE,
+                    ]
+
+                    if action_valid:
+                        if ComputeConnectionActionUUID.CREATE == action_uuid:
+                            try:
+                                should_tunnel(
+                                    ignore_active_kernel=True,
+                                    raise_error=True,
+                                )
+
+                                ssh_tunnel = create_tunnel(
+                                    connect=False,
+                                    fresh=True,
+                                    reconnect=False,
+                                    stop=False,
+                                )
+                                if ssh_tunnel:
+                                    ssh_tunnel.precheck_access(raise_error=True)
+                                    ssh_tunnel.connect()
+                                else:
+                                    raise Exception('Cannot create SSH tunnel instance.')
+                            except Exception as err:
+                                error = ApiError.UNAUTHORIZED_ACCESS.copy()
+                                error.update(message=str(err))
+                                raise ApiError(error)
+
+                        def _callback(
+                            action_uuid=action_uuid,
+                            *args,
+                            **kwargs,
+                        ):
+                            ssh_tunnel = SSHTunnel()
+
+                            if ComputeConnectionActionUUID.CREATE == action_uuid:
+                                tunnel(ignore_active_kernel=True)
+                            elif ComputeConnectionActionUUID.DELETE == action_uuid:
+                                ssh_tunnel.close()
+                            elif ComputeConnectionActionUUID.DESELECT == action_uuid:
+                                ssh_tunnel.stop()
+                            elif ComputeConnectionActionUUID.UPDATE == action_uuid:
+                                ssh_tunnel.reconnect()
+
+                        self.model.state = ComputeConnectionState.PENDING
+                        self.on_update_callback = _callback

mage_ai/api/resources/ComputeServiceResource.py

@@ -0,0 +1,35 @@
+from mage_ai.api.resources.GenericResource import GenericResource
+from mage_ai.data_preparation.models.project import Project
+from mage_ai.services.compute.models import ComputeService
+
+
+class ComputeServiceResource(GenericResource):
+    @classmethod
+    async def collection(self, _query, _meta, user, **kwargs):
+        resource = await self.member(None, user, **kwargs)
+
+        return self.build_result_set(
+            [
+                resource.model,
+            ],
+            user,
+            **kwargs,
+        )
+
+    @classmethod
+    def get_model(self, _pk):
+        return ComputeService.build(Project())
+
+    @classmethod
+    async def member(self, pk, user, **kwargs):
+        query_arg = kwargs.get('query', {})
+
+        with_clusters = query_arg.get('with_clusters', [False])
+        if with_clusters:
+            with_clusters = with_clusters[0]
+
+        compute_service = self.get_model(pk)
+        if with_clusters:
+            compute_service.with_clusters = with_clusters
+
+        return self(compute_service, user, **kwargs)

mage_ai/api/resources/DownloadResource.py

@@ -0,0 +1,56 @@
+import datetime
+from typing import Dict
+
+import jwt
+
+from mage_ai.api.constants import DOWNLOAD_TOKEN_LIFESPAN
+from mage_ai.api.resources.GenericResource import GenericResource
+from mage_ai.data_preparation.models.download import Download
+from mage_ai.data_preparation.models.file import File
+from mage_ai.data_preparation.models.pipeline import Pipeline
+from mage_ai.orchestration.db import safe_db_query
+from mage_ai.settings import JWT_DOWNLOAD_SECRET
+
+
+class DownloadResource(GenericResource):
+
+    @classmethod
+    @safe_db_query
+    async def create(self, payload: Dict, user, **kwargs) -> 'DownloadResource':
+        parent_model = kwargs.get('parent_model')
+
+        ignore_folder_structure = payload.get('ignore_folder_structure', False)
+
+        file_list = []
+
+        if isinstance(parent_model, Pipeline):
+            file_name, file_list = self.fetch_pipeline_files(parent_model)
+        elif isinstance(parent_model, File):
+            if not parent_model.exists():
+                raise Exception(f'File {parent_model.filename} does not exist.')
+            file_name = parent_model.filename
+            file_list.append(parent_model.file_path)
+
+        token = self.generate_download_token(file_name, file_list, ignore_folder_structure)
+        download = Download(token)
+
+        return self(download, user)
+
+    def fetch_pipeline_files(pipeline: Pipeline):
+        config_path = pipeline.config_path
+
+        zip_name = f'{pipeline.uuid}.zip'
+        file_list = [config_path] + [block.file_path for block in pipeline.blocks_by_uuid.values()]
+
+        return [zip_name, file_list]
+
+    def generate_download_token(file_name, file_list, ignore_folder_structure):
+        now = datetime.datetime.utcnow()
+        payload = {
+            'file_name': file_name,
+            'file_list': file_list,
+            'ignore_folder_structure': ignore_folder_structure,
+            'iat': now,
+            'exp': now + datetime.timedelta(seconds=DOWNLOAD_TOKEN_LIFESPAN)
+        }
+        return jwt.encode(payload, JWT_DOWNLOAD_SECRET, algorithm="HS256")

mage_ai/api/resources/ExecutionStateResource.py

@@ -18,7 +18,7 @@ class ExecutionStateResource(GenericResource):
 
         if pipeline_uuid and block_uuid:
             pipeline = await Pipeline.get_async(pipeline_uuid)
-            if pipeline and PipelineType.PYTHON == pipeline.type:
+            if pipeline and pipeline.type in [PipelineType.PYTHON, PipelineType.PYSPARK]:
                 block = pipeline.get_block(block_uuid)
                 if block and block.compute_management_enabled() and block.is_using_spark():
                     arr.append(dict(

mage_ai/api/resources/GlobalHookResource.py

@@ -0,0 +1,192 @@
+import asyncio
+from typing import Dict, List
+
+from mage_ai.api.errors import ApiError
+from mage_ai.api.resources.AsyncBaseResource import AsyncBaseResource
+from mage_ai.api.resources.PipelineResource import PipelineResource
+from mage_ai.authentication.permissions.constants import EntityName
+from mage_ai.data_preparation.models.global_hooks.constants import (
+    DISABLED_RESOURCE_TYPES,
+)
+from mage_ai.data_preparation.models.global_hooks.models import (
+    GlobalHooks,
+    Hook,
+    HookOperation,
+)
+from mage_ai.data_preparation.models.pipeline import Pipeline
+from mage_ai.orchestration.db.models.oauth import User
+from mage_ai.shared.array import find
+from mage_ai.shared.hash import ignore_keys
+
+
+class GlobalHookResource(AsyncBaseResource):
+    @classmethod
+    async def collection(self, query: Dict, _meta: Dict, user: User, **kwargs):
+        operation_types = query.get('operation_type[]', [])
+        if operation_types:
+            operation_types = operation_types[0]
+        if isinstance(operation_types, str):
+            operation_types = [HookOperation(m) for m in operation_types.split(',') if m]
+
+        resource_types = query.get('resource_type[]', [])
+        if resource_types:
+            resource_types = resource_types[0]
+        if isinstance(resource_types, str):
+            resource_types = [EntityName(m) for m in resource_types.split(',') if m]
+
+        global_hooks = GlobalHooks.load_from_file()
+
+        return self.build_result_set(
+            global_hooks.hooks(
+                operation_types=operation_types,
+                resource_types=resource_types,
+            ),
+            user,
+            **kwargs,
+        )
+
+    @classmethod
+    async def create(self, payload: Dict, user: User, **kwargs) -> 'GlobalHookResource':
+        if user and user.id:
+            payload['metadata'] = dict(
+                user=dict(
+                    id=user.id,
+                ),
+            )
+
+        hook = Hook.load(**payload)
+
+        if not hook.uuid or not hook.resource_type or not hook.operation_type:
+            missing = []
+            if not hook.resource_type:
+                missing.append('resource_type')
+            if not hook.operation_type:
+                missing.append('operation_type')
+            if not hook.uuid:
+                missing.append('uuid')
+
+            error = ApiError.RESOURCE_INVALID.copy()
+            error.update(
+                message=f'Hook is missing the following required attributes: {", ".join(missing)}.',
+            )
+            raise ApiError(error)
+
+        if hook.resource_type and hook.resource_type in DISABLED_RESOURCE_TYPES:
+            error = ApiError.RESOURCE_INVALID.copy()
+            error.update(
+                message=f'Hooks cannot be created for resource type {hook.resource_type.value}.',
+            )
+            raise ApiError(error)
+
+        global_hooks = GlobalHooks.load_from_file()
+        if global_hooks.get_hook(
+            operation_type=hook.operation_type,
+            resource_type=hook.resource_type,
+            uuid=hook.uuid,
+        ):
+            error = ApiError.RESOURCE_INVALID.copy()
+            error.update(
+                message=f'Hook {hook.uuid} already exists for resource '
+                        f'{hook.resource_type} and operation {hook.operation_type}.',
+            )
+            raise ApiError(error)
+
+        global_hooks.add_hook(hook, snapshot=True)
+        global_hooks.save()
+
+        return self(hook, user, **kwargs)
+
+    @classmethod
+    async def member(self, pk: str, user: User, **kwargs):
+        query = kwargs.get('query') or {}
+
+        resource_type = query.get('resource_type', [None])
+        if resource_type:
+            resource_type = resource_type[0]
+
+        operation_type = query.get('operation_type', [None])
+        if operation_type:
+            operation_type = operation_type[0]
+
+        include_operation_types = query.get('include_operation_types', [None])
+        if include_operation_types:
+            include_operation_types = include_operation_types[0]
+
+        include_resource_types = query.get('include_resource_types', [None])
+        if include_resource_types:
+            include_resource_types = include_resource_types[0]
+
+        global_hooks = GlobalHooks.load_from_file()
+        hook = global_hooks.get_hook(
+            operation_type=HookOperation(operation_type) if operation_type else operation_type,
+            resource_type=EntityName(resource_type) if resource_type else resource_type,
+            uuid=pk,
+        )
+
+        if not hook and not include_operation_types and not include_resource_types:
+            raise ApiError(ApiError.RESOURCE_NOT_FOUND)
+
+        return self(hook, user, **kwargs)
+
+    async def update(self, payload: Dict, **kwargs):
+        global_hooks = GlobalHooks.load_from_file()
+        self.model = global_hooks.add_hook(
+            self.model,
+            payload=ignore_keys(payload, ['snapshot']),
+            snapshot=payload.get('snapshot') or False,
+            update=True,
+        )
+        global_hooks.save()
+
+    async def delete(self, **kwargs):
+        global_hooks = GlobalHooks.load_from_file()
+        global_hooks.remove_hook(self.model)
+        global_hooks.save()
+
+
+async def __load_pipelines(resource: GlobalHookResource):
+    pipeline_uuids = []
+    for res in resource.result_set():
+        if res.model.pipeline_settings and res.model.pipeline_settings.get('uuid'):
+            pipeline_uuid = res.model.pipeline_settings.get('uuid')
+            if pipeline_uuid:
+                pipeline_uuids.append(pipeline_uuid)
+
+    async def get_pipeline(uuid):
+        try:
+            return await Pipeline.get_async(uuid)
+        except Exception as err:
+            err_message = f'Error loading pipeline {uuid}: {err}.'
+            if err.__class__.__name__ == 'OSError' and 'Too many open files' in err.strerror:
+                raise Exception(err_message)
+            else:
+                print(err_message)
+                return None
+
+    pipelines = await asyncio.gather(
+        *[get_pipeline(uuid) for uuid in pipeline_uuids]
+    )
+    pipelines = [p for p in pipelines if p is not None]
+
+    return PipelineResource.build_result_set(pipelines, resource.current_user)
+
+
+async def __find_pipeline(resource: GlobalHookResource, arr: List[PipelineResource]):
+    pipeline_uuid = resource.model.pipeline_settings and resource.model.pipeline_settings.get(
+        'uuid',
+    )
+
+    if not pipeline_uuid:
+        return
+
+    return find(
+        lambda x, pipeline_uuid=pipeline_uuid: x.uuid == pipeline_uuid,
+        arr,
+    )
+
+
+GlobalHookResource.register_collective_loader(
+    'pipeline_details',
+    load=__load_pipelines,
+    select=__find_pipeline,
+)

mage_ai/api/resources/KernelResource.py

@@ -7,6 +7,7 @@ from mage_ai.server.active_kernel import (
     switch_active_kernel,
 )
 from mage_ai.server.kernels import DEFAULT_KERNEL_NAME, KernelName, kernel_managers
+from mage_ai.services.ssh.aws.emr.utils import tunnel
 
 
 class KernelResource(GenericResource):
@@ -56,4 +57,13 @@ class KernelResource(GenericResource):
                 if 'start_kernel' in str(e):
                     start_kernel()
 
+        def _callback(*args, **kwargs):
+            tunnel(
+                kernel_name=self.model.kernel_name,
+                reconnect=True,
+                validate_conditions=True,
+            )
+
+        self.on_update_callback = _callback
+
         return self

mage_ai/api/resources/OauthResource.py

@@ -1,4 +1,3 @@
-import json
 import urllib.parse
 from datetime import datetime, timedelta
 from urllib.parse import parse_qs, urlparse
@@ -7,10 +6,8 @@ from mage_ai.api.errors import ApiError
 from mage_ai.api.resources.GenericResource import GenericResource
 from mage_ai.authentication.oauth2 import generate_access_token
 from mage_ai.authentication.oauth.constants import (
-    ACTIVE_DIRECTORY_CLIENT_ID,
     GITHUB_CLIENT_ID,
     GITHUB_STATE,
-    OAUTH_PROVIDER_ACTIVE_DIRECTORY,
     OAUTH_PROVIDER_GHE,
     OAUTH_PROVIDER_GITHUB,
     VALID_OAUTH_PROVIDERS,
@@ -24,10 +21,31 @@ from mage_ai.authentication.providers.constants import NAME_TO_PROVIDER
 from mage_ai.data_preparation.git.api import get_oauth_client_id
 from mage_ai.orchestration.db import safe_db_query
 from mage_ai.orchestration.db.models.oauth import Oauth2AccessToken, Oauth2Application
-from mage_ai.settings import ACTIVE_DIRECTORY_DIRECTORY_ID
 
 
 class OauthResource(GenericResource):
+    @classmethod
+    @safe_db_query
+    async def collection(self, query, meta, user, **kwargs):
+        redirect_uri = query.get('redirect_uri', [None])
+        if redirect_uri:
+            redirect_uri = redirect_uri[0]
+
+        oauths = []
+        for provider, provider_class in NAME_TO_PROVIDER.items():
+            try:
+                provider_instance = provider_class()
+                auth_url_response = provider_instance.get_auth_url_response(
+                    redirect_uri=redirect_uri
+                )
+                if auth_url_response:
+                    auth_url_response['provider'] = provider
+                    oauths.append(auth_url_response)
+            except Exception:
+                continue
+
+        return self.build_result_set(oauths, user, **kwargs)
+
     @classmethod
     @safe_db_query
     def create(self, payload, user, **kwargs):
@@ -171,36 +189,13 @@ class OauthResource(GenericResource):
                 for k, v in query.items():
                     query_strings.append(f'{k}={v}')
 
-                model['url'] = f"https://github.com/login/oauth/authorize?{'&'.join(query_strings)}"
-            elif OAUTH_PROVIDER_ACTIVE_DIRECTORY == pk:
-                ad_directory_id = ACTIVE_DIRECTORY_DIRECTORY_ID
-                if ad_directory_id:
-                    from requests.models import PreparedRequest
-
-                    req = PreparedRequest()
-                    req.prepare_url(redirect_uri, dict(provider=pk))
-                    query = dict(
-                        client_id=ACTIVE_DIRECTORY_CLIENT_ID,
-                        redirect_uri=f'https://api.mage.ai/v1/oauth/{pk}',
-                        response_type='code',
-                        scope='User.Read',
-                        state=urllib.parse.quote_plus(
-                            json.dumps(
-                                dict(
-                                    redirect_uri=req.url,
-                                    tenant_id=ad_directory_id,
-                                )
-                            )
-                        ),
-                    )
-                    query_strings = []
-                    for k, v in query.items():
-                        query_strings.append(f'{k}={v}')
-                    model[
-                        'url'
-                    ] = f"https://login.microsoftonline.com/{ad_directory_id}/oauth2/v2.0/authorize?{'&'.join(query_strings)}"  # noqa: E501
+                model[
+                    'url'
+                ] = f"https://github.com/login/oauth/authorize?{'&'.join(query_strings)}"
             elif provider_instance is not None:
-                resp = provider_instance.get_auth_url_response(redirect_uri=redirect_uri)
+                resp = provider_instance.get_auth_url_response(
+                    redirect_uri=redirect_uri
+                )
                 if resp:
                     model.update(resp)
 

mage_ai/api/resources/PipelineResource.py

@@ -67,15 +67,15 @@ class PipelineResource(BaseResource):
             tags = new_tags
 
         pipeline_types = query.get('type[]', [])
-        if pipeline_types:
+        if pipeline_types and len(pipeline_types) == 1:
             pipeline_types = pipeline_types[0]
-        if pipeline_types:
+        if pipeline_types and isinstance(pipeline_types, str):
             pipeline_types = pipeline_types.split(',')
 
         pipeline_statuses = query.get('status[]', [])
-        if pipeline_statuses:
+        if pipeline_statuses and len(pipeline_statuses) == 1:
             pipeline_statuses = pipeline_statuses[0]
-        if pipeline_statuses:
+        if pipeline_statuses and isinstance(pipeline_statuses, str):
             pipeline_statuses = pipeline_statuses.split(',')
 
         from_history_days = query.get('from_history_days', [None])

mage_ai/api/resources/PipelineScheduleResource.py

@@ -3,6 +3,7 @@ import uuid
 from sqlalchemy.orm import selectinload
 
 from mage_ai.api.resources.DatabaseResource import DatabaseResource
+from mage_ai.data_preparation.models.pipeline import Pipeline
 from mage_ai.data_preparation.models.triggers import (
     Trigger,
     add_or_update_trigger_for_pipeline_and_persist,
@@ -146,6 +147,20 @@ class PipelineScheduleResource(DatabaseResource):
         if 'token' not in payload:
             payload['token'] = uuid.uuid4().hex
 
+        if pipeline.should_save_trigger_in_code_automatically():
+            def _callback(resource, kwargs=kwargs, user=user):
+                from mage_ai.api.resources.PipelineTriggerResource import (
+                    PipelineTriggerResource,
+                )
+
+                PipelineTriggerResource.create(
+                    dict(pipeline_schedule_id=resource.id),
+                    user,
+                    **kwargs,
+                )
+
+            self.on_create_callback = _callback
+
         return super().create(payload, user, **kwargs)
 
     @safe_db_query
@@ -257,22 +272,28 @@ class PipelineScheduleResource(DatabaseResource):
 
         resource = super().update(payload)
         updated_model = resource.model
-        trigger = Trigger(
-            name=updated_model.name,
-            pipeline_uuid=updated_model.pipeline_uuid,
-            schedule_interval=updated_model.schedule_interval,
-            schedule_type=updated_model.schedule_type,
-            settings=updated_model.settings,
-            sla=updated_model.sla,
-            start_time=updated_model.start_time,
-            status=updated_model.status,
-            variables=updated_model.variables,
-        )
-        add_or_update_trigger_for_pipeline_and_persist(
-            trigger,
-            updated_model.pipeline_uuid,
-            update_only_if_exists=True,
-        )
+
+        pipeline = Pipeline.get(updated_model.pipeline_uuid)
+        if pipeline:
+            trigger = Trigger(
+                name=updated_model.name,
+                pipeline_uuid=updated_model.pipeline_uuid,
+                schedule_interval=updated_model.schedule_interval,
+                schedule_type=updated_model.schedule_type,
+                settings=updated_model.settings,
+                sla=updated_model.sla,
+                start_time=updated_model.start_time,
+                status=updated_model.status,
+                variables=updated_model.variables,
+            )
+
+            update_only_if_exists = not pipeline.should_save_trigger_in_code_automatically()
+
+            add_or_update_trigger_for_pipeline_and_persist(
+                trigger,
+                pipeline.uuid,
+                update_only_if_exists=update_only_if_exists,
+            )
 
         return self
 

mage_ai/api/resources/ProjectResource.py

@@ -86,6 +86,9 @@ class ProjectResource(GenericResource):
         if 'emr_config' in payload:
             data['emr_config'] = payload['emr_config']
 
+        if 'pipelines' in payload:
+            data['pipelines'] = payload['pipelines']
+
         if 'spark_config' in payload:
             data['spark_config'] = payload['spark_config']
 
@@ -100,9 +103,8 @@ class ProjectResource(GenericResource):
         if should_log_project:
             await UsageStatisticLogger().project_impression()
 
-        if Project(repo_config=repo_config).is_feature_enabled(
-            FeatureUUID.DATA_INTEGRATION_IN_BATCH_PIPELINE,
-        ):
+        project = Project(repo_config=repo_config)
+        if project.is_feature_enabled(FeatureUUID.DATA_INTEGRATION_IN_BATCH_PIPELINE):
             await BlockActionObjectCache.initialize_cache(replace=True)
 
         return self

mage_ai/api/resources/SessionResource.py

@@ -4,8 +4,6 @@ from mage_ai.api.errors import ApiError
 from mage_ai.api.resources.BaseResource import BaseResource
 from mage_ai.authentication.ldap import new_ldap_connection
 from mage_ai.authentication.oauth2 import encode_token, generate_access_token
-from mage_ai.authentication.oauth.active_directory import get_user_info
-from mage_ai.authentication.oauth.constants import OAUTH_PROVIDER_ACTIVE_DIRECTORY
 from mage_ai.authentication.passwords import verify_password
 from mage_ai.authentication.providers.constants import NAME_TO_PROVIDER
 from mage_ai.orchestration.db import safe_db_query
@@ -30,15 +28,8 @@ class SessionResource(BaseResource):
         if token and provider:
             roles = []
             provider_class = NAME_TO_PROVIDER.get(provider)
-            provider_instance = None
             if provider_class is not None:
                 provider_instance = provider_class()
-
-            if provider == OAUTH_PROVIDER_ACTIVE_DIRECTORY:
-                user_info = get_user_info(token)
-                username = user_info.get('userPrincipalName')
-                email = user_info.get('userPrincipalName')
-            elif provider_instance is not None:
                 user_info = await provider_instance.get_user_info(access_token=token)
                 email = user_info.get('email')
                 username = user_info.get('username')