mage-ai 0.9.55__py3-none-any.whl → 0.9.57__py3-none-any.whl

mage_ai/api/parsers/ProjectParser.py

@@ -0,0 +1,29 @@
+from typing import Dict
+
+from mage_ai.api.constants import AuthorizeStatusType
+from mage_ai.api.oauth_scope import OauthScopeType
+from mage_ai.api.operations.constants import OperationType
+from mage_ai.api.parsers.BaseParser import BaseParser
+from mage_ai.shared.hash import extract
+
+
+class ProjectParser(BaseParser):
+    pass
+
+
+async def parse_read(parser, value: Dict, **kwargs) -> Dict:
+    return extract(value, ['features'])
+
+
+ProjectParser.parse_read(
+    parser=parse_read,
+    on_action=[
+        OperationType.LIST,
+    ],
+    on_authorize_status=[
+        AuthorizeStatusType.FAILED,
+    ],
+    scopes=[
+        OauthScopeType.CLIENT_PUBLIC,
+    ],
+)

mage_ai/api/policies/BlockPolicy.py

@@ -166,6 +166,7 @@ BlockPolicy.allow_query([
 
 BlockPolicy.allow_query([
     'extension_uuid',
+    'file_path',
     'force',
 ], scopes=[
     OauthScope.CLIENT_PRIVATE,

mage_ai/api/policies/CustomDesignPolicy.py

@@ -15,6 +15,7 @@ CustomDesignPolicy.allow_actions(
     ],
     scopes=[
         OauthScope.CLIENT_PRIVATE,
+        OauthScope.CLIENT_PUBLIC,
     ],
     condition=lambda policy: policy.has_at_least_viewer_role(),
     override_permission_condition=lambda _policy: True,
@@ -25,6 +26,7 @@ CustomDesignPolicy.allow_read(
     CustomDesignPresenter.default_attributes + [],
     scopes=[
         OauthScope.CLIENT_PRIVATE,
+        OauthScope.CLIENT_PUBLIC,
     ],
     on_action=[
         OperationType.DETAIL,
@@ -50,6 +52,7 @@ CustomDesignPolicy.allow_query(
     ],
     scopes=[
         OauthScope.CLIENT_PRIVATE,
+        OauthScope.CLIENT_PUBLIC,
     ],
     on_action=[
         OperationType.DETAIL,

mage_ai/api/policies/GitBranchPolicy.py

@@ -41,13 +41,22 @@ GitBranchPolicy.allow_write(GitBranchPresenter.default_attributes, scopes=[
 ], condition=lambda policy: policy.has_at_least_editor_role())
 
 GitBranchPolicy.allow_write(GitBranchPresenter.default_attributes + [
+    'add',
+    'add_remote',
+    'add_remote',
+    'checkout',
+    'clone',
+    'commit',
     'delete',
+    'fetch',
     'merge',
     'pull',
     'push',
     'rebase',
     'remote',
+    'remove_remote',
     'reset',
+    'status',
 ], scopes=[
     OauthScope.CLIENT_PRIVATE
 ], on_action=[

mage_ai/api/policies/GitCustomBranchPolicy.py

@@ -42,6 +42,7 @@ GitCustomBranchPolicy.allow_write(GitBranchPresenter.default_attributes, scopes=
 ], condition=lambda policy: policy.has_at_least_editor_role())
 
 GitCustomBranchPolicy.allow_write(GitBranchPresenter.default_attributes + [
+    'clone',
     'delete',
     'fetch',
     'merge',

mage_ai/api/policies/GlobalHookPolicy.py

@@ -10,6 +10,7 @@ WRITEABLE_ATTRIBUTES = [
     'pipeline',
     'predicate',
     'resource_type',
+    'root_project',
     'run_settings',
     'stages',
     'strategies',
@@ -92,6 +93,7 @@ GlobalHookPolicy.allow_query(
         'include_snapshot_validation',
         'operation_type[]',
         'resource_type[]',
+        'root_project',
     ],
     scopes=[
         OauthScope.CLIENT_PRIVATE,
@@ -111,6 +113,7 @@ GlobalHookPolicy.allow_query(
         'include_snapshot_validation',
         'operation_type',
         'resource_type',
+        'root_project',
     ],
     scopes=[
         OauthScope.CLIENT_PRIVATE,

mage_ai/api/policies/ProjectPolicy.py

@@ -12,8 +12,15 @@ ProjectPolicy.allow_actions([
     constants.LIST,
 ], scopes=[
     OauthScope.CLIENT_PRIVATE,
+    OauthScope.CLIENT_PUBLIC,
 ], condition=lambda policy: policy.has_at_least_viewer_role())
 
+ProjectPolicy.allow_actions([
+    constants.CREATE,
+], scopes=[
+    OauthScope.CLIENT_PRIVATE,
+], condition=lambda policy: policy.has_at_least_editor_role())
+
 ProjectPolicy.allow_actions([
     constants.UPDATE,
 ], scopes=[
@@ -24,10 +31,29 @@ ProjectPolicy.allow_read(ProjectPresenter.default_attributes + [
 ], scopes=[
     OauthScope.CLIENT_PRIVATE,
 ], on_action=[
+    constants.CREATE,
     constants.LIST,
     constants.UPDATE,
 ], condition=lambda policy: policy.has_at_least_viewer_role())
 
+ProjectPolicy.allow_read([
+    'features',
+], scopes=[
+    OauthScope.CLIENT_PUBLIC,
+], on_action=[
+    constants.LIST,
+])
+
+ProjectPolicy.allow_write([
+    'repo_path',
+    'uuid',
+    'type',
+], scopes=[
+    OauthScope.CLIENT_PRIVATE,
+], on_action=[
+    constants.CREATE,
+], condition=lambda policy: policy.has_at_least_editor_role())
+
 ProjectPolicy.allow_write([
     'help_improve_mage',
 ], scopes=[
@@ -42,7 +68,9 @@ ProjectPolicy.allow_write([
     'features',
     'openai_api_key',
     'pipelines',
+    'platform_settings',
     'remote_variables_dir',
+    'root_project',
     'spark_config',
 ], scopes=[
     OauthScope.CLIENT_PRIVATE,

mage_ai/api/presenters/ProjectPresenter.py

@@ -5,11 +5,14 @@ class ProjectPresenter(BasePresenter):
     default_attributes = [
         'emr_config',
         'features',
+        'features_defined',
+        'features_override',
         'help_improve_mage',
         'latest_version',
         'name',
         'openai_api_key',
         'pipelines',
+        'platform_settings',
         'project_uuid',
         'projects',
         'remote_variables_dir',
@@ -18,4 +21,5 @@ class ProjectPresenter(BasePresenter):
         'settings',
         'spark_config',
         'version',
+        'workspace_config_defaults',
     ]

mage_ai/api/resources/BlockResource.py

@@ -1,4 +1,5 @@
 import math
+import os
 import urllib.parse
 from typing import Dict
 
@@ -837,6 +838,7 @@ class BlockResource(GenericResource):
 
         pipeline = kwargs.get('parent_model')
         if pipeline:
+            pk = urllib.parse.unquote(pk)
             block = pipeline.get_block(pk, block_type=block_type, extension_uuid=extension_uuid)
             if block:
                 return self(block, user, **kwargs)
@@ -849,6 +851,17 @@ class BlockResource(GenericResource):
                 error.update(message=message)
                 raise ApiError(error)
 
+        file_path = query.get('file_path', [None])
+        if file_path:
+            file_path = file_path[0]
+        if file_path:
+            block = Block.get_block_from_file_path(urllib.parse.unquote(file_path))
+            if block:
+                return self(block, user, **kwargs)
+            else:
+                error.update(ApiError.RESOURCE_NOT_FOUND)
+                raise ApiError(error)
+
         block_type_and_uuid = urllib.parse.unquote(pk)
         parts = block_type_and_uuid.split('/')
 
@@ -857,7 +870,7 @@ class BlockResource(GenericResource):
             raise ApiError(error)
 
         block_type = parts[0]
-        block_uuid_with_extension = '/'.join(parts[1:])
+        block_uuid_with_extension = os.path.sep.join(parts[1:])
         parts2 = block_uuid_with_extension.split('.')
 
         language = None

mage_ai/api/resources/FileResource.py

@@ -107,7 +107,7 @@ class FileResource(GenericResource):
     @safe_db_query
     def get_model(self, pk, **kwargs):
         file_path = urllib.parse.unquote(pk)
-        return File.from_path(file_path, get_repo_path())
+        return File.from_path(file_path, get_repo_path(root_project=True))
 
     @safe_db_query
     def delete(self, **kwargs):

mage_ai/api/resources/GitFileResource.py

@@ -55,7 +55,7 @@ class GitFileResource(GenericResource):
                 error = str(err)
 
         return self(dict(
-            content=file.content(),
+            content=file.content() if os.isfile(file.file_path) else None,
             content_from_base=content_from_base,
             content_from_compare=content_from_compare,
             error=error,

mage_ai/api/resources/GlobalHookResource.py

@@ -15,6 +15,7 @@ from mage_ai.data_preparation.models.global_hooks.models import (
 )
 from mage_ai.data_preparation.models.pipeline import Pipeline
 from mage_ai.orchestration.db.models.oauth import User
+from mage_ai.settings.utils import base_repo_path
 from mage_ai.shared.array import find
 from mage_ai.shared.hash import ignore_keys
 
@@ -34,7 +35,14 @@ class GlobalHookResource(AsyncBaseResource):
         if isinstance(resource_types, str):
             resource_types = [EntityName(m) for m in resource_types.split(',') if m]
 
-        global_hooks = GlobalHooks.load_from_file()
+        root_project = query.get('root_project', [False])
+        if root_project:
+            root_project = root_project[0]
+
+        global_hooks = GlobalHooks.load_from_file(
+            all_global_hooks=False,
+            repo_path=base_repo_path() if root_project else None,
+        )
 
         return self.build_result_set(
             global_hooks.hooks(
@@ -47,6 +55,14 @@ class GlobalHookResource(AsyncBaseResource):
 
     @classmethod
     async def create(self, payload: Dict, user: User, **kwargs) -> 'GlobalHookResource':
+        query = kwargs.get('query') or {}
+
+        root_project = query.get('root_project', [False])
+        if root_project:
+            root_project = root_project[0]
+        if not root_project:
+            root_project = payload.get('root_project')
+
         if user and user.id:
             payload['metadata'] = dict(
                 user=dict(
@@ -54,7 +70,7 @@ class GlobalHookResource(AsyncBaseResource):
                 ),
             )
 
-        hook = Hook.load(**payload)
+        hook = Hook.load(**ignore_keys(payload, ['root_project']))
 
         if not hook.uuid or not hook.resource_type or not hook.operation_type:
             missing = []
@@ -78,7 +94,10 @@ class GlobalHookResource(AsyncBaseResource):
             )
             raise ApiError(error)
 
-        global_hooks = GlobalHooks.load_from_file()
+        global_hooks = GlobalHooks.load_from_file(
+            all_global_hooks=False,
+            repo_path=base_repo_path() if root_project else None,
+        )
         if global_hooks.get_hook(
             operation_type=hook.operation_type,
             resource_type=hook.resource_type,
@@ -92,7 +111,7 @@ class GlobalHookResource(AsyncBaseResource):
             raise ApiError(error)
 
         global_hooks.add_hook(hook, snapshot=True)
-        global_hooks.save()
+        global_hooks.save(repo_path=base_repo_path() if root_project else None)
 
         return self(hook, user, **kwargs)
 
@@ -104,6 +123,10 @@ class GlobalHookResource(AsyncBaseResource):
         if resource_type:
             resource_type = resource_type[0]
 
+        root_project = query.get('root_project', [False])
+        if root_project:
+            root_project = root_project[0]
+
         operation_type = query.get('operation_type', [None])
         if operation_type:
             operation_type = operation_type[0]
@@ -116,7 +139,10 @@ class GlobalHookResource(AsyncBaseResource):
         if include_resource_types:
             include_resource_types = include_resource_types[0]
 
-        global_hooks = GlobalHooks.load_from_file()
+        global_hooks = GlobalHooks.load_from_file(
+            all_global_hooks=False,
+            repo_path=base_repo_path() if root_project else None,
+        )
         hook = global_hooks.get_hook(
             operation_type=HookOperation(operation_type) if operation_type else operation_type,
             resource_type=EntityName(resource_type) if resource_type else resource_type,
@@ -129,19 +155,38 @@ class GlobalHookResource(AsyncBaseResource):
         return self(hook, user, **kwargs)
 
     async def update(self, payload: Dict, **kwargs):
-        global_hooks = GlobalHooks.load_from_file()
+        query = kwargs.get('query') or {}
+
+        root_project = query.get('root_project', [False])
+        if root_project:
+            root_project = root_project[0]
+
+        global_hooks = GlobalHooks.load_from_file(
+            all_global_hooks=False,
+            repo_path=base_repo_path() if root_project else None,
+        )
         self.model = global_hooks.add_hook(
             self.model,
-            payload=ignore_keys(payload, ['snapshot']),
+            payload=ignore_keys(payload, ['root_project', 'snapshot']),
             snapshot=payload.get('snapshot') or False,
             update=True,
         )
-        global_hooks.save()
+        global_hooks.save(repo_path=base_repo_path() if root_project else None)
 
     async def delete(self, **kwargs):
-        global_hooks = GlobalHooks.load_from_file()
+        query = kwargs.get('query') or {}
+
+        root_project = query.get('root_project', [False])
+        if root_project:
+            root_project = root_project[0]
+
+        global_hooks = GlobalHooks.load_from_file(
+            all_global_hooks=False,
+            repo_path=base_repo_path() if root_project else None,
+        )
+
         global_hooks.remove_hook(self.model)
-        global_hooks.save()
+        global_hooks.save(repo_path=base_repo_path() if root_project else None)
 
 
 async def __load_pipelines(resource: GlobalHookResource):

mage_ai/api/resources/PipelineResource.py

@@ -427,18 +427,9 @@ class PipelineResource(BaseResource):
 
     @classmethod
     @safe_db_query
-    async def get_model(
-        self,
-        pk,
-        query: Dict = None,
-        resource_class=None,
-        resource_id: str = None,
-        **kwargs,
-    ):
+    async def __fetch_model(self, pipeline_uuid: str, **kqwargs):
         all_projects = project_platform_activated()
 
-        pipeline_uuid = urllib.parse.unquote(pk)
-
         if all_projects:
             return await get_pipeline_from_platform_async(
                 pipeline_uuid,
@@ -446,10 +437,20 @@ class PipelineResource(BaseResource):
 
         return await Pipeline.get_async(pipeline_uuid, all_projects=all_projects)
 
+    @classmethod
+    @safe_db_query
+    async def get_model(
+        self,
+        pk,
+        **kwargs,
+    ):
+        pipeline_uuid = urllib.parse.unquote(pk)
+        return await self.__fetch_model(pipeline_uuid, **kwargs)
+
     @classmethod
     @safe_db_query
     async def member(self, pk, user, **kwargs):
-        pipeline = await Pipeline.get_async(pk, all_projects=project_platform_activated())
+        pipeline = await self.__fetch_model(pk, **kwargs)
 
         api_operation_action = kwargs.get('api_operation_action', None)
         if api_operation_action != DELETE:

mage_ai/api/resources/ProjectResource.py

@@ -1,13 +1,27 @@
+import os
+import subprocess
 import uuid
 
+import yaml
+
 from mage_ai.api.resources.GenericResource import GenericResource
 from mage_ai.cache.block_action_object import BlockActionObjectCache
+from mage_ai.data_preparation.models.block.dbt.profiles import PROFILES_FILE_NAME
 from mage_ai.data_preparation.models.project import Project
 from mage_ai.data_preparation.models.project.constants import FeatureUUID
-from mage_ai.data_preparation.repo_manager import get_repo_config
+from mage_ai.data_preparation.repo_manager import (
+    ProjectType,
+    get_repo_config,
+    init_repo,
+)
 from mage_ai.orchestration.db import safe_db_query
-from mage_ai.settings.platform import activate_project, project_platform_activated
-from mage_ai.shared.hash import merge_dict
+from mage_ai.settings.platform import (
+    activate_project,
+    project_platform_activated,
+    update_settings,
+)
+from mage_ai.settings.utils import base_repo_path
+from mage_ai.shared.hash import combine_into, merge_dict
 from mage_ai.usage_statistics.logger import UsageStatisticLogger
 
 
@@ -17,8 +31,11 @@ async def build_project(repo_config=None, root_project: bool = False, **kwargs):
     model = merge_dict(project.repo_config.to_dict(), dict(
         emr_config=project.emr_config,
         features=project.features,
+        features_defined=project.features_defined,
+        features_override=project.features_override,
         latest_version=await project.latest_version(),
         name=project.name,
+        platform_settings=project.platform_settings,
         project_uuid=project.project_uuid,
         projects=project.projects(),
         remote_variables_dir=project.remote_variables_dir,
@@ -27,6 +44,7 @@ async def build_project(repo_config=None, root_project: bool = False, **kwargs):
         settings=project.settings,
         spark_config=project.spark_config,
         version=project.version,
+        workspace_config_defaults=project.workspace_config_defaults,
     ))
 
     if kwargs:
@@ -39,20 +57,83 @@ class ProjectResource(GenericResource):
     @classmethod
     @safe_db_query
     async def collection(self, query, meta, user, **kwargs):
+
         project = await self.member(None, user, **kwargs)
-        collection = [project.model]
 
+        other_projects = []
         if not project.model.get('root_project') and project_platform_activated():
             root_project = await self.member(None, user, root_project=True, **kwargs)
             if root_project and root_project.model and root_project.model.get('projects'):
-                collection.append(root_project)
+                other_projects.append(root_project)
+
+        features = {}
+        if other_projects:
+            for project2 in other_projects:
+                combine_into(project2.features_defined or {}, features)
+
+        combine_into(project.features_defined or {}, features)
+
+        features2 = project.features or {}
+        combine_into(features, features2)
+
+        project.features = features2
 
         return self.build_result_set(
-            collection,
+            [project] + other_projects,
             user,
             **kwargs,
         )
 
+    @classmethod
+    @safe_db_query
+    async def create(self, payload, user, **kwargs):
+        project_uuid = payload.get('uuid')
+        project_repo_path = payload.get('repo_path')
+        project_type = payload.get('type')
+
+        directory = os.path.join(base_repo_path(), project_repo_path)
+        if ProjectType.STANDALONE == project_type:
+            init_repo(
+                os.path.join(directory, project_uuid),
+                project_type=project_type,
+                project_uuid=project_uuid,
+            )
+        elif 'dbt' == project_type:
+            proc = subprocess.run(
+                [
+                    'dbt',
+                    'init',
+                    project_uuid,
+                    '-s',
+                ],
+                cwd=directory,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                timeout=10,
+            )
+            proc.check_returncode()
+
+            with open(os.path.join(directory, project_uuid, PROFILES_FILE_NAME), 'w') as f:
+                content = yaml.safe_dump({
+                    project_uuid: dict(
+                        outputs=dict(
+                            dev=dict(
+                                dbname='',
+                                host='',
+                                password='postgres',
+                                port=5432,
+                                schema='public',
+                                type='postgres',
+                                user='postgres',
+                            ),
+                        ),
+                        target='dev',
+                    ),
+                })
+                f.write(content)
+
+        return self({}, user, **kwargs)
+
     @classmethod
     @safe_db_query
     async def member(self, _, user, **kwargs):
@@ -64,7 +145,14 @@ class ProjectResource(GenericResource):
         if payload.get('activate_project'):
             activate_project(payload.get('activate_project'))
 
-        repo_config = get_repo_config()
+        platform_settings = payload.get('platform_settings')
+        root_project = payload.get('root_project')
+        repo_config = get_repo_config(root_project=True if root_project else False)
+
+        if root_project and platform_settings:
+            update_settings(settings=platform_settings)
+            self.model = await build_project(repo_config)
+            return self
 
         data = {}
         should_log_project = self.model.get('help_improve_mage') or False

mage_ai/authentication/operation_history/models.py

@@ -105,7 +105,7 @@ class OperationHistoryReader:
     async def load_most_recent_history(self, timestamp: int = None) -> OperationHistory:
         try:
             file_path = self.build_file_path()
-            if self.storage.path_exists(file_path):
+            if file_path and self.storage.path_exists(file_path):
                 line = await read_last_line_async()
                 return OperationHistory(**json.loads(line))
         except Exception as err:

mage_ai/cache/dbt/cache.py

@@ -168,7 +168,7 @@ class DBTCache(BaseCache):
                             absolute_project_file_path(project_path),
                         ),
                     )
-                    project = (mapping.get(project_path) or {}).get('project')
+                project = (mapping.get(project_path) or {}).get('project')
                 models = get_models(project=project)
                 same_value = mapping[project_path].get('models') == models
                 mapping[project_path]['models'] = models

mage_ai/cache/dbt/utils.py

@@ -4,8 +4,10 @@ from typing import Dict, List
 
 import aiofiles
 import yaml
+from jinja2 import Template
 
 from mage_ai.cache.dbt.constants import PROFILES_FILENAME, PROJECT_FILENAMES
+from mage_ai.data_preparation.shared.utils import get_template_vars
 from mage_ai.settings.utils import base_repo_path
 from mage_ai.shared.files import get_full_file_paths_containing_item
 from mage_ai.shared.multi import run_parallel_multiple_args
@@ -44,7 +46,16 @@ def get_models(dirname: str = None, project: Dict = None) -> List[str]:
 async def read_content_async(file_path: str) -> Dict:
     if os.path.exists(file_path):
         async with aiofiles.open(file_path, mode='r') as f:
-            config = yaml.safe_load(await f.read()) or {}
+            content = await f.read()
+            try:
+                content_interpolated = Template(content).render(
+                    variables=lambda x: x,
+                    **get_template_vars(),
+                )
+            except Exception as err:
+                print(f'[WARNING] DBT.cache.read_content_async {file_path}: {err}.')
+                content_interpolated = content
+            config = yaml.safe_load(content_interpolated) or {}
             config['file_path'] = remove_base_repo_path(file_path)
             config['uuid'] = os.path.dirname(remove_base_repo_path(file_path))
             return config
@@ -85,7 +96,16 @@ async def build_mapping_async(file_path: str) -> Dict:
 def read_content(file_path: str) -> Dict:
     if os.path.exists(file_path):
         with open(file_path, mode='r') as f:
-            config = yaml.safe_load(f.read()) or {}
+            content = f.read()
+            try:
+                content_interpolated = Template(content).render(
+                    variables=lambda x: x,
+                    **get_template_vars(),
+                )
+            except Exception as err:
+                print(f'[WARNING] DBT.cache.read_content_async {file_path}: {err}.')
+                content_interpolated = content
+            config = yaml.safe_load(content_interpolated) or {}
             config['file_path'] = remove_base_repo_path(file_path)
             return config