mage-ai 0.9.56__py3-none-any.whl → 0.9.57__py3-none-any.whl

mage_ai/api/policies/GlobalHookPolicy.py

@@ -10,6 +10,7 @@ WRITEABLE_ATTRIBUTES = [
     'pipeline',
     'predicate',
     'resource_type',
+    'root_project',
     'run_settings',
     'stages',
     'strategies',
@@ -92,6 +93,7 @@ GlobalHookPolicy.allow_query(
         'include_snapshot_validation',
         'operation_type[]',
         'resource_type[]',
+        'root_project',
     ],
     scopes=[
         OauthScope.CLIENT_PRIVATE,
@@ -111,6 +113,7 @@ GlobalHookPolicy.allow_query(
         'include_snapshot_validation',
         'operation_type',
         'resource_type',
+        'root_project',
     ],
     scopes=[
         OauthScope.CLIENT_PRIVATE,

mage_ai/api/policies/ProjectPolicy.py

@@ -68,7 +68,9 @@ ProjectPolicy.allow_write([
     'features',
     'openai_api_key',
     'pipelines',
+    'platform_settings',
     'remote_variables_dir',
+    'root_project',
     'spark_config',
 ], scopes=[
     OauthScope.CLIENT_PRIVATE,

mage_ai/api/presenters/ProjectPresenter.py

@@ -5,11 +5,14 @@ class ProjectPresenter(BasePresenter):
     default_attributes = [
         'emr_config',
         'features',
+        'features_defined',
+        'features_override',
         'help_improve_mage',
         'latest_version',
         'name',
         'openai_api_key',
         'pipelines',
+        'platform_settings',
         'project_uuid',
         'projects',
         'remote_variables_dir',

mage_ai/api/resources/GlobalHookResource.py

@@ -15,6 +15,7 @@ from mage_ai.data_preparation.models.global_hooks.models import (
 )
 from mage_ai.data_preparation.models.pipeline import Pipeline
 from mage_ai.orchestration.db.models.oauth import User
+from mage_ai.settings.utils import base_repo_path
 from mage_ai.shared.array import find
 from mage_ai.shared.hash import ignore_keys
 
@@ -34,7 +35,14 @@ class GlobalHookResource(AsyncBaseResource):
         if isinstance(resource_types, str):
             resource_types = [EntityName(m) for m in resource_types.split(',') if m]
 
-        global_hooks = GlobalHooks.load_from_file()
+        root_project = query.get('root_project', [False])
+        if root_project:
+            root_project = root_project[0]
+
+        global_hooks = GlobalHooks.load_from_file(
+            all_global_hooks=False,
+            repo_path=base_repo_path() if root_project else None,
+        )
 
         return self.build_result_set(
             global_hooks.hooks(
@@ -47,6 +55,14 @@ class GlobalHookResource(AsyncBaseResource):
 
     @classmethod
     async def create(self, payload: Dict, user: User, **kwargs) -> 'GlobalHookResource':
+        query = kwargs.get('query') or {}
+
+        root_project = query.get('root_project', [False])
+        if root_project:
+            root_project = root_project[0]
+        if not root_project:
+            root_project = payload.get('root_project')
+
         if user and user.id:
             payload['metadata'] = dict(
                 user=dict(
@@ -54,7 +70,7 @@ class GlobalHookResource(AsyncBaseResource):
                 ),
             )
 
-        hook = Hook.load(**payload)
+        hook = Hook.load(**ignore_keys(payload, ['root_project']))
 
         if not hook.uuid or not hook.resource_type or not hook.operation_type:
             missing = []
@@ -78,7 +94,10 @@ class GlobalHookResource(AsyncBaseResource):
             )
             raise ApiError(error)
 
-        global_hooks = GlobalHooks.load_from_file()
+        global_hooks = GlobalHooks.load_from_file(
+            all_global_hooks=False,
+            repo_path=base_repo_path() if root_project else None,
+        )
         if global_hooks.get_hook(
             operation_type=hook.operation_type,
             resource_type=hook.resource_type,
@@ -92,7 +111,7 @@ class GlobalHookResource(AsyncBaseResource):
             raise ApiError(error)
 
         global_hooks.add_hook(hook, snapshot=True)
-        global_hooks.save()
+        global_hooks.save(repo_path=base_repo_path() if root_project else None)
 
         return self(hook, user, **kwargs)
 
@@ -104,6 +123,10 @@ class GlobalHookResource(AsyncBaseResource):
         if resource_type:
             resource_type = resource_type[0]
 
+        root_project = query.get('root_project', [False])
+        if root_project:
+            root_project = root_project[0]
+
         operation_type = query.get('operation_type', [None])
         if operation_type:
             operation_type = operation_type[0]
@@ -116,7 +139,10 @@ class GlobalHookResource(AsyncBaseResource):
         if include_resource_types:
             include_resource_types = include_resource_types[0]
 
-        global_hooks = GlobalHooks.load_from_file()
+        global_hooks = GlobalHooks.load_from_file(
+            all_global_hooks=False,
+            repo_path=base_repo_path() if root_project else None,
+        )
         hook = global_hooks.get_hook(
             operation_type=HookOperation(operation_type) if operation_type else operation_type,
             resource_type=EntityName(resource_type) if resource_type else resource_type,
@@ -129,19 +155,38 @@ class GlobalHookResource(AsyncBaseResource):
         return self(hook, user, **kwargs)
 
     async def update(self, payload: Dict, **kwargs):
-        global_hooks = GlobalHooks.load_from_file()
+        query = kwargs.get('query') or {}
+
+        root_project = query.get('root_project', [False])
+        if root_project:
+            root_project = root_project[0]
+
+        global_hooks = GlobalHooks.load_from_file(
+            all_global_hooks=False,
+            repo_path=base_repo_path() if root_project else None,
+        )
         self.model = global_hooks.add_hook(
             self.model,
-            payload=ignore_keys(payload, ['snapshot']),
+            payload=ignore_keys(payload, ['root_project', 'snapshot']),
             snapshot=payload.get('snapshot') or False,
             update=True,
         )
-        global_hooks.save()
+        global_hooks.save(repo_path=base_repo_path() if root_project else None)
 
     async def delete(self, **kwargs):
-        global_hooks = GlobalHooks.load_from_file()
+        query = kwargs.get('query') or {}
+
+        root_project = query.get('root_project', [False])
+        if root_project:
+            root_project = root_project[0]
+
+        global_hooks = GlobalHooks.load_from_file(
+            all_global_hooks=False,
+            repo_path=base_repo_path() if root_project else None,
+        )
+
         global_hooks.remove_hook(self.model)
-        global_hooks.save()
+        global_hooks.save(repo_path=base_repo_path() if root_project else None)
 
 
 async def __load_pipelines(resource: GlobalHookResource):

mage_ai/api/resources/ProjectResource.py

@@ -15,7 +15,11 @@ from mage_ai.data_preparation.repo_manager import (
     init_repo,
 )
 from mage_ai.orchestration.db import safe_db_query
-from mage_ai.settings.platform import activate_project, project_platform_activated
+from mage_ai.settings.platform import (
+    activate_project,
+    project_platform_activated,
+    update_settings,
+)
 from mage_ai.settings.utils import base_repo_path
 from mage_ai.shared.hash import combine_into, merge_dict
 from mage_ai.usage_statistics.logger import UsageStatisticLogger
@@ -28,8 +32,10 @@ async def build_project(repo_config=None, root_project: bool = False, **kwargs):
         emr_config=project.emr_config,
         features=project.features,
         features_defined=project.features_defined,
+        features_override=project.features_override,
         latest_version=await project.latest_version(),
         name=project.name,
+        platform_settings=project.platform_settings,
         project_uuid=project.project_uuid,
         projects=project.projects(),
         remote_variables_dir=project.remote_variables_dir,
@@ -139,7 +145,14 @@ class ProjectResource(GenericResource):
         if payload.get('activate_project'):
             activate_project(payload.get('activate_project'))
 
-        repo_config = get_repo_config()
+        platform_settings = payload.get('platform_settings')
+        root_project = payload.get('root_project')
+        repo_config = get_repo_config(root_project=True if root_project else False)
+
+        if root_project and platform_settings:
+            update_settings(settings=platform_settings)
+            self.model = await build_project(repo_config)
+            return self
 
         data = {}
         should_log_project = self.model.get('help_improve_mage') or False

mage_ai/cache/dbt/cache.py

@@ -168,7 +168,7 @@ class DBTCache(BaseCache):
                             absolute_project_file_path(project_path),
                         ),
                     )
-                    project = (mapping.get(project_path) or {}).get('project')
+                project = (mapping.get(project_path) or {}).get('project')
                 models = get_models(project=project)
                 same_value = mapping[project_path].get('models') == models
                 mapping[project_path]['models'] = models

mage_ai/cache/dbt/utils.py

@@ -4,8 +4,10 @@ from typing import Dict, List
 
 import aiofiles
 import yaml
+from jinja2 import Template
 
 from mage_ai.cache.dbt.constants import PROFILES_FILENAME, PROJECT_FILENAMES
+from mage_ai.data_preparation.shared.utils import get_template_vars
 from mage_ai.settings.utils import base_repo_path
 from mage_ai.shared.files import get_full_file_paths_containing_item
 from mage_ai.shared.multi import run_parallel_multiple_args
@@ -44,7 +46,16 @@ def get_models(dirname: str = None, project: Dict = None) -> List[str]:
 async def read_content_async(file_path: str) -> Dict:
     if os.path.exists(file_path):
         async with aiofiles.open(file_path, mode='r') as f:
-            config = yaml.safe_load(await f.read()) or {}
+            content = await f.read()
+            try:
+                content_interpolated = Template(content).render(
+                    variables=lambda x: x,
+                    **get_template_vars(),
+                )
+            except Exception as err:
+                print(f'[WARNING] DBT.cache.read_content_async {file_path}: {err}.')
+                content_interpolated = content
+            config = yaml.safe_load(content_interpolated) or {}
             config['file_path'] = remove_base_repo_path(file_path)
             config['uuid'] = os.path.dirname(remove_base_repo_path(file_path))
             return config
@@ -85,7 +96,16 @@ async def build_mapping_async(file_path: str) -> Dict:
 def read_content(file_path: str) -> Dict:
     if os.path.exists(file_path):
         with open(file_path, mode='r') as f:
-            config = yaml.safe_load(f.read()) or {}
+            content = f.read()
+            try:
+                content_interpolated = Template(content).render(
+                    variables=lambda x: x,
+                    **get_template_vars(),
+                )
+            except Exception as err:
+                print(f'[WARNING] DBT.cache.read_content_async {file_path}: {err}.')
+                content_interpolated = content
+            config = yaml.safe_load(content_interpolated) or {}
             config['file_path'] = remove_base_repo_path(file_path)
             return config
 

mage_ai/data_preparation/models/block/__init__.py

@@ -919,16 +919,26 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
         parts = get_path_parts(file_path)
 
         if parts and len(parts) >= 3:
+            # If file_path == transformers/test4.py
+            # parts ==
+            # ('/home/src/default_repo/default_platform2/project3', 'transformers', 'test4.py')
+
+            # If project platform platform activated, then parts ==
             # ('/home/src', 'default_repo', 'data_loaders/astral_violet.py')
+
             root_project_full_path, path, file_path_base = parts
 
-            # ('data_loaders', 'astral_violet.py')
-            file_parts = Path(file_path_base).parts
-            block_type = inflection.singularize(str(file_parts[0]))
-            block_uuid = str(Path(*file_parts[1:]))
+            if project_platform_activated():
+                # ('data_loaders', 'astral_violet.py')
+                file_parts = Path(file_path_base).parts
+                block_type = inflection.singularize(str(file_parts[0]))
+                block_uuid = str(Path(*file_parts[1:]).with_suffix(''))
+            else:
+                block_type = inflection.singularize(str(path))
+                block_uuid = str(Path(file_path_base).with_suffix(''))
 
-            configuration = dict(file_source=dict(path=file_path))
             extension = Path(file_path).suffix.replace('.', '')
+            configuration = dict(file_path=file_path, file_source=dict(path=file_path))
             language = FILE_EXTENSION_TO_BLOCK_LANGUAGE.get(extension)
 
             return self.get_block(

mage_ai/data_preparation/models/block/dbt/block_sql.py

@@ -65,7 +65,9 @@ class DBTBlockSQL(DBTBlock, ProjectPlatformAccessible):
             if file_path:
                 return file_path
 
-        file_path = self.configuration.get('file_path')
+        file_path = self.configuration.get('file_path') or ''
+        if file_path and Path(file_path).parts[0] == 'dbt':
+            file_path = os.path.join(*Path(file_path).parts[1:]) or ''
 
         return str((Path(self.repo_path)) / DBT_DIRECTORY_NAME / file_path)
 
@@ -82,9 +84,15 @@ class DBTBlockSQL(DBTBlock, ProjectPlatformAccessible):
             if project_path:
                 return project_path
 
+        try:
+            diff = Path(self.file_path).relative_to(self.base_project_path)
+            if diff:
+                diff = Path(diff).parts[0]
+        except ValueError:
+            diff = self.file_path
+
         return str(
-            Path(self.base_project_path) /
-            self.configuration.get('file_path', '').split(os.sep)[0]
+            Path(self.base_project_path) / diff,
         )
 
     @property
@@ -187,6 +195,7 @@ class DBTBlockSQL(DBTBlock, ProjectPlatformAccessible):
         # Get upstream nodes via dbt list
         with Profiles(self.project_path, self.pipeline.variables) as profiles:
             try:
+
                 args = [
                     'list',
                     # project-dir

mage_ai/data_preparation/models/file.py

@@ -484,7 +484,7 @@ def traverse(
 
 def __should_update_dbt_cache(dir_path: str, filename: str) -> bool:
     project_model = Project(root_project=True)
-    if project_model and project_model.is_feature_enabled(FeatureUUID.DBT_V2):
+    if project_model and project_model.is_feature_enabled(FeatureUUID.DBT_V2) and dir_path:
         # If the file is a SQL or YAML file
         if (
             filename.endswith('.sql') or

mage_ai/data_preparation/models/global_hooks/models.py

@@ -1085,9 +1085,9 @@ class GlobalHooks(BaseDataClass):
             user=user,
         )
 
-    def save(self, file_path: str = None) -> None:
+    def save(self, file_path: str = None, repo_path: str = None) -> None:
         if not file_path:
-            file_path = self.file_path()
+            file_path = self.file_path(repo_path=repo_path)
 
         content_original = None
         if os.path.exists(file_path):

mage_ai/data_preparation/models/project/__init__.py

@@ -9,6 +9,7 @@ from mage_ai.data_preparation.repo_manager import get_cluster_type, get_repo_con
 from mage_ai.server.constants import VERSION
 from mage_ai.settings.platform import (
     active_project_settings,
+    platform_settings,
     project_platform_activated,
     project_platform_settings,
 )
@@ -38,6 +39,14 @@ class Project():
         self.version = VERSION
         self._features = None
 
+        self.__repo_config_root_project = None
+
+        if project_platform_activated():
+            self.__repo_config_root_project = get_repo_config(
+                repo_path=get_repo_path(root_project=True),
+                root_project=True,
+            )
+
     @property
     def workspace_config_defaults(self) -> Dict:
         config = self.repo_config.workspace_config_defaults or {}
@@ -92,6 +101,11 @@ class Project():
         data = {}
         features = self.repo_config.features if self.repo_config else {}
 
+        if project_platform_activated() and self.__repo_config_root_project:
+            settings = platform_settings()
+            if settings.get('features') and (settings.get('features') or {}).get('override'):
+                features.update(self.features_override)
+
         for uuid in FeatureUUID:
             key = uuid.value
             if FeatureUUID.PROJECT_PLATFORM == uuid:
@@ -102,6 +116,15 @@ class Project():
 
         return data
 
+    @property
+    def features_override(self) -> Dict:
+        if project_platform_activated() and self.__repo_config_root_project:
+            settings = self.platform_settings()
+            if settings.get('features') and (settings.get('features') or {}).get('override'):
+                return self.__repo_config_root_project.features or {}
+
+        return {}
+
     @property
     def emr_config(self) -> Dict:
         return self.repo_config.emr_config or None
@@ -128,6 +151,10 @@ class Project():
 
         return False
 
+    def platform_settings(self) -> Dict:
+        if project_platform_activated():
+            return platform_settings(mage_projects_only=True)
+
     def repo_path_for_database_query(self, key: str) -> List[str]:
         if self.settings:
             query_arr = dig(self.settings, ['database', 'query', key])
@@ -145,7 +172,7 @@ class Project():
         return project_platform_settings(mage_projects_only=True)
 
     def is_feature_enabled(self, feature_name: FeatureUUID) -> bool:
-        feature_enabled = self.repo_config.features.get(feature_name.value, False)
+        feature_enabled = self.features.get(feature_name.value, False)
 
         if is_debug():
             print(f'[Project.is_feature_enabled]: {feature_name} | {feature_enabled}')