squirrels 0.1.0__py3-none-any.whl → 0.6.0.post0__py3-none-any.whl

squirrels/_api_routes/project.py

@@ -0,0 +1,289 @@
+"""
+Project metadata routes
+"""
+from typing import Any
+from fastapi import FastAPI, Depends, Request
+from fastapi.responses import JSONResponse
+from fastapi.security import HTTPBearer
+from dataclasses import asdict
+from cachetools import TTLCache
+import time
+
+from .. import _utils as u, _constants as c
+from .._schemas import response_models as rm
+from .._parameter_configs import APIParamFieldInfo
+from .._parameter_sets import ParameterSet
+from .._exceptions import ConfigurationError, InvalidInputError
+from .._manifest import PermissionScope, AuthType, AuthStrategy
+from .._schemas.query_param_models import get_query_models_for_parameters
+from .._schemas.auth_models import AbstractUser
+from .base import RouteBase
+
+
+class ProjectRoutes(RouteBase):
+    """Project metadata and data catalog routes"""
+    
+    def __init__(self, get_bearer_token: HTTPBearer, project, no_cache: bool = False):
+        super().__init__(get_bearer_token, project, no_cache)
+        
+        # Setup caches
+        self.parameters_cache = TTLCache(
+            maxsize=self.env_vars.parameters_cache_size, 
+            ttl=self.env_vars.parameters_cache_ttl_minutes*60
+        )
+
+    async def _get_parameters_helper(
+        self, parameters_tuple: tuple[str, ...] | None, entity_type: str, entity_name: str, entity_scope: PermissionScope,
+        user: AbstractUser, selections: tuple[tuple[str, Any], ...]
+    ) -> ParameterSet:
+        """Helper for getting parameters"""
+        selections_dict = dict(selections)
+        if "x_parent_param" not in selections_dict:
+            if len(selections_dict) > 1:
+                raise InvalidInputError(400, "invalid_input_for_cascading_parameters", f"The parameters endpoint takes at most 1 widget parameter selection (unless x_parent_param is provided). Got {selections_dict}")
+            elif len(selections_dict) == 1:
+                parent_param = next(iter(selections_dict))
+                selections_dict["x_parent_param"] = parent_param
+        
+        parent_param = selections_dict.get("x_parent_param")
+        if parent_param is not None and parent_param not in selections_dict:
+            # this condition is possible for multi-select parameters with empty selection
+            selections_dict[parent_param] = list()
+        
+        if not self.authenticator.can_user_access_scope(user, entity_scope):
+            raise self.project._permission_error(user, entity_type, entity_name, entity_scope.name)
+        
+        param_set = self.param_cfg_set.apply_selections(parameters_tuple, selections_dict, user, parent_param=parent_param)
+        return param_set
+
+    async def _get_parameters_cachable(
+        self, parameters_tuple: tuple[str, ...] | None, entity_type: str, entity_name: str, entity_scope: PermissionScope,
+        user: AbstractUser, selections: tuple[tuple[str, Any], ...]
+    ) -> ParameterSet:
+        """Cachable version of parameters helper"""
+        return await self.do_cachable_action(
+            self.parameters_cache, self._get_parameters_helper, parameters_tuple, entity_type, entity_name, entity_scope, user, selections
+        )
+        
+    def setup_routes(
+        self, app: FastAPI, param_fields: dict[str, APIParamFieldInfo]
+    ):
+        """Setup project metadata routes"""
+        project_name = self.manifest_cfg.project_variables.name
+        project_version = str(self.manifest_cfg.project_variables.major_version)
+        label = self.manifest_cfg.project_variables.label
+        description = self.manifest_cfg.project_variables.description
+        auth_type = self.manifest_cfg.project_variables.auth_type.value
+
+        elevated_access_level = self.env_vars.elevated_access_level
+        if elevated_access_level != "admin":
+            self.logger.warning(f"{c.SQRL_PERMISSIONS_ELEVATED_ACCESS_LEVEL} has been set to a non-admin access level. For security reasons, DO NOT expose the APIs for this app publicly!")
+
+        # Project metadata endpoint
+        @app.get("/", summary="Get project metadata and API endpoint paths", tags=["Project Metadata"], response_class=JSONResponse)
+        async def get_project_metadata(request: Request) -> rm.ProjectMetadataModel:
+            project_name_for_api = u.normalize_name_for_api(project_name)
+            auth_strategy = self.manifest_cfg.project_variables.auth_strategy
+            is_external = (auth_strategy == AuthStrategy.EXTERNAL)
+
+            _, root_path = self._get_base_url_for_current_app(request)
+            api_routes = rm.ApiRoutesModel(
+                get_data_catalog_url = root_path + "/data-catalog",
+                get_parameters_url = root_path + "/parameters",
+                get_dataset_parameters_url = root_path + "/datasets/{dataset_name}/parameters",
+                get_dataset_results_url = root_path + "/datasets/{dataset_name}",
+                get_dashboard_parameters_url = root_path + "/dashboards/{dashboard_name}/parameters",
+                get_dashboard_results_url = root_path + "/dashboards/{dashboard_name}",
+                trigger_build_url = root_path + "/build",
+                get_query_result_url = root_path + "/query-result",
+                get_compiled_model_url = root_path + "/compiled-models/{model_name}",
+                get_user_session_url = root_path + "/auth/user-session",
+                list_providers_url = root_path + "/auth/providers",
+                login_url = None if is_external else root_path + "/auth/login",
+                logout_url = root_path + "/auth/logout",
+                change_password_url = None if is_external else root_path + "/auth/password",
+                list_api_keys_url = None if is_external else root_path + "/auth/api-keys",
+                create_api_key_url = None if is_external else root_path + "/auth/api-keys",
+                revoke_api_key_url = None if is_external else root_path + "/auth/api-keys/{key_id}",
+                list_user_fields_url = None if is_external else root_path + "/auth/user-management/user-fields",
+                list_users_url = None if is_external else root_path + "/auth/user-management/users",
+                add_user_url = None if is_external else root_path + "/auth/user-management/users",
+                update_user_url = None if is_external else root_path + "/auth/user-management/users/{username}",
+                delete_user_url = None if is_external else root_path + "/auth/user-management/users/{username}",
+            )
+
+            return rm.ProjectMetadataModel(
+                name = project_name, 
+                name_for_api = project_name_for_api,
+                version = project_version,
+                label = label,
+                description = description,
+                auth_strategy = auth_strategy.value,
+                auth_type = auth_type,
+                password_requirements = None if is_external else self.authenticator.password_requirements,
+                elevated_access_level = elevated_access_level,
+                api_routes = api_routes,
+            )
+        
+        # Data catalog endpoint
+        data_catalog_path = '/data-catalog'
+        
+        async def get_data_catalog0(user: AbstractUser) -> rm.CatalogModel:
+            parameters = self.param_cfg_set.apply_selections(None, {}, user)
+            parameters_model = parameters.to_api_response_model0()
+            full_parameters_list = [p.name for p in parameters_model.parameters]
+            user_has_elevated_privileges = u.user_has_elevated_privileges(user.access_level, elevated_access_level)
+
+            dataset_items: list[rm.DatasetItemModel] = []
+            for name, config in self.manifest_cfg.datasets.items():
+                if self.authenticator.can_user_access_scope(user, config.scope):
+                    name_for_api = u.normalize_name_for_api(name)
+                    metadata = self.project.dataset_metadata(name).to_json()
+                    parameters = config.parameters if config.parameters is not None else full_parameters_list
+                    
+                    # Build dataset-specific configurables list
+                    if user_has_elevated_privileges:
+                        dataset_configurables_defaults = self.manifest_cfg.get_default_configurables(overrides=config.configurables)
+                        dataset_configurables_list = [
+                            rm.ConfigurableOverrideModel(name=name, default=default)
+                            for name, default in dataset_configurables_defaults.items()
+                        ]
+                    else:
+                        dataset_configurables_list = []
+                    
+                    dataset_items.append(rm.DatasetItemModel(
+                        name=name, name_for_api=name_for_api, 
+                        label=config.label, 
+                        description=config.description,
+                        schema=metadata["schema"], # type: ignore
+                        configurables=dataset_configurables_list,
+                        parameters=parameters
+                    ))
+            
+            dashboard_items: list[rm.DashboardItemModel] = []
+            for name, dashboard in self.project._dashboards.items():
+                config = dashboard.config
+                if self.authenticator.can_user_access_scope(user, config.scope):
+                    name_for_api = u.normalize_name_for_api(name)
+
+                    try:
+                        dashboard_format = self.project._dashboards[name].get_dashboard_format()
+                    except KeyError:
+                        raise ConfigurationError(f"No dashboard file found for: {name}")
+
+                    if user_has_elevated_privileges:
+                        dashboard_configurables_defaults = self.manifest_cfg.get_default_configurables(overrides=config.configurables)
+                        dashboard_configurables_list = [
+                            rm.ConfigurableOverrideModel(name=name, default=default)
+                            for name, default in dashboard_configurables_defaults.items()
+                        ]
+                    else:
+                        dashboard_configurables_list = []
+                    
+                    parameters = config.parameters if config.parameters is not None else full_parameters_list
+                    dashboard_items.append(rm.DashboardItemModel(
+                        name=name, name_for_api=name_for_api, 
+                        label=config.label, 
+                        description=config.description, 
+                        result_format=dashboard_format,
+                        configurables=dashboard_configurables_list,
+                        parameters=parameters
+                    ))
+            
+            if user_has_elevated_privileges:
+                compiled_dag = await self.project._get_compiled_dag(user)
+                connections_items = self.project._get_all_connections()
+                data_models = self.project._get_all_data_models(compiled_dag)
+                lineage_items = self.project._get_all_data_lineage(compiled_dag)
+                configurables_list = [
+                    rm.ConfigurableItemModel(name=name, label=cfg.label, default=cfg.default, description=cfg.description)
+                    for name, cfg in self.manifest_cfg.configurables.items()
+                ]
+            else:
+                connections_items = []
+                data_models = []
+                lineage_items = []
+                configurables_list = []
+
+            return rm.CatalogModel(
+                parameters=parameters_model.parameters, 
+                datasets=dataset_items, 
+                dashboards=dashboard_items,
+                connections=connections_items,
+                models=data_models,
+                lineage=lineage_items,
+                configurables=configurables_list,
+            )
+        
+        @app.get(data_catalog_path, tags=["Project Metadata"], summary="Get catalog of datasets and dashboards available for user")
+        async def get_data_catalog(
+            user: AbstractUser = Depends(self.get_current_user)
+        ) -> rm.CatalogModel:
+            """
+            Get catalog of datasets and dashboards available for the authenticated user.
+            
+            For admin users, this endpoint will also return detailed information about all models and their lineage in the project.
+            """
+            start = time.time()
+            
+            # If authentication is required, require user to be authenticated to access catalog
+            if self.manifest_cfg.project_variables.auth_type == AuthType.REQUIRED and user.access_level == "guest":
+                raise InvalidInputError(401, "user_required", "Authentication is required to access the data catalog")
+            data_catalog = await get_data_catalog0(user)
+            
+            self.logger.log_activity_time("GET REQUEST for DATA CATALOG", start)
+            return data_catalog
+        
+        async def get_data_catalog_for_mcp(user: AbstractUser) -> rm.CatalogModelForMcp:
+            """Get data catalog for MCP tools/resources. Takes user object."""
+            data_catalog = await get_data_catalog0(user)
+            return rm.CatalogModelForMcp(parameters=data_catalog.parameters, datasets=data_catalog.datasets)
+        
+        # Store the MCP function as an instance attribute for access by McpServerBuilder
+        self._get_data_catalog_for_mcp = get_data_catalog_for_mcp
+        
+        # Project-level parameters endpoints
+        project_level_parameters_path = '/parameters'
+        parameters_description = "Selections of one parameter may cascade the available options in another parameter. " \
+                "For example, if the dataset has parameters for 'country' and 'city', available options for 'city' would " \
+                "depend on the selected option 'country'. If a parameter has 'trigger_refresh' as true, provide the parameter " \
+                "selection to this endpoint whenever it changes to refresh the parameter options of children parameters."
+
+        QueryModelForGetProjectParams, QueryModelForPostProjectParams = get_query_models_for_parameters(param_fields)
+
+        async def get_parameters_definition(
+            parameters_list: list[str] | None, entity_type: str, entity_name: str, entity_scope: PermissionScope,
+            user: AbstractUser, params: dict
+        ) -> rm.ParametersModel:
+            # self._validate_request_params(all_request_params, params, headers)
+
+            get_parameters_function = self._get_parameters_helper if self.no_cache else self._get_parameters_cachable
+            selections = self.get_selections_as_immutable(params, uncached_keys=set())
+            parameters_tuple = tuple(parameters_list) if parameters_list is not None else None
+            result = await get_parameters_function(parameters_tuple, entity_type, entity_name, entity_scope, user, selections)
+            return result.to_api_response_model0()
+
+        @app.get(project_level_parameters_path, tags=["Project Metadata"], description=parameters_description)
+        async def get_project_parameters(
+            params: QueryModelForGetProjectParams, user=Depends(self.get_current_user)
+        ) -> rm.ParametersModel:
+            start = time.time()
+            result = await get_parameters_definition(
+                None, "project", "", PermissionScope.PUBLIC, user, asdict(params)
+            )
+            self.logger.log_activity_time("GET REQUEST for PROJECT PARAMETERS", start)
+            return result
+
+        @app.post(project_level_parameters_path, tags=["Project Metadata"], description=parameters_description)
+        async def get_project_parameters_with_post(
+            params: QueryModelForPostProjectParams, user=Depends(self.get_current_user)
+        ) -> rm.ParametersModel:
+            start = time.time()
+            result = await get_parameters_definition(
+                None, "project", "", PermissionScope.PUBLIC, user, params.model_dump()
+            )
+            self.logger.log_activity_time("POST REQUEST for PROJECT PARAMETERS", start)
+            return result
+        
+        return get_parameters_definition
+