squirrels 0.1.0__py3-none-any.whl → 0.6.0.post0__py3-none-any.whl

squirrels/_constants.py

@@ -1,64 +1,141 @@
-# Squirrels CLI commands
-INIT_CMD = 'init'
-LOAD_MODULES_CMD = 'load-modules'
-GET_CREDS_CMD = 'get-all-credentials'
-SET_CRED_CMD = 'set-credential'
-DELETE_CRED_CMD = 'delete-credential'
-TEST_CMD = 'test'
-RUN_CMD = 'run'
-
-# Manifest file keys
-PROJ_VARS_KEY = 'project_variables'
-DB_CONNECTIONS_KEY = 'db_connections'
-DB_CREDENTIALS_KEY = 'credential_key'
-URL_KEY = 'url'
-DB_CONNECTION_KEY = 'db_connection'
-MODULES_KEY = 'modules'
-DATASET_LABEL_KEY = 'label'
-DATASETS_KEY = 'datasets'
-HEADERS_KEY = 'headers'
-DATABASE_VIEWS_KEY = 'database_views'
-FILE_KEY = 'file'
-FINAL_VIEW_KEY = 'final_view'
-BASE_PATH_KEY = 'base_path'
-SETTINGS_KEY = 'settings'
-
-# Project variable keys
-PRODUCT_KEY = 'product'
-MAJOR_VERSION_KEY = 'major_version'
-MINOR_VERSION_KEY = 'minor_version'
-
-# Database credentials keys
-CREDENTIALS_KEY = 'credentials'
-USERNAME_KEY = 'username'
-PASSWORD_KEY = 'password'
-DEFAULT_DB_CONN = 'default'
-
-# Folder/File names
-MANIFEST_FILE = 'squirrels.yaml'
-CONNECTIONS_FILE = 'connections.py'
-OUTPUTS_FOLDER = 'outputs'
-MODULES_FOLDER = 'modules'
-DATASETS_FOLDER = 'datasets'
-PARAMETERS_FILE = 'parameters.py'
-PARAMETERS_OUTPUT = 'parameters.json'
-DATABASE_VIEW_SQL_FILE = 'database_view1.sql.j2'
-DATABASE_VIEW_PY_FILE = 'database_view1.py'
-FINAL_VIEW_SQL_NAME = 'final_view.sql.j2'
-FINAL_VIEW_PY_NAME = 'final_view.py'
-FINAL_VIEW_OUT_STEM = 'final_view'
-CONTEXT_FILE = 'context.py'
-SELECTIONS_CFG_FILE = 'selections.cfg'
-
-# Dataset setting names
-PARAMETERS_CACHE_SIZE_SETTING = 'parameters.cache.size'
-PARAMETERS_CACHE_TTL_SETTING = 'parameters.cache.ttl'
-RESULTS_CACHE_SIZE_SETTING = 'results.cache.size'
-RESULTS_CACHE_TTL_SETTING = 'results.cache.ttl'
-
-# Selection cfg sections
-PARAMETERS_SECTION = 'parameters'
-
-# Init Command Choices
-FILE_TYPE_CHOICES = ['sql', 'py']
-DATABASE_CHOICES = ['sample_database', 'seattle_weather']
+# Squirrels CLI commands
+NEW_CMD = 'new'
+INIT_CMD = 'init'
+GET_FILE_CMD = 'get-file'
+DEPS_CMD = 'deps'
+BUILD_CMD = 'build'
+COMPILE_CMD = 'compile'
+RUN_CMD = 'run'
+DUCKDB_CMD = 'duckdb'
+
+# Environment variables
+SQRL_SECRET_KEY = 'SQRL_SECRET__KEY'
+SQRL_SECRET_ADMIN_PASSWORD = 'SQRL_SECRET__ADMIN_PASSWORD'
+
+SQRL_AUTH_DB_FILE_PATH = 'SQRL_AUTH__DB_FILE_PATH'
+SQRL_AUTH_TOKEN_EXPIRE_MINUTES = 'SQRL_AUTH__TOKEN_EXPIRE_MINUTES'
+SQRL_AUTH_CREDENTIAL_ORIGINS = 'SQRL_AUTH__ALLOWED_ORIGINS_FOR_COOKIES'
+
+SQRL_PERMISSIONS_ELEVATED_ACCESS_LEVEL = 'SQRL_PERMISSIONS__ELEVATED_ACCESS_LEVEL'
+
+SQRL_PARAMETERS_CACHE_SIZE = 'SQRL_PARAMETERS__CACHE_SIZE'
+SQRL_PARAMETERS_CACHE_TTL_MINUTES = 'SQRL_PARAMETERS__CACHE_TTL_MINUTES'
+SQRL_PARAMETERS_DATASOURCE_REFRESH_MINUTES = 'SQRL_PARAMETERS__DATASOURCE_REFRESH_MINUTES'
+
+SQRL_DATASETS_CACHE_SIZE = 'SQRL_DATASETS__CACHE_SIZE'
+SQRL_DATASETS_CACHE_TTL_MINUTES = 'SQRL_DATASETS__CACHE_TTL_MINUTES'
+SQRL_DATASETS_MAX_ROWS_FOR_AI = 'SQRL_DATASETS__MAX_ROWS_FOR_AI'
+SQRL_DATASETS_MAX_ROWS_OUTPUT = 'SQRL_DATASETS__MAX_ROWS_OUTPUT'
+SQRL_DATASETS_SQL_TIMEOUT_SECONDS = 'SQRL_DATASETS__SQL_TIMEOUT_SECONDS'
+
+SQRL_DASHBOARDS_CACHE_SIZE = 'SQRL_DASHBOARDS__CACHE_SIZE'
+SQRL_DASHBOARDS_CACHE_TTL_MINUTES = 'SQRL_DASHBOARDS__CACHE_TTL_MINUTES'
+
+SQRL_SEEDS_INFER_SCHEMA = 'SQRL_SEEDS__INFER_SCHEMA'
+SQRL_SEEDS_NA_VALUES = 'SQRL_SEEDS__NA_VALUES'
+
+SQRL_CONNECTIONS_DEFAULT_NAME_USED = 'SQRL_CONNECTIONS__DEFAULT_NAME_USED'
+
+SQRL_VDL_CATALOG_DB_PATH = 'SQRL_VDL__CATALOG_DB_PATH'
+SQRL_VDL_DATA_PATH = 'SQRL_VDL__DATA_PATH'
+
+SQRL_STUDIO_BASE_URL = 'SQRL_STUDIO__BASE_URL'
+
+SQRL_LOGGING_LEVEL = 'SQRL_LOGGING__LEVEL'
+SQRL_LOGGING_FORMAT = 'SQRL_LOGGING__FORMAT'
+SQRL_LOGGING_TO_FILE = 'SQRL_LOGGING__TO_FILE'
+SQRL_LOGGING_FILE_SIZE_MB = 'SQRL_LOGGING__FILE_SIZE_MB'
+SQRL_LOGGING_FILE_BACKUP_COUNT = 'SQRL_LOGGING__FILE_BACKUP_COUNT'
+
+# Folder/File names
+PACKAGE_DATA_FOLDER = '_package_data'
+BASE_PROJECT_FOLDER = 'base_project'
+
+GLOBAL_ENV_FOLDER = '.squirrels'
+MANIFEST_JINJA_FILE = 'squirrels.yml.j2'
+CONNECTIONS_YML_FILE = 'connections.yml'
+PARAMETERS_YML_FILE = 'parameters.yml'
+DASHBOARDS_YML_FILE = 'dashboards.yml'
+MANIFEST_FILE = 'squirrels.yml'
+DUCKDB_INIT_FILE = 'duckdb_init.sql'
+DOTENV_FILE = '.env'
+DOTENV_LOCAL_FILE = '.env.local'
+GITIGNORE_FILE = '.gitignore'
+
+LOGS_FOLDER = 'logs'
+LOGS_FILE = 'squirrels.log'
+
+RESOURCES_FOLDER = 'resources'
+PUBLIC_FOLDER = 'public'
+LOGOS_FOLDER = 'logos'
+PACKAGES_FOLDER = 'sqrl_packages'
+
+MACROS_FOLDER = 'macros'
+MACROS_FILE = 'macros_example.sql'
+
+MODELS_FOLDER = 'models'
+SOURCES_FILE = 'sources.yml'
+BUILDS_FOLDER = 'builds'
+BUILD_FILE_STEM = 'build_example'
+DBVIEWS_FOLDER = 'dbviews'
+DBVIEW_FILE_STEM = 'dbview_example'
+FEDERATES_FOLDER = 'federates'
+FEDERATE_FILE_STEM = 'federate_example'
+
+DASHBOARDS_FOLDER = 'dashboards'
+DASHBOARD_FILE_STEM = 'dashboard_example'
+
+PYCONFIGS_FOLDER = 'pyconfigs'
+CONNECTIONS_FILE = 'connections.py'
+CONTEXT_FILE = 'context.py'
+PARAMETERS_FILE = 'parameters.py'
+USER_FILE = 'user.py'
+ADMIN_USERNAME = 'admin'
+
+TARGET_FOLDER = 'target'
+COMPILE_FOLDER = 'compile'
+COMPILE_BUILDTIME_FOLDER = 'buildtime'
+COMPILE_RUNTIME_FOLDER = 'runtime'
+DB_FILE = 'auth.sqlite'
+
+SEEDS_FOLDER = 'seeds'
+SEED_CATEGORY_FILE_STEM = 'seed_categories'
+SEED_SUBCATEGORY_FILE_STEM = 'seed_subcategories'
+
+# Test sets
+DEFAULT_TEST_SET_NAME = 'default'
+
+# Init Command Choices
+SQL_FILE_TYPE = 'sql'
+PYTHON_FILE_TYPE = 'py'
+FILE_TYPE_CHOICES = [SQL_FILE_TYPE, PYTHON_FILE_TYPE]
+
+YML_FORMAT = 'yml'
+PYTHON_FORMAT = 'py'
+CONF_FORMAT_CHOICES = [YML_FORMAT, PYTHON_FORMAT]
+
+PYTHON_FORMAT2 = 'py (recommended)'
+CONF_FORMAT_CHOICES2 = [(PYTHON_FORMAT2, PYTHON_FORMAT), YML_FORMAT]
+
+EXPENSES_DB = 'expenses.db'
+WEATHER_DB = 'weather.db'
+GOOGLE_LOGO = 'google.ico'
+
+# Dashboard formats
+PNG = "png"
+HTML = "html"
+
+# Function names
+MAIN_FUNC = "main"
+
+# Regex
+DATE_REGEX = r"^\d{4}\-\d{2}\-\d{2}$"
+COLOR_REGEX = r"^#[0-9a-fA-F]{6}$"
+
+# OAuth2
+SUPPORTED_SCOPES = ['read']
+SUPPORTED_GRANT_TYPES = ['authorization_code', 'refresh_token']
+SUPPORTED_RESPONSE_TYPES = ['code']
+
+# Miscellaneous
+LATEST_API_VERSION_MOUNT_PATH = "/api/0"

squirrels/_dashboards.py

@@ -0,0 +1,179 @@
+from typing import Type, TypeVar, Callable, Coroutine, Any
+from typing_extensions import Self
+from enum import Enum
+from dataclasses import dataclass
+from pydantic import BaseModel, Field, model_validator, ValidationError
+import matplotlib.figure as figure
+import os, time, io, abc, typing
+
+from ._arguments.run_time_args import DashboardArgs
+from ._py_module import PyModule
+from ._manifest import AnalyticsOutputConfig, AuthType, PermissionScope, ConfigurableOverride
+from ._exceptions import InvalidInputError, ConfigurationError, FileExecutionError
+from . import _constants as c, _utils as u
+
+
+class Dashboard(metaclass=abc.ABCMeta):
+    """
+    Abstract parent class for all Dashboard classes.
+    """
+    
+    @property
+    @abc.abstractmethod
+    def _content(self) -> bytes | str:
+        pass
+    
+    @property
+    @abc.abstractmethod
+    def _format(self) -> str:
+        pass
+
+
+class PngDashboard(Dashboard):
+    """
+    Instantiate a Dashboard in PNG format from a matplotlib figure or bytes
+    """
+    
+    def __init__(self, content: figure.Figure | io.BytesIO | bytes) -> None:
+        """
+        Constructor for PngDashboard
+
+        Arguments:
+            content: The content of the dashboard as a matplotlib.figure.Figure or bytes
+        """
+        if isinstance(content, figure.Figure):
+            buffer = io.BytesIO()
+            content.savefig(buffer, format=c.PNG)
+            content = buffer.getvalue()
+        
+        if isinstance(content, io.BytesIO):
+            content = content.getvalue()
+        
+        self.__content = content
+
+    @property
+    def _content(self) -> bytes:
+        return self.__content
+    
+    @property
+    def _format(self) -> typing.Literal['png']:
+        return c.PNG
+    
+    def _repr_png_(self):
+        return self._content
+    
+
+class HtmlDashboard(Dashboard):
+    """
+    Instantiate a Dashboard from an HTML string
+    """
+
+    def __init__(self, content: io.StringIO | str) -> None:
+        """
+        Constructor for HtmlDashboard
+
+        Arguments:
+            content: The content of the dashboard as HTML string
+        """
+        if isinstance(content, io.StringIO):
+            content = content.getvalue()
+        
+        self.__content = content
+
+    @property
+    def _content(self) -> str:
+        return self.__content
+    
+    @property
+    def _format(self) -> typing.Literal['html']:
+        return c.HTML
+    
+    def _repr_html_(self):
+        return self._content
+
+
+T = TypeVar('T', bound=Dashboard)
+
+
+class DashboardFormat(Enum):
+    PNG = "png"
+    HTML = "html"
+
+class DashboardDependencies(BaseModel):
+    name: str
+    dataset: str
+    fixed_parameters: dict[str, str] = Field(default_factory=dict)
+
+class DashboardConfig(AnalyticsOutputConfig):
+    format: DashboardFormat = Field(default=DashboardFormat.PNG)
+    depends_on: list[DashboardDependencies] = Field(default_factory=list)
+
+
+@dataclass
+class DashboardDefinition:
+    dashboard_name: str
+    filepath: str
+    config: DashboardConfig
+    project_path: str
+
+    @property
+    def dashboard_func(self) -> Callable[[DashboardArgs], Coroutine[Any, Any, Dashboard]]:
+        if not hasattr(self, '_dashboard_func'):
+            module = PyModule(self.filepath, self.project_path)
+            self._dashboard_func = module.get_func_or_class(c.MAIN_FUNC)
+        return self._dashboard_func
+
+    def get_dashboard_format(self) -> str:
+        return self.config.format.value
+    
+    async def get_dashboard(self, args: DashboardArgs, *, dashboard_type: Type[T] = Dashboard) -> T:
+        try:
+            dashboard = await self.dashboard_func(args)
+            assert isinstance(dashboard, dashboard_type), f"Function does not return expected Dashboard type: {dashboard_type}"
+        except (InvalidInputError, ConfigurationError, FileExecutionError) as e:
+            raise e
+        except Exception as e:
+            raise FileExecutionError(f'Failed to run "{c.MAIN_FUNC}" function for dashboard "{self.dashboard_name}"', e) from e
+
+        return dashboard
+
+
+class DashboardsIO:
+
+    @classmethod
+    def load_files(
+        cls, logger: u.Logger, project_path: str, auth_type: AuthType = AuthType.OPTIONAL, project_configurables: dict[str, Any] | None = None
+    ) -> dict[str, DashboardDefinition]:
+        start = time.time()
+        
+        if project_configurables is None:
+            project_configurables = {}
+        
+        default_scope = PermissionScope.PROTECTED if auth_type == AuthType.REQUIRED else PermissionScope.PUBLIC
+        
+        dashboards_by_name = {}
+        for dp, _, filenames in os.walk(u.Path(project_path, c.DASHBOARDS_FOLDER)):
+            for file in filenames:
+                filepath = os.path.join(dp, file)
+                file_stem, extension = os.path.splitext(file)
+                if not extension == '.py':
+                    continue
+                
+                # Check for corresponding .yml file
+                yml_path = os.path.join(dp, file_stem + '.yml')
+                config_dict = u.load_yaml_config(yml_path) if os.path.exists(yml_path) else {}
+                try:
+                    config = DashboardConfig(name=file_stem, project_configurables=project_configurables, **config_dict)
+                except ValidationError as e:
+                    raise ConfigurationError(f'Failed to process dashboard config "{file_stem}". ' + str(e)) from e
+                
+                if config.scope is None:
+                    config.scope = default_scope
+                
+                if auth_type == AuthType.REQUIRED and config.scope == PermissionScope.PUBLIC:
+                    raise ConfigurationError(f'Authentication is required, so dashboard "{file_stem}" cannot be public. Update the scope in "{yml_path}"')
+                
+                dashboards_by_name[file_stem] = DashboardDefinition(file_stem, filepath, config, project_path)
+                
+        logger.log_activity_time("loading files for dashboards", start)
+        return dashboards_by_name