squirrels 0.4.1__py3-none-any.whl → 0.5.0__py3-none-any.whl

squirrels/_constants.py

@@ -1,35 +1,78 @@
 # Squirrels CLI commands
-INIT_CMD = 'init'
+INIT_CMD = 'new'
 GET_FILE_CMD = 'get-file'
 DEPS_CMD = 'deps'
+BUILD_CMD = 'build'
 COMPILE_CMD = 'compile'
 RUN_CMD = 'run'
+DUCKDB_CMD = 'duckdb'
 
-# Environment config keys
-JWT_SECRET_KEY = 'jwt_secret'
+# Environment variables
+SQRL_SECRET_KEY = 'SQRL_SECRET__KEY'
+SQRL_SECRET_ADMIN_PASSWORD = 'SQRL_SECRET__ADMIN_PASSWORD'
+
+SQRL_AUTH_DB_FILE_PATH = 'SQRL_AUTH__DB_FILE_PATH'
+SQRL_AUTH_TOKEN_EXPIRE_MINUTES = 'SQRL_AUTH__TOKEN_EXPIRE_MINUTES'
+SQRL_AUTH_CREDENTIAL_ORIGINS = 'SQRL_AUTH__ALLOWED_ORIGINS_FOR_COOKIES'
+
+SQRL_PARAMETERS_CACHE_SIZE = 'SQRL_PARAMETERS__CACHE_SIZE'
+SQRL_PARAMETERS_CACHE_TTL_MINUTES = 'SQRL_PARAMETERS__CACHE_TTL_MINUTES'
+SQRL_PARAMETERS_DATASOURCE_REFRESH_MINUTES = 'SQRL_PARAMETERS__DATASOURCE_REFRESH_MINUTES'
+
+SQRL_DATASETS_CACHE_SIZE = 'SQRL_DATASETS__CACHE_SIZE'
+SQRL_DATASETS_CACHE_TTL_MINUTES = 'SQRL_DATASETS__CACHE_TTL_MINUTES'
+SQRL_DATASETS_MAX_ROWS_FOR_AI = 'SQRL_DATASETS__MAX_ROWS_FOR_AI'
+
+SQRL_DASHBOARDS_CACHE_SIZE = 'SQRL_DASHBOARDS__CACHE_SIZE'
+SQRL_DASHBOARDS_CACHE_TTL_MINUTES = 'SQRL_DASHBOARDS__CACHE_TTL_MINUTES'
+
+SQRL_PERMISSIONS_ELEVATED_ACCESS_LEVEL = 'SQRL_PERMISSIONS__ELEVATED_ACCESS_LEVEL'
+
+SQRL_SEEDS_INFER_SCHEMA = 'SQRL_SEEDS__INFER_SCHEMA'
+SQRL_SEEDS_NA_VALUES = 'SQRL_SEEDS__NA_VALUES'
+
+SQRL_CONNECTIONS_DEFAULT_NAME_USED = 'SQRL_CONNECTIONS__DEFAULT_NAME_USED'
+
+SQRL_VDL_CATALOG_DB_PATH = 'SQRL_VDL__CATALOG_DB_PATH'
+SQRL_VDL_DATA_PATH = 'SQRL_VDL__DATA_PATH'
+
+SQRL_STUDIO_BASE_URL = 'SQRL_STUDIO__BASE_URL'
+
+SQRL_LOGGING_LOG_LEVEL = 'SQRL_LOGGING__LOG_LEVEL'
+SQRL_LOGGING_LOG_FORMAT = 'SQRL_LOGGING__LOG_FORMAT'
+SQRL_LOGGING_LOG_TO_FILE = 'SQRL_LOGGING__LOG_TO_FILE'
+SQRL_LOGGING_LOG_FILE_SIZE_MB = 'SQRL_LOGGING__LOG_FILE_SIZE_MB'
+SQRL_LOGGING_LOG_FILE_BACKUP_COUNT = 'SQRL_LOGGING__LOG_FILE_BACKUP_COUNT'
 
 # Folder/File names
-PACKAGE_DATA_FOLDER = 'package_data'
+PACKAGE_DATA_FOLDER = '_package_data'
 BASE_PROJECT_FOLDER = 'base_project'
-ASSETS_FOLDER = 'assets'
-TEMPLATES_FOLDER = 'templates'
 
-ENV_CONFIG_FILE = 'env.yml'
+GLOBAL_ENV_FOLDER = '.squirrels'
 MANIFEST_JINJA_FILE = 'squirrels.yml.j2'
 CONNECTIONS_YML_FILE = 'connections.yml'
 PARAMETERS_YML_FILE = 'parameters.yml'
 DASHBOARDS_YML_FILE = 'dashboards.yml'
 MANIFEST_FILE = 'squirrels.yml'
+DUCKDB_INIT_FILE = 'duckdb_init.sql'
+DOTENV_FILE = '.env'
+DOTENV_LOCAL_FILE = '.env.local'
+GITIGNORE_FILE = '.gitignore'
 
 LOGS_FOLDER = 'logs'
 LOGS_FILE = 'squirrels.log'
 
 DATABASE_FOLDER = 'assets'
 PACKAGES_FOLDER = 'sqrl_packages'
+PUBLIC_FOLDER = 'public'
 
 MACROS_FOLDER = 'macros'
+MACROS_FILE = 'macros_example.sql'
 
 MODELS_FOLDER = 'models'
+SOURCES_FILE = 'sources.yml'
+BUILDS_FOLDER = 'builds'
+BUILD_FILE_STEM = 'build_example'
 DBVIEWS_FOLDER = 'dbviews'
 DBVIEW_FILE_STEM = 'dbview_example'
 FEDERATES_FOLDER = 'federates'
@@ -43,39 +86,23 @@ AUTH_FILE = 'auth.py'
 CONNECTIONS_FILE = 'connections.py'
 CONTEXT_FILE = 'context.py'
 PARAMETERS_FILE = 'parameters.py'
+USER_FILE = 'user.py'
+ADMIN_USERNAME = 'admin'
 
 TARGET_FOLDER = 'target'
 COMPILE_FOLDER = 'compile'
+COMPILE_BUILDTIME_FOLDER = 'buildtime'
+COMPILE_RUNTIME_FOLDER = 'runtime'
+DB_FILE = 'auth.sqlite'
+DEFAULT_VDL_CATALOG_DB_PATH = 'ducklake:{project_path}/target/vdl_catalog.duckdb'
+DEFAULT_VDL_DATA_PATH = '{project_path}/target/vdl_data/'
 
 SEEDS_FOLDER = 'seeds'
-CATEGORY_SEED_FILE = 'seed_categories.csv'
-SUBCATEGORY_SEED_FILE = 'seed_subcategories.csv'
-
-# Dataset setting names
-AUTH_TOKEN_EXPIRE_SETTING = 'auth.token.expire_minutes'
-PARAMETERS_CACHE_SIZE_SETTING = 'parameters.cache.size'
-PARAMETERS_CACHE_TTL_SETTING = 'parameters.cache.ttl_minutes'
-RESULTS_CACHE_SIZE_SETTING = 'results.cache.size' # deprecated
-RESULTS_CACHE_TTL_SETTING = 'results.cache.ttl_minutes' # deprecated
-DATASETS_CACHE_SIZE_SETTING = 'datasets.cache.size'
-DATASETS_CACHE_TTL_SETTING = 'datasets.cache.ttl_minutes'
-DASHBOARDS_CACHE_SIZE_SETTING = 'dashboards.cache.size'
-DASHBOARDS_CACHE_TTL_SETTING = 'dashboards.cache.ttl_minutes'
-TEST_SET_DEFAULT_USED_SETTING = 'selection_test_sets.default_name_used'
+SEED_CATEGORY_FILE_STEM = 'seed_categories'
+SEED_SUBCATEGORY_FILE_STEM = 'seed_subcategories'
+
+# Test sets
 DEFAULT_TEST_SET_NAME = 'default'
-DB_CONN_DEFAULT_USED_SETTING = 'connections.default_name_used'
-DEFAULT_DB_CONN = 'default'
-DEFAULT_MATERIALIZE_SETTING = 'defaults.federates.materialized'
-DEFAULT_MATERIALIZE = 'table'
-SEEDS_INFER_SCHEMA_SETTING = 'seeds.infer_schema'
-SEEDS_NA_VALUES_SETTING = 'seeds.na_values'
-IN_MEMORY_DB_SETTING = 'in_memory_database'
-SQLITE = 'sqlite'
-DUCKDB = 'duckdb'
-
-# Selection cfg sections
-USER_ATTRIBUTES_SECTION = 'user_attributes'
-PARAMETERS_SECTION = 'parameters'
 
 # Init Command Choices
 SQL_FILE_TYPE = 'sql'
@@ -97,10 +124,13 @@ PNG = "png"
 HTML = "html"
 
 # Function names
-GET_USER_FUNC = "get_user_if_valid"
-DEP_FUNC = "dependencies"
 MAIN_FUNC = "main"
 
 # Regex
-date_regex = r"^\d{4}\-\d{2}\-\d{2}$"
-color_regex = r"^#[0-9a-fA-F]{6}$"
+DATE_REGEX = r"^\d{4}\-\d{2}\-\d{2}$"
+COLOR_REGEX = r"^#[0-9a-fA-F]{6}$"
+
+# OAuth2
+SUPPORTED_SCOPES = ['read']
+SUPPORTED_GRANT_TYPES = ['authorization_code', 'refresh_token']
+SUPPORTED_RESPONSE_TYPES = ['code']

squirrels/_dashboards.py

@@ -0,0 +1,160 @@
+from typing import Type, TypeVar, Callable, Coroutine, Any
+from enum import Enum
+from dataclasses import dataclass
+from pydantic import BaseModel, Field
+import matplotlib.figure as figure
+import os, time, io, abc, typing
+
+from ._arguments.run_time_args import DashboardArgs
+from ._py_module import PyModule
+from ._manifest import AnalyticsOutputConfig
+from ._exceptions import InvalidInputError, ConfigurationError, FileExecutionError
+from . import _constants as c, _utils as u
+
+
+class Dashboard(metaclass=abc.ABCMeta):
+    """
+    Abstract parent class for all Dashboard classes.
+    """
+    
+    @property
+    @abc.abstractmethod
+    def _content(self) -> bytes | str:
+        pass
+    
+    @property
+    @abc.abstractmethod
+    def _format(self) -> str:
+        pass
+
+
+class PngDashboard(Dashboard):
+    """
+    Instantiate a Dashboard in PNG format from a matplotlib figure or bytes
+    """
+    
+    def __init__(self, content: figure.Figure | io.BytesIO | bytes) -> None:
+        """
+        Constructor for PngDashboard
+
+        Arguments:
+            content: The content of the dashboard as a matplotlib.figure.Figure or bytes
+        """
+        if isinstance(content, figure.Figure):
+            buffer = io.BytesIO()
+            content.savefig(buffer, format=c.PNG)
+            content = buffer.getvalue()
+        
+        if isinstance(content, io.BytesIO):
+            content = content.getvalue()
+        
+        self.__content = content
+
+    @property
+    def _content(self) -> bytes:
+        return self.__content
+    
+    @property
+    def _format(self) -> typing.Literal['png']:
+        return c.PNG
+    
+    def _repr_png_(self):
+        return self._content
+    
+
+class HtmlDashboard(Dashboard):
+    """
+    Instantiate a Dashboard from an HTML string
+    """
+
+    def __init__(self, content: io.StringIO | str) -> None:
+        """
+        Constructor for HtmlDashboard
+
+        Arguments:
+            content: The content of the dashboard as HTML string
+        """
+        if isinstance(content, io.StringIO):
+            content = content.getvalue()
+        
+        self.__content = content
+
+    @property
+    def _content(self) -> str:
+        return self.__content
+    
+    @property
+    def _format(self) -> typing.Literal['html']:
+        return c.HTML
+    
+    def _repr_html_(self):
+        return self._content
+
+
+T = TypeVar('T', bound=Dashboard)
+
+
+class DashboardFormat(Enum):
+    PNG = "png"
+    HTML = "html"
+
+class DashboardDependencies(BaseModel):
+    name: str
+    dataset: str
+    fixed_parameters: list[dict[str, str]] = Field(default_factory=list)
+
+class DashboardConfig(AnalyticsOutputConfig):
+    format: DashboardFormat = Field(default=DashboardFormat.PNG)
+    depends_on: list[DashboardDependencies] = Field(default_factory=list)
+
+
+@dataclass
+class DashboardDefinition:
+    dashboard_name: str
+    filepath: str
+    config: DashboardConfig
+
+    @property
+    def dashboard_func(self) -> Callable[[DashboardArgs], Coroutine[Any, Any, Dashboard]]:
+        if not hasattr(self, '_dashboard_func'):
+            module = PyModule(self.filepath)
+            self._dashboard_func = module.get_func_or_class(c.MAIN_FUNC)
+        return self._dashboard_func
+
+    def get_dashboard_format(self) -> str:
+        return self.config.format.value
+    
+    async def get_dashboard(self, args: DashboardArgs, *, dashboard_type: Type[T] = Dashboard) -> T:
+        try:
+            dashboard = await self.dashboard_func(args)
+            assert isinstance(dashboard, dashboard_type), f"Function does not return expected Dashboard type: {dashboard_type}"
+        except (InvalidInputError, ConfigurationError, FileExecutionError) as e:
+            raise e
+        except Exception as e:
+            raise FileExecutionError(f'Failed to run "{c.MAIN_FUNC}" function for dashboard "{self.dashboard_name}"', e) from e
+
+        return dashboard
+
+
+class DashboardsIO:
+
+    @classmethod
+    def load_files(cls, logger: u.Logger, base_path: str) -> dict[str, DashboardDefinition]:
+        start = time.time()
+        
+        dashboards_by_name = {}
+        for dp, _, filenames in os.walk(u.Path(base_path, c.DASHBOARDS_FOLDER)):
+            for file in filenames:
+                filepath = os.path.join(dp, file)
+                file_stem, extension = os.path.splitext(file)
+                if not extension == '.py':
+                    continue
+                
+                # Check for corresponding .yml file
+                yml_path = os.path.join(dp, file_stem + '.yml')
+                config_dict = u.load_yaml_config(yml_path) if os.path.exists(yml_path) else {}
+                config = DashboardConfig(name=file_stem, **config_dict)
+                dashboards_by_name[file_stem] = DashboardDefinition(file_stem, filepath, config)
+                
+        logger.log_activity_time("loading files for dashboards", start)
+        return dashboards_by_name