squirrels 0.5.0b4__py3-none-any.whl → 0.5.1__py3-none-any.whl

squirrels/_command_line.py

@@ -1,4 +1,5 @@
 from argparse import ArgumentParser, _SubParsersAction
+from pathlib import Path
 import sys, asyncio, traceback, io, subprocess
 
 sys.path.append('.')
@@ -9,18 +10,31 @@ from ._initializer import Initializer
 from ._package_loader import PackageLoaderIO
 from ._project import SquirrelsProject
 from . import _constants as c, _utils as u
+from ._compile_prompts import prompt_compile_options
 
 
 def _run_duckdb_cli(project: SquirrelsProject, ui: bool):
-    _, target_init_path = u._read_duckdb_init_sql()
-    init_args = f"-init {target_init_path}" if target_init_path else ""
+    init_sql = u._read_duckdb_init_sql(datalake_db_path=project._datalake_db_path)
+    
+    target_init_path = None
+    if init_sql:
+        target_init_path = Path(c.TARGET_FOLDER, c.DUCKDB_INIT_FILE)
+        target_init_path.parent.mkdir(parents=True, exist_ok=True)
+        target_init_path.write_text(init_sql)
+    
+    init_args = ["-init", str(target_init_path)] if target_init_path else []
     command = ['duckdb']
     if ui:
         command.append('-ui')
     if init_args:
-        command.extend(init_args.split())
-    command.extend(['-readonly', project._duckdb_venv_path])
-    print(f'Running command: {" ".join(command)}')
+        command.extend(init_args)
+    
+    print("Starting DuckDB CLI with command:")
+    print(f"$ {' '.join(command)}")
+    print()
+    print("To exit the DuckDB CLI, enter '.exit'")
+    print()
+    
     try:
         subprocess.run(command, check=True)
     except FileNotFoundError:
@@ -33,19 +47,19 @@ def main():
     """
     Main entry point for the squirrels command line utilities.
     """
-    def with_help(parser: ArgumentParser):
-        parser.add_argument('-h', '--help', action="help", help="Show this help message and exit")
-        return parser
-
-    parser = with_help(ArgumentParser(description="Command line utilities from the squirrels python package", add_help=False))
+    # Create a parent parser with common logging options
+    parent_parser = ArgumentParser(add_help=False)
+    parent_parser.add_argument('-h', '--help', action="help", help="Show this help message and exit")
+    parent_parser.add_argument('--log-level', type=str, choices=["DEBUG", "INFO", "WARNING"], help='Level of logging to use. Default is from SQRL_LOGGING__LOG_LEVEL environment variable or INFO.')
+    parent_parser.add_argument('--log-format', type=str, choices=["text", "json"], help='Format of the log records. Default is from SQRL_LOGGING__LOG_FORMAT environment variable or text.')
+    parent_parser.add_argument('--log-to-file', action='store_true', help='Enable logging to file(s) in the "logs/" folder with rotation and retention policies.')
+    
+    parser = ArgumentParser(description="Command line utilities from the squirrels python package", add_help=False, parents=[parent_parser])
     parser.add_argument('-V', '--version', action='store_true', help='Show the version and exit')
-    parser.add_argument('--log-level', type=str, choices=["DEBUG", "INFO", "WARNING"], default="INFO", help='Level of logging to use')
-    parser.add_argument('--log-format', type=str, choices=["text", "json"], default="text", help='Format of the log records')
-    parser.add_argument('--log-file', type=str, default=c.LOGS_FILE, help=f'Name of log file to write to in the "logs/" folder. Default is {c.LOGS_FILE}. If name is empty, then file logging is disabled')
     subparsers = parser.add_subparsers(title='commands', dest='command')
 
     def add_subparser(subparsers: _SubParsersAction, cmd: str, help_text: str):
-        subparser = with_help(subparsers.add_parser(cmd, description=help_text, help=help_text, add_help=False))
+        subparser: ArgumentParser = subparsers.add_parser(cmd, description=help_text, help=help_text, add_help=False, parents=[parent_parser])
         return subparser
 
     init_parser = add_subparser(subparsers, c.INIT_CMD, 'Create a new squirrels project')
@@ -92,25 +106,26 @@ def main():
     deps_parser = add_subparser(subparsers, c.DEPS_CMD, f'Load all packages specified in {c.MANIFEST_FILE} (from git)')
 
     compile_parser = add_subparser(subparsers, c.COMPILE_CMD, 'Create rendered SQL files in the folder "./target/compile"')
-    compile_dataset_group = compile_parser.add_mutually_exclusive_group(required=True)
-    compile_dataset_group.add_argument('-d', '--dataset', type=str, help="Select dataset to use for dataset traits. Is required, unless using --all-datasets")
-    compile_dataset_group.add_argument('-D', '--all-datasets', action="store_true", help="Compile models for all datasets. Only required if --dataset is not specified")
+    compile_parser.add_argument('-y', '--yes', action='store_true', help='Disable prompts and assume the default for all settings not overridden by CLI options')
+    compile_parser.add_argument('-c', '--clear', action='store_true', help='Clear the "target/compile/" folder before compiling')
+    compile_scope_group = compile_parser.add_mutually_exclusive_group(required=False)
+    compile_scope_group.add_argument('--buildtime-only', action='store_true', help='Compile only buildtime models')
+    compile_scope_group.add_argument('--runtime-only', action='store_true', help='Compile only runtime models')
     compile_test_set_group = compile_parser.add_mutually_exclusive_group(required=False)
-    compile_test_set_group.add_argument('-t', '--test-set', type=str, help="The selection test set to use. If not specified, default selections are used, unless using --all-test-sets")
-    compile_test_set_group.add_argument('-T', '--all-test-sets', action="store_true", help="Compile models for all selection test sets")
-    compile_parser.add_argument('-s', '--select', type=str, help="Select single model to compile. If not specified, all models for the dataset are compiled. Ignored if using --all-datasets")
-    compile_parser.add_argument('-r', '--runquery', action='store_true', help='Runs all target models, and produce the results as csv files')
-
-    build_parser = add_subparser(subparsers, c.BUILD_CMD, 'Build the virtual data environment (with duckdb) for the project')
+    compile_test_set_group.add_argument('-t', '--test-set', type=str, help="The selection test set to use. If not specified, default selections are used (unless using --all-test-sets). Not applicable for buildtime models")
+    compile_test_set_group.add_argument('-T', '--all-test-sets', action="store_true", help="Compile models for all selection test sets. Not applicable for buildtime models")
+    compile_parser.add_argument('-s', '--select', type=str, help="Select single model to compile. If not specified, all models are compiled")
+    compile_parser.add_argument('-r', '--runquery', action='store_true', help='Run runtime models and write CSV outputs too. Does not apply to buildtime models')
+    
+    build_parser = add_subparser(subparsers, c.BUILD_CMD, 'Build the Virtual Data Lake (VDL) for the project')
     build_parser.add_argument('-f', '--full-refresh', action='store_true', help='Drop all tables before building')
     build_parser.add_argument('-s', '--select', type=str, help="Select one static model to build. If not specified, all models are built")
-    build_parser.add_argument('--stage', type=str, help='If the venv file is in use, stage the duckdb file to replace the venv later')
 
     duckdb_parser = add_subparser(subparsers, c.DUCKDB_CMD, 'Run the duckdb command line tool')
     duckdb_parser.add_argument('--ui', action='store_true', help='Run the duckdb local UI')
 
     run_parser = add_subparser(subparsers, c.RUN_CMD, 'Run the API server')
-    run_parser.add_argument('--build', action='store_true', help='Build the virtual data environment (with duckdb) first before running the API server')
+    run_parser.add_argument('--build', action='store_true', help='Build the VDL first (without full refresh) before running the API server')
     run_parser.add_argument('--no-cache', action='store_true', help='Do not cache any api results')
     run_parser.add_argument('--host', type=str, default='127.0.0.1', help="The host to run on")
     run_parser.add_argument('--port', type=int, default=4465, help="The port to run on")
@@ -126,12 +141,12 @@ def main():
     elif args.command is None:
         print(f'Command is missing. Enter "squirrels -h" for help.')
     else:
-        project = SquirrelsProject(log_level=args.log_level, log_format=args.log_format, log_file=args.log_file)
+        project = SquirrelsProject(load_dotenv_globally=True, log_level=args.log_level, log_format=args.log_format, log_to_file=args.log_to_file)
         try:
             if args.command == c.DEPS_CMD:
                 PackageLoaderIO.load_packages(project._logger, project._manifest_cfg, reload=True)
             elif args.command == c.BUILD_CMD:
-                task = project.build(full_refresh=args.full_refresh, select=args.select, stage_file=args.stage)
+                task = project.build(full_refresh=args.full_refresh, select=args.select)
                 asyncio.run(task)
                 print()
             elif args.command == c.DUCKDB_CMD:
@@ -143,11 +158,32 @@ def main():
                 server = ApiServer(args.no_cache, project)
                 server.run(args)
             elif args.command == c.COMPILE_CMD:
-                task = project.compile(
-                    dataset=args.dataset, do_all_datasets=args.all_datasets, selected_model=args.select, test_set=args.test_set, 
-                    do_all_test_sets=args.all_test_sets, runquery=args.runquery
-                )
-                asyncio.run(task)
+                # Derive final options with optional interactive prompts (unless --yes is provided)
+                buildtime_only = args.buildtime_only
+                runtime_only = args.runtime_only
+                test_set = args.test_set
+                do_all_test_sets = args.all_test_sets
+                selected_model = args.select
+
+                try:
+                    if not args.yes:
+                        buildtime_only, runtime_only, test_set, do_all_test_sets, selected_model = prompt_compile_options(
+                            project,
+                            buildtime_only=buildtime_only,
+                            runtime_only=runtime_only,
+                            test_set=test_set,
+                            do_all_test_sets=do_all_test_sets,
+                            selected_model=selected_model,
+                        )
+
+                    task = project.compile(
+                        selected_model=selected_model, test_set=test_set, do_all_test_sets=do_all_test_sets, runquery=args.runquery,
+                        clear=args.clear, buildtime_only=buildtime_only, runtime_only=runtime_only
+                    )
+                    asyncio.run(task)
+                
+                except KeyError:
+                    pass
             else:
                 print(f'Error: No such command "{args.command}". Enter "squirrels -h" for help.')
 

squirrels/_compile_prompts.py

@@ -0,0 +1,147 @@
+from typing import Tuple, Optional
+import inquirer
+
+from ._project import SquirrelsProject
+from . import _constants as c, _models as m
+
+
+def prompt_compile_options(
+    project: SquirrelsProject,
+    *,
+    buildtime_only: bool,
+    runtime_only: bool,
+    test_set: Optional[str],
+    do_all_test_sets: bool,
+    selected_model: Optional[str],
+) -> Tuple[bool, bool, Optional[str], bool, Optional[str]]:
+    """
+    Interactive prompts to derive compile options when --yes is not provided.
+
+    Returns updated values for:
+    (buildtime_only, runtime_only, test_set, do_all_test_sets, selected_model)
+    """
+    # 1) Scope prompt (skip if explicit flags provided)
+    if not buildtime_only and not runtime_only:
+        scope_answer = inquirer.prompt([
+            inquirer.List(
+                'scope',
+                message='Select scope',
+                choices=[
+                    ('All models', 'all'),
+                    ('Buildtime models only', 'buildtime'),
+                    ('Runtime models only', 'runtime'),
+                ],
+                default='all',
+            )
+        ]) or {}
+        scope_val = scope_answer['scope']
+        if scope_val == 'buildtime':
+            buildtime_only, runtime_only = True, False
+        elif scope_val == 'runtime':
+            buildtime_only, runtime_only = False, True
+        else:
+            buildtime_only, runtime_only = False, False
+
+    # 2) Runtime test set prompt (only if runtime is included and -t/-T not provided)
+    runtime_included, buildtime_included = not buildtime_only, not runtime_only
+    if runtime_included and not test_set and not do_all_test_sets:
+        test_mode_answer = inquirer.prompt([
+            inquirer.List(
+                'ts_mode',
+                message='Runtime selections for parameters, configurables, and user attributes?',
+                choices=[
+                    ('Use default selections', 'default'),
+                    ('Use a custom test set', 'custom'),
+                    ('Use all test sets (including default selections)', 'all'),
+                ],
+                default='default',
+            )
+        ]) or {}
+        ts_mode = test_mode_answer['ts_mode']
+        if ts_mode == 'all':
+            do_all_test_sets = True
+        elif ts_mode == 'custom':
+            # Build available test set list
+            ts_names = list(project._manifest_cfg.selection_test_sets.keys())
+            ts_answer = inquirer.prompt([
+                inquirer.List(
+                    'test_set',
+                    message='Pick a selection test set',
+                    choices=ts_names if ts_names else [c.DEFAULT_TEST_SET_NAME],
+                    default=c.DEFAULT_TEST_SET_NAME,
+                )
+            ]) or {}
+            test_set = ts_answer['test_set']
+
+    # 3) Model selection prompt (only if -s not provided)
+    if not selected_model:
+        # Ask whether to compile all or a specific model
+        model_mode_answer = inquirer.prompt([
+            inquirer.List(
+                'model_mode',
+                message='Compile all models in scope or just a specific model?',
+                choices=[
+                    ('All models', 'all'),
+                    ('Select a specific model', 'one'),
+                ],
+                default='all',
+            )
+        ]) or {}
+        model_mode = model_mode_answer['model_mode']
+
+        if model_mode == 'one':
+            # Build list of runtime query models (dbviews + federates) and build models if included
+            models_dict = project._get_models_dict(always_python_df=False)
+
+            valid_model_types: list[m.ModelType] = []
+            if runtime_included:
+                valid_model_types.extend([m.ModelType.DBVIEW, m.ModelType.FEDERATE])
+            if buildtime_included:
+                valid_model_types.append(m.ModelType.BUILD)
+
+            runtime_names = sorted([
+                (f"({model.model_type.value}) {name}", name) for name, model in models_dict.items()
+                if isinstance(model, m.QueryModel) and model.model_type in valid_model_types
+            ])
+            if runtime_names:
+                model_answer = inquirer.prompt([
+                    inquirer.List(
+                        'selected_model',
+                        message='Pick a runtime model to compile',
+                        choices=runtime_names,
+                    )
+                ]) or {}
+                selected_model = model_answer['selected_model']
+
+    # Tips and equivalent command without prompts
+    def _maybe_quote(val: str) -> str:
+        return f'"{val}"' if any(ch.isspace() for ch in val) else val
+
+    parts = ["sqrl", "compile", "-y"]
+    if buildtime_only:
+        parts.append("--buildtime-only")
+    elif runtime_only:
+        parts.append("--runtime-only")
+    if do_all_test_sets:
+        parts.append("-T")
+    elif test_set:
+        parts.extend(["-t", _maybe_quote(test_set)])
+    if selected_model:
+        parts.extend(["-s", _maybe_quote(selected_model)])
+
+    # Pretty tips block
+    tips_header = " Compile tips "
+    border = "=" * 80
+    print(border)
+    print(tips_header)
+    print("-" * len(border))
+    print("Equivalent command (no prompts):")
+    print(f"  $ {' '.join(parts)}")
+    print()
+    print("You can also:")
+    print("  - Add -c/--clear to clear the 'target/compile/' folder before compiling")
+    print("  - Add -r/--runquery to generate CSVs for runtime models")
+    print(border)
+    print()
+
+    return buildtime_only, runtime_only, test_set, do_all_test_sets, selected_model

squirrels/_connection_set.py

@@ -1,7 +1,7 @@
 from typing import Any
 from dataclasses import dataclass, field
 from sqlalchemy import Engine
-import time, polars as pl
+import time, polars as pl, duckdb
 
 from . import _utils as u, _constants as c, _py_module as pm
 from ._arguments.init_time_args import ConnectionsArgs
@@ -38,6 +38,13 @@ class ConnectionSet:
             elif isinstance(conn, ConnectionProperties) and conn.type == ConnectionTypeEnum.SQLALCHEMY:
                 with conn.engine.connect() as connection:
                     df = pl.read_database(query, connection, execute_options={"parameters": placeholders})
+            elif isinstance(conn, ConnectionProperties) and conn.type == ConnectionTypeEnum.DUCKDB:
+                path = conn.uri # .replace("duckdb://", "")
+                dd_conn = duckdb.connect(path)
+                try:
+                    df = dd_conn.sql(query, params=placeholders).pl()
+                finally:
+                    dd_conn.close()
             else:
                 df = pl.read_database(query, conn, execute_options={"parameters": placeholders}) # type: ignore
             return df
@@ -83,7 +90,9 @@ class ConnectionSetIO:
         connections: dict[str, ConnectionProperties | Any] = {}
         
         for config in manifest_cfg.connections.values():
-            connections[config.name] = ConnectionProperties(label=config.label, type=config.type, uri=config.uri)
+            connections[config.name] = ConnectionProperties(
+                label=config.label, type=config.type, uri=config.uri, sa_create_engine_args=config.sa_create_engine_args
+            )
 
         pm.run_pyconfig_main(base_path, c.CONNECTIONS_FILE, {"connections": connections, "sqrl": conn_args})
 

squirrels/_constants.py

@@ -17,21 +17,32 @@ SQRL_AUTH_CREDENTIAL_ORIGINS = 'SQRL_AUTH__ALLOWED_ORIGINS_FOR_COOKIES'
 
 SQRL_PARAMETERS_CACHE_SIZE = 'SQRL_PARAMETERS__CACHE_SIZE'
 SQRL_PARAMETERS_CACHE_TTL_MINUTES = 'SQRL_PARAMETERS__CACHE_TTL_MINUTES'
+SQRL_PARAMETERS_DATASOURCE_REFRESH_MINUTES = 'SQRL_PARAMETERS__DATASOURCE_REFRESH_MINUTES'
 
 SQRL_DATASETS_CACHE_SIZE = 'SQRL_DATASETS__CACHE_SIZE'
 SQRL_DATASETS_CACHE_TTL_MINUTES = 'SQRL_DATASETS__CACHE_TTL_MINUTES'
+SQRL_DATASETS_MAX_ROWS_FOR_AI = 'SQRL_DATASETS__MAX_ROWS_FOR_AI'
 
 SQRL_DASHBOARDS_CACHE_SIZE = 'SQRL_DASHBOARDS__CACHE_SIZE'
 SQRL_DASHBOARDS_CACHE_TTL_MINUTES = 'SQRL_DASHBOARDS__CACHE_TTL_MINUTES'
 
+SQRL_PERMISSIONS_ELEVATED_ACCESS_LEVEL = 'SQRL_PERMISSIONS__ELEVATED_ACCESS_LEVEL'
+
 SQRL_SEEDS_INFER_SCHEMA = 'SQRL_SEEDS__INFER_SCHEMA'
 SQRL_SEEDS_NA_VALUES = 'SQRL_SEEDS__NA_VALUES'
 
-SQRL_TEST_SETS_DEFAULT_NAME_USED = 'SQRL_TEST_SETS__DEFAULT_NAME_USED'
-
 SQRL_CONNECTIONS_DEFAULT_NAME_USED = 'SQRL_CONNECTIONS__DEFAULT_NAME_USED'
 
-SQRL_DUCKDB_VENV_DB_FILE_PATH = 'SQRL_DUCKDB_VENV__DB_FILE_PATH'
+SQRL_VDL_CATALOG_DB_PATH = 'SQRL_VDL__CATALOG_DB_PATH'
+SQRL_VDL_DATA_PATH = 'SQRL_VDL__DATA_PATH'
+
+SQRL_STUDIO_BASE_URL = 'SQRL_STUDIO__BASE_URL'
+
+SQRL_LOGGING_LOG_LEVEL = 'SQRL_LOGGING__LOG_LEVEL'
+SQRL_LOGGING_LOG_FORMAT = 'SQRL_LOGGING__LOG_FORMAT'
+SQRL_LOGGING_LOG_TO_FILE = 'SQRL_LOGGING__LOG_TO_FILE'
+SQRL_LOGGING_LOG_FILE_SIZE_MB = 'SQRL_LOGGING__LOG_FILE_SIZE_MB'
+SQRL_LOGGING_LOG_FILE_BACKUP_COUNT = 'SQRL_LOGGING__LOG_FILE_BACKUP_COUNT'
 
 # Folder/File names
 PACKAGE_DATA_FOLDER = '_package_data'
@@ -53,6 +64,7 @@ LOGS_FILE = 'squirrels.log'
 
 DATABASE_FOLDER = 'assets'
 PACKAGES_FOLDER = 'sqrl_packages'
+PUBLIC_FOLDER = 'public'
 
 MACROS_FOLDER = 'macros'
 MACROS_FILE = 'macros_example.sql'
@@ -78,17 +90,19 @@ USER_FILE = 'user.py'
 ADMIN_USERNAME = 'admin'
 
 TARGET_FOLDER = 'target'
-DB_FILE = 'auth.sqlite'
 COMPILE_FOLDER = 'compile'
-DUCKDB_VENV_FILE = 'venv.duckdb'
+COMPILE_BUILDTIME_FOLDER = 'buildtime'
+COMPILE_RUNTIME_FOLDER = 'runtime'
+DB_FILE = 'auth.sqlite'
+DEFAULT_VDL_CATALOG_DB_PATH = 'ducklake:{project_path}/target/vdl_catalog.duckdb'
+DEFAULT_VDL_DATA_PATH = '{project_path}/target/vdl_data/'
 
 SEEDS_FOLDER = 'seeds'
 SEED_CATEGORY_FILE_STEM = 'seed_categories'
 SEED_SUBCATEGORY_FILE_STEM = 'seed_subcategories'
 
-# Selection cfg sections
-USER_ATTRIBUTES_SECTION = 'user_attributes'
-PARAMETERS_SECTION = 'parameters'
+# Test sets
+DEFAULT_TEST_SET_NAME = 'default'
 
 # Init Command Choices
 SQL_FILE_TYPE = 'sql'