squirrels 0.5.0b3__py3-none-any.whl → 0.6.0.post0__py3-none-any.whl

squirrels/_command_line.py

@@ -1,4 +1,6 @@
 from argparse import ArgumentParser, _SubParsersAction
+from pathlib import Path
+from update_checker import UpdateChecker
 import sys, asyncio, traceback, io, subprocess
 
 sys.path.append('.')
@@ -9,22 +11,35 @@ from ._initializer import Initializer
 from ._package_loader import PackageLoaderIO
 from ._project import SquirrelsProject
 from . import _constants as c, _utils as u
+from ._compile_prompts import prompt_compile_options
 
 
 def _run_duckdb_cli(project: SquirrelsProject, ui: bool):
-    _, target_init_path = u._read_duckdb_init_sql()
-    init_args = f"-init {target_init_path}" if target_init_path else ""
+    init_sql = u._read_duckdb_init_sql(datalake_db_path=project._vdl_catalog_db_path)
+    
+    target_init_path = None
+    if init_sql:
+        target_init_path = Path(c.TARGET_FOLDER, c.DUCKDB_INIT_FILE)
+        target_init_path.parent.mkdir(parents=True, exist_ok=True)
+        target_init_path.write_text(init_sql)
+    
+    init_args = ["-init", str(target_init_path)] if target_init_path else []
     command = ['duckdb']
     if ui:
         command.append('-ui')
     if init_args:
-        command.extend(init_args.split())
-    command.extend(['-readonly', project._duckdb_venv_path])
-    print(f'Running command: {" ".join(command)}')
+        command.extend(init_args)
+    
+    print("Starting DuckDB CLI with command:")
+    print(f"$ {' '.join(command)}")
+    print()
+    print("To exit the DuckDB CLI, enter '.exit'")
+    print()
+    
     try:
         subprocess.run(command, check=True)
     except FileNotFoundError:
-        print("DuckDB CLI not found. Please install it from: https://duckdb.org/docs/installation/")
+        print("DuckDB CLI not found. Please install it from: https://duckdb.org/install")
     except subprocess.CalledProcessError:
         pass # ignore errors that occured on duckdb shell commands
 
@@ -33,32 +48,45 @@ def main():
     """
     Main entry point for the squirrels command line utilities.
     """
-    def with_help(parser: ArgumentParser):
-        parser.add_argument('-h', '--help', action="help", help="Show this help message and exit")
-        return parser
-
-    parser = with_help(ArgumentParser(description="Command line utilities from the squirrels python package", add_help=False))
+    # Create a parent parser with common logging options
+    parent_parser = ArgumentParser(add_help=False)
+    parent_parser.add_argument('-h', '--help', action="help", help="Show this help message and exit")
+    parent_parser.add_argument('--log-level', type=str, choices=["DEBUG", "INFO", "WARNING"], help='Level of logging to use. Default is from SQRL_LOGGING__LEVEL environment variable or INFO.')
+    parent_parser.add_argument('--log-format', type=str, choices=["text", "json"], help='Format of the log records. Default is from SQRL_LOGGING__FORMAT environment variable or text.')
+    parent_parser.add_argument(
+        '--log-to-file', action='store_true', 
+        help='Enable logging to file(s) in the "logs/" folder (or a custom folder) with rotation and retention policies. '
+            'Default is False unless overridden by the SQRL_LOGGING__TO_FILE environment variable.'
+    )
+    
+    parser = ArgumentParser(description="Command line utilities from the squirrels python package", add_help=False, parents=[parent_parser])
     parser.add_argument('-V', '--version', action='store_true', help='Show the version and exit')
-    parser.add_argument('--log-level', type=str, choices=["DEBUG", "INFO", "WARNING"], default="INFO", help='Level of logging to use')
-    parser.add_argument('--log-format', type=str, choices=["text", "json"], default="text", help='Format of the log records')
-    parser.add_argument('--log-file', type=str, default=c.LOGS_FILE, help=f'Name of log file to write to in the "logs/" folder. Default is {c.LOGS_FILE}. If name is empty, then file logging is disabled')
     subparsers = parser.add_subparsers(title='commands', dest='command')
 
     def add_subparser(subparsers: _SubParsersAction, cmd: str, help_text: str):
-        subparser = with_help(subparsers.add_parser(cmd, description=help_text, help=help_text, add_help=False))
+        subparser: ArgumentParser = subparsers.add_parser(cmd, description=help_text, help=help_text, add_help=False, parents=[parent_parser])
         return subparser
 
-    init_parser = add_subparser(subparsers, c.INIT_CMD, 'Create a new squirrels project')
+    new_parser = add_subparser(subparsers, c.NEW_CMD, 'Create a new squirrels project')
     
-    init_parser.add_argument('name', nargs='?', type=str, help='The name of the project folder to create. Ignored if --curr-dir is used')
-    init_parser.add_argument('--curr-dir', action='store_true', help='Create the project in the current directory')
-    init_parser.add_argument('--use-defaults', action='store_true', help='Use default values for unspecified options (except project folder name) instead of prompting for input')
+    new_parser.add_argument('name', nargs='?', type=str, help='The name of the project folder to create. Ignored if --curr-dir is used')
+    new_parser.add_argument('--curr-dir', action='store_true', help='Create the project in the current directory')
+    new_parser.add_argument('--use-defaults', action='store_true', help='Use default values for unspecified options (except project folder name) instead of prompting for input')
+    new_parser.add_argument('--connections', type=str, choices=c.CONF_FORMAT_CHOICES, help=f'Configure database connections as yaml (default) or python')
+    new_parser.add_argument('--parameters', type=str, choices=c.CONF_FORMAT_CHOICES, help=f'Configure parameters as python (default) or yaml')
+    new_parser.add_argument('--build', type=str, choices=c.FILE_TYPE_CHOICES, help='Create build model as sql (default) or python file')
+    new_parser.add_argument('--federate', type=str, choices=c.FILE_TYPE_CHOICES, help='Create federated model as sql (default) or python file')
+    new_parser.add_argument('--dashboard', type=str, choices=['y', 'n'], help=f'Include (y) or exclude (n, default) a sample dashboard file')
+    new_parser.add_argument('--admin-password', type=str, help='The password for the admin user. If --use-defaults is used, then a random password is generated')
+
+    init_parser = add_subparser(subparsers, c.INIT_CMD, 'Create a new squirrels project in the current directory (alias for "new --curr-dir")')
+    init_parser.add_argument('--use-defaults', action='store_true', help='Use default values for unspecified options instead of prompting for input')
     init_parser.add_argument('--connections', type=str, choices=c.CONF_FORMAT_CHOICES, help=f'Configure database connections as yaml (default) or python')
     init_parser.add_argument('--parameters', type=str, choices=c.CONF_FORMAT_CHOICES, help=f'Configure parameters as python (default) or yaml')
     init_parser.add_argument('--build', type=str, choices=c.FILE_TYPE_CHOICES, help='Create build model as sql (default) or python file')
     init_parser.add_argument('--federate', type=str, choices=c.FILE_TYPE_CHOICES, help='Create federated model as sql (default) or python file')
     init_parser.add_argument('--dashboard', type=str, choices=['y', 'n'], help=f'Include (y) or exclude (n, default) a sample dashboard file')
-    init_parser.add_argument('--admin-password', type=str, help='The password for the admin user. If --use-defaults is used, then a random password is generated')
+    init_parser.add_argument('--admin-password', type=str, help='The password for the admin user. By default, a random password is generated')
 
     def with_file_format_options(parser: ArgumentParser):
         help_text = "Create model as sql (default) or python file"
@@ -68,70 +96,93 @@ def main():
     get_file_help_text = "Get a sample file for the squirrels project. If the file name already exists, it will be suffixed with a timestamp."
     get_file_parser = add_subparser(subparsers, c.GET_FILE_CMD, get_file_help_text)
     get_file_subparsers = get_file_parser.add_subparsers(title='file_name', dest='file_name')
-    add_subparser(get_file_subparsers, c.DOTENV_FILE, f'Get sample {c.DOTENV_FILE} and {c.DOTENV_FILE}.example files')
-    add_subparser(get_file_subparsers, c.GITIGNORE_FILE, f'Get a sample {c.GITIGNORE_FILE} file')
-    manifest_parser = add_subparser(get_file_subparsers, c.MANIFEST_FILE, f'Get a sample {c.MANIFEST_FILE} file')
+    add_subparser(get_file_subparsers, c.DOTENV_FILE, f'Creates sample {c.DOTENV_FILE} and {c.DOTENV_FILE}.example files')
+    add_subparser(get_file_subparsers, c.GITIGNORE_FILE, f'Creates a sample {c.GITIGNORE_FILE} file')
+    manifest_parser = add_subparser(get_file_subparsers, c.MANIFEST_FILE, f'Creates a sample {c.MANIFEST_FILE} file')
     manifest_parser.add_argument("--no-connections", action='store_true', help=f'Exclude the connections section')
     manifest_parser.add_argument("--parameters", action='store_true', help=f'Include the parameters section')
     manifest_parser.add_argument("--dashboards", action='store_true', help=f'Include the dashboards section')
-    add_subparser(get_file_subparsers, c.USER_FILE, f'Get a sample {c.USER_FILE} file')
-    add_subparser(get_file_subparsers, c.CONNECTIONS_FILE, f'Get a sample {c.CONNECTIONS_FILE} file')
-    add_subparser(get_file_subparsers, c.PARAMETERS_FILE, f'Get a sample {c.PARAMETERS_FILE} file')
-    add_subparser(get_file_subparsers, c.CONTEXT_FILE, f'Get a sample {c.CONTEXT_FILE} file')
-    add_subparser(get_file_subparsers, c.MACROS_FILE, f'Get a sample {c.MACROS_FILE} file')
-    add_subparser(get_file_subparsers, c.SOURCES_FILE, f'Get a sample {c.SOURCES_FILE} file')
-    with_file_format_options(add_subparser(get_file_subparsers, c.BUILD_FILE_STEM, f'Get a sample build model file'))
-    add_subparser(get_file_subparsers, c.DBVIEW_FILE_STEM, f'Get a sample dbview model file')
-    with_file_format_options(add_subparser(get_file_subparsers, c.FEDERATE_FILE_STEM, f'Get a sample federate model file'))
-    add_subparser(get_file_subparsers, c.DASHBOARD_FILE_STEM, f'Get a sample dashboard file')
-    add_subparser(get_file_subparsers, c.EXPENSES_DB, f'Get the sample SQLite database on expenses')
-    add_subparser(get_file_subparsers, c.WEATHER_DB, f'Get the sample SQLite database on weather')
-    add_subparser(get_file_subparsers, c.SEED_CATEGORY_FILE_STEM, f'Get the sample seed files for categories lookup')
-    add_subparser(get_file_subparsers, c.SEED_SUBCATEGORY_FILE_STEM, f'Get the sample seed files for subcategories lookup')
+    add_subparser(get_file_subparsers, c.USER_FILE, f'Creates a sample {c.USER_FILE} file')
+    add_subparser(get_file_subparsers, c.CONNECTIONS_FILE, f'Creates a sample {c.CONNECTIONS_FILE} file')
+    add_subparser(get_file_subparsers, c.PARAMETERS_FILE, f'Creates a sample {c.PARAMETERS_FILE} file')
+    add_subparser(get_file_subparsers, c.CONTEXT_FILE, f'Creates a sample {c.CONTEXT_FILE} file')
+    add_subparser(get_file_subparsers, c.MACROS_FILE, f'Creates a sample {c.MACROS_FILE} file')
+    add_subparser(get_file_subparsers, c.SOURCES_FILE, f'Creates a sample {c.SOURCES_FILE} file')
+    with_file_format_options(add_subparser(get_file_subparsers, c.BUILD_FILE_STEM, f'Creates a sample build model file'))
+    add_subparser(get_file_subparsers, c.DBVIEW_FILE_STEM, f'Creates a sample dbview model file')
+    with_file_format_options(add_subparser(get_file_subparsers, c.FEDERATE_FILE_STEM, f'Creates a sample federate model file'))
+    add_subparser(get_file_subparsers, c.DASHBOARD_FILE_STEM, f'Creates a sample dashboard file')
+    add_subparser(get_file_subparsers, c.EXPENSES_DB, f'Creates the sample SQLite database on expenses')
+    add_subparser(get_file_subparsers, c.WEATHER_DB, f'Creates the sample SQLite database on weather')
+    add_subparser(get_file_subparsers, c.SEED_CATEGORY_FILE_STEM, f'Creates the sample seed files for categories lookup')
+    add_subparser(get_file_subparsers, c.SEED_SUBCATEGORY_FILE_STEM, f'Creates the sample seed files for subcategories lookup')
     
     deps_parser = add_subparser(subparsers, c.DEPS_CMD, f'Load all packages specified in {c.MANIFEST_FILE} (from git)')
 
     compile_parser = add_subparser(subparsers, c.COMPILE_CMD, 'Create rendered SQL files in the folder "./target/compile"')
-    compile_dataset_group = compile_parser.add_mutually_exclusive_group(required=True)
-    compile_dataset_group.add_argument('-d', '--dataset', type=str, help="Select dataset to use for dataset traits. Is required, unless using --all-datasets")
-    compile_dataset_group.add_argument('-D', '--all-datasets', action="store_true", help="Compile models for all datasets. Only required if --dataset is not specified")
+    compile_parser.add_argument('-y', '--yes', action='store_true', help='Disable prompts and assume the default for all settings not overridden by CLI options')
+    compile_parser.add_argument('-c', '--clear', action='store_true', help='Clear the "target/compile/" folder before compiling')
+    compile_scope_group = compile_parser.add_mutually_exclusive_group(required=False)
+    compile_scope_group.add_argument('--buildtime-only', action='store_true', help='Compile only buildtime models')
+    compile_scope_group.add_argument('--runtime-only', action='store_true', help='Compile only runtime models')
     compile_test_set_group = compile_parser.add_mutually_exclusive_group(required=False)
-    compile_test_set_group.add_argument('-t', '--test-set', type=str, help="The selection test set to use. If not specified, default selections are used, unless using --all-test-sets")
-    compile_test_set_group.add_argument('-T', '--all-test-sets', action="store_true", help="Compile models for all selection test sets")
-    compile_parser.add_argument('-s', '--select', type=str, help="Select single model to compile. If not specified, all models for the dataset are compiled. Ignored if using --all-datasets")
-    compile_parser.add_argument('-r', '--runquery', action='store_true', help='Runs all target models, and produce the results as csv files')
-
-    build_parser = add_subparser(subparsers, c.BUILD_CMD, 'Build the virtual data environment (with duckdb) for the project')
+    compile_test_set_group.add_argument('-t', '--test-set', type=str, help="The selection test set to use. If not specified, default selections are used (unless using --all-test-sets). Not applicable for buildtime models")
+    compile_test_set_group.add_argument('-T', '--all-test-sets', action="store_true", help="Compile models for all selection test sets. Not applicable for buildtime models")
+    compile_parser.add_argument('-s', '--select', type=str, help="Select single model to compile. If not specified, all models are compiled")
+    compile_parser.add_argument('-r', '--runquery', action='store_true', help='Run runtime models and write CSV outputs too. Does not apply to buildtime models')
+    
+    build_parser = add_subparser(subparsers, c.BUILD_CMD, 'Build the Virtual Data Lake (VDL) for the project')
     build_parser.add_argument('-f', '--full-refresh', action='store_true', help='Drop all tables before building')
     build_parser.add_argument('-s', '--select', type=str, help="Select one static model to build. If not specified, all models are built")
-    build_parser.add_argument('--stage', type=str, help='If the venv file is in use, stage the duckdb file to replace the venv later')
 
     duckdb_parser = add_subparser(subparsers, c.DUCKDB_CMD, 'Run the duckdb command line tool')
     duckdb_parser.add_argument('--ui', action='store_true', help='Run the duckdb local UI')
 
     run_parser = add_subparser(subparsers, c.RUN_CMD, 'Run the API server')
-    run_parser.add_argument('--build', action='store_true', help='Build the virtual data environment (with duckdb) first before running the API server')
+    run_parser.add_argument('--build', action='store_true', help='Build the VDL first (without full refresh) before running the API server')
     run_parser.add_argument('--no-cache', action='store_true', help='Do not cache any api results')
     run_parser.add_argument('--host', type=str, default='127.0.0.1', help="The host to run on")
-    run_parser.add_argument('--port', type=int, default=4465, help="The port to run on")
+    run_parser.add_argument('--port', type=int, default=8000, help="The port to run on")
+    run_parser.add_argument(
+        '--forwarded-allow-ips', type=str, default=None, 
+        help="Comma-separated list of IP Addresses, IP Networks, or literals (e.g. UNIX Socket path) to trust with proxy headers. "
+            "Defaults to the FORWARDED_ALLOW_IPS environment variable if available, or '127.0.0.1'. The literal '*' means trust everything."
+    )
 
     args, _ = parser.parse_known_args()
     
+    update_checker_result = UpdateChecker().check("squirrels", __version__)
+    if update_checker_result:
+        current_version = update_checker_result.running_version
+        latest_version = update_checker_result.available_version
+        print("\033[1;33mA newer version of Squirrels is available!\033[0m")
+        print(f"{current_version} -> \033[1;32m{latest_version}\033[0m (released on {update_checker_result.release_date.date()})")
+        print()
+        print("See \033[4mhttps://docs.pysquirrels.com/releases\033[0m for release notes.")
+        print()
+
     if args.version:
-        print(__version__)
-    elif args.command == c.INIT_CMD:
+        print(f"Squirrels version: \033[1;32m{__version__}\033[0m")
+    elif args.command == c.NEW_CMD:
         Initializer(project_name=args.name, use_curr_dir=args.curr_dir).init_project(args)
+    elif args.command == c.INIT_CMD:
+        Initializer(project_name=None, use_curr_dir=True).init_project(args)
     elif args.command == c.GET_FILE_CMD:
         Initializer().get_file(args)
     elif args.command is None:
         print(f'Command is missing. Enter "squirrels -h" for help.')
     else:
-        project = SquirrelsProject(log_level=args.log_level, log_format=args.log_format, log_file=args.log_file)
+        project = SquirrelsProject(
+            load_dotenv_globally=True, 
+            log_level=args.log_level, 
+            log_format=args.log_format, 
+            log_to_file=args.log_to_file
+        )
         try:
             if args.command == c.DEPS_CMD:
                 PackageLoaderIO.load_packages(project._logger, project._manifest_cfg, reload=True)
             elif args.command == c.BUILD_CMD:
-                task = project.build(full_refresh=args.full_refresh, select=args.select, stage_file=args.stage)
+                task = project.build(full_refresh=args.full_refresh, select=args.select)
                 asyncio.run(task)
                 print()
             elif args.command == c.DUCKDB_CMD:
@@ -143,11 +194,32 @@ def main():
                 server = ApiServer(args.no_cache, project)
                 server.run(args)
             elif args.command == c.COMPILE_CMD:
-                task = project.compile(
-                    dataset=args.dataset, do_all_datasets=args.all_datasets, selected_model=args.select, test_set=args.test_set, 
-                    do_all_test_sets=args.all_test_sets, runquery=args.runquery
-                )
-                asyncio.run(task)
+                # Derive final options with optional interactive prompts (unless --yes is provided)
+                buildtime_only = args.buildtime_only
+                runtime_only = args.runtime_only
+                test_set = args.test_set
+                do_all_test_sets = args.all_test_sets
+                selected_model = args.select
+
+                try:
+                    if not args.yes:
+                        buildtime_only, runtime_only, test_set, do_all_test_sets, selected_model = prompt_compile_options(
+                            project,
+                            buildtime_only=buildtime_only,
+                            runtime_only=runtime_only,
+                            test_set=test_set,
+                            do_all_test_sets=do_all_test_sets,
+                            selected_model=selected_model,
+                        )
+
+                    task = project.compile(
+                        selected_model=selected_model, test_set=test_set, do_all_test_sets=do_all_test_sets, runquery=args.runquery,
+                        clear=args.clear, buildtime_only=buildtime_only, runtime_only=runtime_only
+                    )
+                    asyncio.run(task)
+                
+                except KeyError:
+                    pass
             else:
                 print(f'Error: No such command "{args.command}". Enter "squirrels -h" for help.')
 

squirrels/_compile_prompts.py

@@ -0,0 +1,147 @@
+from typing import Tuple, Optional
+import inquirer
+
+from ._project import SquirrelsProject
+from . import _constants as c, _models as m
+
+
+def prompt_compile_options(
+    project: SquirrelsProject,
+    *,
+    buildtime_only: bool,
+    runtime_only: bool,
+    test_set: Optional[str],
+    do_all_test_sets: bool,
+    selected_model: Optional[str],
+) -> Tuple[bool, bool, Optional[str], bool, Optional[str]]:
+    """
+    Interactive prompts to derive compile options when --yes is not provided.
+
+    Returns updated values for:
+    (buildtime_only, runtime_only, test_set, do_all_test_sets, selected_model)
+    """
+    # 1) Scope prompt (skip if explicit flags provided)
+    if not buildtime_only and not runtime_only:
+        scope_answer = inquirer.prompt([
+            inquirer.List(
+                'scope',
+                message='Select scope',
+                choices=[
+                    ('All models', 'all'),
+                    ('Buildtime models only', 'buildtime'),
+                    ('Runtime models only', 'runtime'),
+                ],
+                default='all',
+            )
+        ]) or {}
+        scope_val = scope_answer['scope']
+        if scope_val == 'buildtime':
+            buildtime_only, runtime_only = True, False
+        elif scope_val == 'runtime':
+            buildtime_only, runtime_only = False, True
+        else:
+            buildtime_only, runtime_only = False, False
+
+    # 2) Runtime test set prompt (only if runtime is included and -t/-T not provided)
+    runtime_included, buildtime_included = not buildtime_only, not runtime_only
+    if runtime_included and not test_set and not do_all_test_sets:
+        test_mode_answer = inquirer.prompt([
+            inquirer.List(
+                'ts_mode',
+                message='Runtime selections for parameters, configurables, and user attributes?',
+                choices=[
+                    ('Use default selections', 'default'),
+                    ('Use a custom test set', 'custom'),
+                    ('Use all test sets (including default selections)', 'all'),
+                ],
+                default='default',
+            )
+        ]) or {}
+        ts_mode = test_mode_answer['ts_mode']
+        if ts_mode == 'all':
+            do_all_test_sets = True
+        elif ts_mode == 'custom':
+            # Build available test set list
+            ts_names = list(project._manifest_cfg.selection_test_sets.keys())
+            ts_answer = inquirer.prompt([
+                inquirer.List(
+                    'test_set',
+                    message='Pick a selection test set',
+                    choices=ts_names if ts_names else [c.DEFAULT_TEST_SET_NAME],
+                    default=c.DEFAULT_TEST_SET_NAME,
+                )
+            ]) or {}
+            test_set = ts_answer['test_set']
+
+    # 3) Model selection prompt (only if -s not provided)
+    if not selected_model:
+        # Ask whether to compile all or a specific model
+        model_mode_answer = inquirer.prompt([
+            inquirer.List(
+                'model_mode',
+                message='Compile all models in scope or just a specific model?',
+                choices=[
+                    ('All models', 'all'),
+                    ('Select a specific model', 'one'),
+                ],
+                default='all',
+            )
+        ]) or {}
+        model_mode = model_mode_answer['model_mode']
+
+        if model_mode == 'one':
+            # Build list of runtime query models (dbviews + federates) and build models if included
+            models_dict = project._get_models_dict(always_python_df=False)
+
+            valid_model_types: list[m.ModelType] = []
+            if runtime_included:
+                valid_model_types.extend([m.ModelType.DBVIEW, m.ModelType.FEDERATE])
+            if buildtime_included:
+                valid_model_types.append(m.ModelType.BUILD)
+
+            runtime_names = sorted([
+                (f"({model.model_type.value}) {name}", name) for name, model in models_dict.items()
+                if isinstance(model, m.QueryModel) and model.model_type in valid_model_types
+            ])
+            if runtime_names:
+                model_answer = inquirer.prompt([
+                    inquirer.List(
+                        'selected_model',
+                        message='Pick a runtime model to compile',
+                        choices=runtime_names,
+                    )
+                ]) or {}
+                selected_model = model_answer['selected_model']
+
+    # Tips and equivalent command without prompts
+    def _maybe_quote(val: str) -> str:
+        return f'"{val}"' if any(ch.isspace() for ch in val) else val
+
+    parts = ["sqrl", "compile", "-y"]
+    if buildtime_only:
+        parts.append("--buildtime-only")
+    elif runtime_only:
+        parts.append("--runtime-only")
+    if do_all_test_sets:
+        parts.append("-T")
+    elif test_set:
+        parts.extend(["-t", _maybe_quote(test_set)])
+    if selected_model:
+        parts.extend(["-s", _maybe_quote(selected_model)])
+
+    # Pretty tips block
+    tips_header = " Compile tips "
+    border = "=" * 80
+    print(border)
+    print(tips_header)
+    print("-" * len(border))
+    print("Equivalent command (no prompts):")
+    print(f"  $ {' '.join(parts)}")
+    print()
+    print("You can also:")
+    print("  - Add -c/--clear to clear the 'target/compile/' folder before compiling")
+    print("  - Add -r/--runquery to generate CSVs for runtime models")
+    print(border)
+    print()
+
+    return buildtime_only, runtime_only, test_set, do_all_test_sets, selected_model

squirrels/_connection_set.py

@@ -1,10 +1,10 @@
 from typing import Any
 from dataclasses import dataclass, field
 from sqlalchemy import Engine
-import time, polars as pl
+import time, polars as pl, duckdb
 
 from . import _utils as u, _constants as c, _py_module as pm
-from ._arguments._init_time_args import ConnectionsArgs
+from ._arguments.init_time_args import ConnectionsArgs
 from ._manifest import ManifestConfig, ConnectionProperties, ConnectionTypeEnum
 
 
@@ -38,6 +38,13 @@ class ConnectionSet:
             elif isinstance(conn, ConnectionProperties) and conn.type == ConnectionTypeEnum.SQLALCHEMY:
                 with conn.engine.connect() as connection:
                     df = pl.read_database(query, connection, execute_options={"parameters": placeholders})
+            elif isinstance(conn, ConnectionProperties) and conn.type == ConnectionTypeEnum.DUCKDB:
+                path = conn.uri # .replace("duckdb://", "")
+                dd_conn = duckdb.connect(path)
+                try:
+                    df = dd_conn.sql(query, params=placeholders).pl()
+                finally:
+                    dd_conn.close()
             else:
                 df = pl.read_database(query, conn, execute_options={"parameters": placeholders}) # type: ignore
             return df
@@ -61,17 +68,9 @@ class ConnectionSet:
 class ConnectionSetIO:
 
     @classmethod
-    def load_conn_py_args(cls, logger: u.Logger, base_path: str, env_vars: dict[str, str], manifest_cfg: ManifestConfig) -> ConnectionsArgs:
-        start = time.time()
-        
-        proj_vars = manifest_cfg.project_variables.model_dump()
-        conn_args = ConnectionsArgs(base_path, proj_vars, env_vars)
-        
-        logger.log_activity_time("setting up arguments for connections.py", start)
-        return conn_args
-
-    @classmethod
-    def load_from_file(cls, logger: u.Logger, base_path: str, manifest_cfg: ManifestConfig, conn_args: ConnectionsArgs) -> ConnectionSet:
+    def load_from_file(
+        cls, logger: u.Logger, project_path: str, manifest_cfg: ManifestConfig, conn_args: ConnectionsArgs
+    ) -> ConnectionSet:
         """
         Takes the DB connection engines from both the squirrels.yml and connections.py files and merges them
         into a single ConnectionSet
@@ -83,9 +82,11 @@ class ConnectionSetIO:
         connections: dict[str, ConnectionProperties | Any] = {}
         
         for config in manifest_cfg.connections.values():
-            connections[config.name] = ConnectionProperties(label=config.label, type=config.type, uri=config.uri)
+            connections[config.name] = ConnectionProperties(
+                label=config.label, type=config.type, uri=config.uri, sa_create_engine_args=config.sa_create_engine_args
+            )
 
-        pm.run_pyconfig_main(base_path, c.CONNECTIONS_FILE, {"connections": connections, "sqrl": conn_args})
+        pm.run_pyconfig_main(project_path, c.CONNECTIONS_FILE, {"connections": connections, "sqrl": conn_args})
 
         conn_set = ConnectionSet(connections)
 

squirrels/_constants.py

@@ -1,5 +1,6 @@
 # Squirrels CLI commands
-INIT_CMD = 'new'
+NEW_CMD = 'new'
+INIT_CMD = 'init'
 GET_FILE_CMD = 'get-file'
 DEPS_CMD = 'deps'
 BUILD_CMD = 'build'
@@ -13,12 +14,19 @@ SQRL_SECRET_ADMIN_PASSWORD = 'SQRL_SECRET__ADMIN_PASSWORD'
 
 SQRL_AUTH_DB_FILE_PATH = 'SQRL_AUTH__DB_FILE_PATH'
 SQRL_AUTH_TOKEN_EXPIRE_MINUTES = 'SQRL_AUTH__TOKEN_EXPIRE_MINUTES'
+SQRL_AUTH_CREDENTIAL_ORIGINS = 'SQRL_AUTH__ALLOWED_ORIGINS_FOR_COOKIES'
+
+SQRL_PERMISSIONS_ELEVATED_ACCESS_LEVEL = 'SQRL_PERMISSIONS__ELEVATED_ACCESS_LEVEL'
 
 SQRL_PARAMETERS_CACHE_SIZE = 'SQRL_PARAMETERS__CACHE_SIZE'
 SQRL_PARAMETERS_CACHE_TTL_MINUTES = 'SQRL_PARAMETERS__CACHE_TTL_MINUTES'
+SQRL_PARAMETERS_DATASOURCE_REFRESH_MINUTES = 'SQRL_PARAMETERS__DATASOURCE_REFRESH_MINUTES'
 
 SQRL_DATASETS_CACHE_SIZE = 'SQRL_DATASETS__CACHE_SIZE'
 SQRL_DATASETS_CACHE_TTL_MINUTES = 'SQRL_DATASETS__CACHE_TTL_MINUTES'
+SQRL_DATASETS_MAX_ROWS_FOR_AI = 'SQRL_DATASETS__MAX_ROWS_FOR_AI'
+SQRL_DATASETS_MAX_ROWS_OUTPUT = 'SQRL_DATASETS__MAX_ROWS_OUTPUT'
+SQRL_DATASETS_SQL_TIMEOUT_SECONDS = 'SQRL_DATASETS__SQL_TIMEOUT_SECONDS'
 
 SQRL_DASHBOARDS_CACHE_SIZE = 'SQRL_DASHBOARDS__CACHE_SIZE'
 SQRL_DASHBOARDS_CACHE_TTL_MINUTES = 'SQRL_DASHBOARDS__CACHE_TTL_MINUTES'
@@ -26,11 +34,18 @@ SQRL_DASHBOARDS_CACHE_TTL_MINUTES = 'SQRL_DASHBOARDS__CACHE_TTL_MINUTES'
 SQRL_SEEDS_INFER_SCHEMA = 'SQRL_SEEDS__INFER_SCHEMA'
 SQRL_SEEDS_NA_VALUES = 'SQRL_SEEDS__NA_VALUES'
 
-SQRL_TEST_SETS_DEFAULT_NAME_USED = 'SQRL_TEST_SETS__DEFAULT_NAME_USED'
-
 SQRL_CONNECTIONS_DEFAULT_NAME_USED = 'SQRL_CONNECTIONS__DEFAULT_NAME_USED'
 
-SQRL_DUCKDB_VENV_DB_FILE_PATH = 'SQRL_DUCKDB_VENV__DB_FILE_PATH'
+SQRL_VDL_CATALOG_DB_PATH = 'SQRL_VDL__CATALOG_DB_PATH'
+SQRL_VDL_DATA_PATH = 'SQRL_VDL__DATA_PATH'
+
+SQRL_STUDIO_BASE_URL = 'SQRL_STUDIO__BASE_URL'
+
+SQRL_LOGGING_LEVEL = 'SQRL_LOGGING__LEVEL'
+SQRL_LOGGING_FORMAT = 'SQRL_LOGGING__FORMAT'
+SQRL_LOGGING_TO_FILE = 'SQRL_LOGGING__TO_FILE'
+SQRL_LOGGING_FILE_SIZE_MB = 'SQRL_LOGGING__FILE_SIZE_MB'
+SQRL_LOGGING_FILE_BACKUP_COUNT = 'SQRL_LOGGING__FILE_BACKUP_COUNT'
 
 # Folder/File names
 PACKAGE_DATA_FOLDER = '_package_data'
@@ -50,7 +65,9 @@ GITIGNORE_FILE = '.gitignore'
 LOGS_FOLDER = 'logs'
 LOGS_FILE = 'squirrels.log'
 
-DATABASE_FOLDER = 'assets'
+RESOURCES_FOLDER = 'resources'
+PUBLIC_FOLDER = 'public'
+LOGOS_FOLDER = 'logos'
 PACKAGES_FOLDER = 'sqrl_packages'
 
 MACROS_FOLDER = 'macros'
@@ -69,7 +86,6 @@ DASHBOARDS_FOLDER = 'dashboards'
 DASHBOARD_FILE_STEM = 'dashboard_example'
 
 PYCONFIGS_FOLDER = 'pyconfigs'
-AUTH_FILE = 'auth.py'
 CONNECTIONS_FILE = 'connections.py'
 CONTEXT_FILE = 'context.py'
 PARAMETERS_FILE = 'parameters.py'
@@ -77,17 +93,17 @@ USER_FILE = 'user.py'
 ADMIN_USERNAME = 'admin'
 
 TARGET_FOLDER = 'target'
-DB_FILE = 'auth.sqlite'
 COMPILE_FOLDER = 'compile'
-DUCKDB_VENV_FILE = 'venv.duckdb'
+COMPILE_BUILDTIME_FOLDER = 'buildtime'
+COMPILE_RUNTIME_FOLDER = 'runtime'
+DB_FILE = 'auth.sqlite'
 
 SEEDS_FOLDER = 'seeds'
 SEED_CATEGORY_FILE_STEM = 'seed_categories'
 SEED_SUBCATEGORY_FILE_STEM = 'seed_subcategories'
 
-# Selection cfg sections
-USER_ATTRIBUTES_SECTION = 'user_attributes'
-PARAMETERS_SECTION = 'parameters'
+# Test sets
+DEFAULT_TEST_SET_NAME = 'default'
 
 # Init Command Choices
 SQL_FILE_TYPE = 'sql'
@@ -103,6 +119,7 @@ CONF_FORMAT_CHOICES2 = [(PYTHON_FORMAT2, PYTHON_FORMAT), YML_FORMAT]
 
 EXPENSES_DB = 'expenses.db'
 WEATHER_DB = 'weather.db'
+GOOGLE_LOGO = 'google.ico'
 
 # Dashboard formats
 PNG = "png"
@@ -114,3 +131,11 @@ MAIN_FUNC = "main"
 # Regex
 DATE_REGEX = r"^\d{4}\-\d{2}\-\d{2}$"
 COLOR_REGEX = r"^#[0-9a-fA-F]{6}$"
+
+# OAuth2
+SUPPORTED_SCOPES = ['read']
+SUPPORTED_GRANT_TYPES = ['authorization_code', 'refresh_token']
+SUPPORTED_RESPONSE_TYPES = ['code']
+
+# Miscellaneous
+LATEST_API_VERSION_MOUNT_PATH = "/api/0"