squirrels 0.5.0b2__py3-none-any.whl → 0.5.0b4__py3-none-any.whl

squirrels/_initializer.py

@@ -1,10 +1,11 @@
 from typing import Optional
 from datetime import datetime
+from pathlib import Path
 import inquirer, os, shutil, secrets
 
 from . import _constants as c, _utils as u
 
-base_proj_dir = u.Path(os.path.dirname(__file__), c.PACKAGE_DATA_FOLDER, c.BASE_PROJECT_FOLDER)
+base_proj_dir = Path(os.path.dirname(__file__), c.PACKAGE_DATA_FOLDER, c.BASE_PROJECT_FOLDER)
 
 TMP_FOLDER = "tmp"
 
@@ -14,22 +15,23 @@ class Initializer:
         self.project_name = project_name if not use_curr_dir else None
         self.use_curr_dir = use_curr_dir
 
-    def _path_exists(self, filepath: u.Path) -> bool:
+    def _path_exists(self, filepath: Path) -> bool:
         return os.path.exists(filepath)
     
-    def _files_have_same_content(self, file1: u.Path, file2: u.Path) -> bool:
+    def _files_have_same_content(self, file1: Path, file2: Path) -> bool:
         with open(file1, 'rb') as f1, open(file2, 'rb') as f2:
             return f1.read() == f2.read()
     
-    def _add_timestamp_to_filename(self, path: u.Path) -> u.Path:
+    def _add_timestamp_to_filename(self, path: Path) -> Path:
         timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
         new_filename = f"{path.stem}_{timestamp}{path.suffix}"
         return path.with_name(new_filename)
     
-    def _copy_file(self, filepath: u.Path, *, src_folder: str = ""):
-        src_path = u.Path(base_proj_dir, src_folder, filepath)
+    def _copy_file(self, filepath: Path, *, src_folder: str = "", src_file: Path | None = None):
+        src_file = src_file if src_file is not None else filepath
+        src_path = Path(base_proj_dir, src_folder, src_file)
         
-        filepath2 = u.Path(self.project_name, filepath) if self.project_name else filepath
+        filepath2 = Path(self.project_name, filepath) if self.project_name else filepath
         dest_dir = os.path.dirname(filepath2)
         if dest_dir != "":
             os.makedirs(dest_dir, exist_ok=True)
@@ -51,38 +53,38 @@ class Initializer:
             shutil.copy(src_path, filepath2)
 
     def _copy_macros_file(self, filepath: str):
-        self._copy_file(u.Path(c.MACROS_FOLDER, filepath))
+        self._copy_file(Path(c.MACROS_FOLDER, filepath))
 
     def _copy_models_file(self, filepath: str):
-        self._copy_file(u.Path(c.MODELS_FOLDER, filepath))
+        self._copy_file(Path(c.MODELS_FOLDER, filepath))
 
     def _copy_build_file(self, filepath: str):
-        self._copy_file(u.Path(c.MODELS_FOLDER, c.BUILDS_FOLDER, filepath))
+        self._copy_file(Path(c.MODELS_FOLDER, c.BUILDS_FOLDER, filepath))
 
     def _copy_dbview_file(self, filepath: str):
-        self._copy_file(u.Path(c.MODELS_FOLDER, c.DBVIEWS_FOLDER, filepath))
+        self._copy_file(Path(c.MODELS_FOLDER, c.DBVIEWS_FOLDER, filepath))
 
     def _copy_federate_file(self, filepath: str):
-        self._copy_file(u.Path(c.MODELS_FOLDER, c.FEDERATES_FOLDER, filepath))
+        self._copy_file(Path(c.MODELS_FOLDER, c.FEDERATES_FOLDER, filepath))
 
     def _copy_database_file(self, filepath: str):
-        self._copy_file(u.Path(c.DATABASE_FOLDER, filepath))
+        self._copy_file(Path(c.DATABASE_FOLDER, filepath))
     
     def _copy_pyconfig_file(self, filepath: str):
-        self._copy_file(u.Path(c.PYCONFIGS_FOLDER, filepath))
+        self._copy_file(Path(c.PYCONFIGS_FOLDER, filepath))
     
     def _copy_seed_file(self, filepath: str):
-        self._copy_file(u.Path(c.SEEDS_FOLDER, filepath))
+        self._copy_file(Path(c.SEEDS_FOLDER, filepath))
     
     def _copy_dashboard_file(self, filepath: str):
-        self._copy_file(u.Path(c.DASHBOARDS_FOLDER, filepath))
+        self._copy_file(Path(c.DASHBOARDS_FOLDER, filepath))
 
     def _create_manifest_file(self, has_connections: bool, has_parameters: bool):
         def get_content(file_name: Optional[str]) -> str:
             if file_name is None:
                 return ""
             
-            yaml_path = u.Path(base_proj_dir, file_name)
+            yaml_path = Path(base_proj_dir, file_name)
             return yaml_path.read_text()
         
         file_name_dict = {
@@ -93,10 +95,10 @@ class Initializer:
         
         manifest_template = get_content(c.MANIFEST_JINJA_FILE)
         manifest_content = u.render_string(manifest_template, **substitutions)
-        output_path = u.Path(base_proj_dir, TMP_FOLDER, c.MANIFEST_FILE)
+        output_path = Path(base_proj_dir, TMP_FOLDER, c.MANIFEST_FILE)
         output_path.write_text(manifest_content)
         
-        self._copy_file(u.Path(c.MANIFEST_FILE), src_folder=TMP_FOLDER)
+        self._copy_file(Path(c.MANIFEST_FILE), src_folder=TMP_FOLDER)
     
     def _copy_dotenv_files(self, admin_password: str | None = None):
         substitutions = {
@@ -104,14 +106,17 @@ class Initializer:
             "random_admin_password": admin_password if admin_password else secrets.token_urlsafe(8),
         }
 
-        dotenv_path = u.Path(base_proj_dir, c.DOTENV_FILE)
+        dotenv_path = Path(base_proj_dir, c.DOTENV_FILE)
         contents = u.render_string(dotenv_path.read_text(), **substitutions)
 
-        output_path = u.Path(base_proj_dir, TMP_FOLDER, c.DOTENV_FILE)
+        output_path = Path(base_proj_dir, TMP_FOLDER, c.DOTENV_FILE)
         output_path.write_text(contents)
 
-        self._copy_file(u.Path(c.DOTENV_FILE), src_folder=TMP_FOLDER)
-        self._copy_file(u.Path(c.DOTENV_FILE + ".example"))
+        self._copy_file(Path(c.DOTENV_FILE), src_folder=TMP_FOLDER)
+        self._copy_file(Path(c.DOTENV_FILE + ".example"))
+
+    def _copy_gitignore_file(self):
+        self._copy_file(Path(c.GITIGNORE_FILE), src_file=Path("gitignore"))
 
     def init_project(self, args):
         options = ["connections", "parameters", "build", "federate", "dashboard", "admin_password"]
@@ -127,7 +132,7 @@ class Initializer:
             self.project_name = answers['project_name']
 
         answers = { x: getattr(args, x) for x in options }
-        if DASHBOARD in answers:
+        if answers.get(DASHBOARD) is not None:
             answers[DASHBOARD] = (answers[DASHBOARD] == 'y') # convert 'y' or 'n' to boolean
 
         if not args.use_defaults:
@@ -221,7 +226,7 @@ class Initializer:
         self._copy_dotenv_files(admin_password)
         self._create_manifest_file(connections_use_yaml, parameters_use_yaml)
         
-        self._copy_file(u.Path(c.GITIGNORE_FILE))
+        self._copy_gitignore_file()
         
         if connections_use_py:
             self._copy_pyconfig_file(c.CONNECTIONS_FILE)
@@ -272,7 +277,7 @@ class Initializer:
             print(f"You may also run `sqrl get-file {c.GITIGNORE_FILE}` to add a sample {c.GITIGNORE_FILE} file to your project.")
             print()
         elif args.file_name == c.GITIGNORE_FILE:
-            self._copy_file(u.Path(c.GITIGNORE_FILE))
+            self._copy_gitignore_file()
         elif args.file_name == c.MANIFEST_FILE:
             self._create_manifest_file(not args.no_connections, args.parameters)
         elif args.file_name in (c.USER_FILE, c.CONNECTIONS_FILE, c.PARAMETERS_FILE, c.CONTEXT_FILE):

squirrels/_manifest.py

@@ -39,7 +39,7 @@ class _ConfigWithNameBaseModel(BaseModel):
     name: str
 
 
-class ConnectionType(Enum):
+class ConnectionTypeEnum(Enum):
     SQLALCHEMY = "sqlalchemy"
     CONNECTORX = "connectorx"
     ADBC = "adbc"
@@ -54,7 +54,7 @@ class ConnectionProperties(BaseModel):
         uri: The URI for the connection
     """
     label: str | None = None
-    type: ConnectionType = Field(default=ConnectionType.SQLALCHEMY)
+    type: ConnectionTypeEnum = Field(default=ConnectionTypeEnum.SQLALCHEMY)
     uri: str
     sa_create_engine_args: dict[str, Any] = Field(default_factory=dict)
 
@@ -64,14 +64,14 @@ class ConnectionProperties(BaseModel):
         Creates and caches a SQLAlchemy engine if the connection type is sqlalchemy.
         Returns None for other connection types.
         """
-        if self.type == ConnectionType.SQLALCHEMY:
+        if self.type == ConnectionTypeEnum.SQLALCHEMY:
             return create_engine(self.uri, **self.sa_create_engine_args)
         else:
             raise ValueError(f'Connection type "{self.type}" does not support engine property')
 
     @cached_property
     def dialect(self) -> str:
-        if self.type == ConnectionType.SQLALCHEMY:
+        if self.type == ConnectionTypeEnum.SQLALCHEMY:
             dialect = self.engine.dialect.name
         else:
             url = urlparse(self.uri)
@@ -83,7 +83,7 @@ class ConnectionProperties(BaseModel):
     
     @cached_property
     def attach_uri_for_duckdb(self) -> str | None:
-        if self.type == ConnectionType.SQLALCHEMY:
+        if self.type == ConnectionTypeEnum.SQLALCHEMY:
             url = self.engine.url
             host = url.host
             port = url.port

squirrels/_model_builder.py

@@ -68,7 +68,7 @@ class ModelBuilder:
         # If the development copy is already in use, a concurrent build is not allowed
         duckdb_dev_lock_path = u.Path(self._duckdb_venv_path + ".dev.lock")
         if duckdb_dev_lock_path.exists():
-            raise InvalidInputError(60, "An existing build process is already running and a concurrent build is not allowed")
+            raise InvalidInputError(409, "Concurrent build not allowed", "An existing build process is already running and a concurrent build is not allowed")
         duckdb_dev_lock_path.touch(exist_ok=False)
         
         # Ensure the lock file is deleted even if an exception is raised

squirrels/_model_configs.py

@@ -57,7 +57,7 @@ class BuildModelConfig(QueryModelConfig):
         else:
             raise ValueError(f"Invalid materialization: {self.materialization}")
         
-        create_prefix = f"CREATE OR REPLACE {materialization} {model_name} AS\n"
+        create_prefix = f"CREATE OR REPLACE {materialization} {model_name} AS\n\n"
         return create_prefix + select_query
 
 
@@ -70,5 +70,5 @@ class FederateModelConfig(QueryModelConfig):
 
     def get_sql_for_create(self, model_name: str, select_query: str) -> str:
         materialization = "TABLE" if self.eager else "VIEW"
-        create_prefix = f"CREATE {materialization} {model_name} AS\n"
+        create_prefix = f"CREATE {materialization} {model_name} AS\n\n"
         return create_prefix + select_query

squirrels/_model_queries.py

@@ -3,7 +3,7 @@ from dataclasses import dataclass, field
 from typing import Callable, Generic, TypeVar, Any
 import polars as pl, pandas as pd
 
-from .arguments.run_time_args import BuildModelArgs
+from ._arguments.run_time_args import BuildModelArgs
 from ._model_configs import ModelConfig
 
 

squirrels/_models.py

@@ -7,9 +7,10 @@ from pathlib import Path
 import asyncio, os, re, time, duckdb, sqlglot
 import polars as pl, pandas as pd, networkx as nx
 
-from . import _constants as c, _utils as u, _py_module as pm, _model_queries as mq, _model_configs as mc, _sources as src, _api_response_models as arm
+from . import _constants as c, _utils as u, _py_module as pm, _model_queries as mq, _model_configs as mc, _sources as src
+from ._schemas import response_models as rm
 from ._exceptions import FileExecutionError, InvalidInputError
-from .arguments.run_time_args import ContextArgs, ModelArgs, BuildModelArgs
+from ._arguments.run_time_args import ContextArgs, ModelArgs, BuildModelArgs
 from ._auth import BaseUser
 from ._connection_set import ConnectionsArgs, ConnectionSet, ConnectionProperties
 from ._manifest import DatasetConfig
@@ -173,7 +174,7 @@ class StaticModel(DataModel):
         try:
             return self._load_duckdb_view_to_python_df(local_conn, use_venv=True)
         except Exception as e:
-            raise InvalidInputError(61, f'Model "{self.name}" depends on static data models that cannot be found.')
+            raise InvalidInputError(409, f'Dependent data model not found.', f'Model "{self.name}" depends on static data models that cannot be found. Trying building the virtual data environment first.')
         finally:
             local_conn.close()
     
@@ -253,8 +254,12 @@ class SourceModel(StaticModel):
             connection_props = self.conn_set.get_connection(conn_name)
             if isinstance(connection_props, ConnectionProperties):
                 dialect = connection_props.dialect
+                attach_uri = connection_props.attach_uri_for_duckdb
             else:
-                raise u.ConfigurationError(f'Unable to use connection "{conn_name}" for source "{self.name}"')
+                raise u.ConfigurationError(f'Unable to use connection "{conn_name}" for source "{self.name}". Connection "{conn_name}" must be a ConnectionProperties object')
+            
+            if attach_uri is None:
+                raise u.ConfigurationError(f'Loading to duckdb is not supported for source "{self.name}" since its connection "{conn_name}" uses an unsupported dialect')
             
             result = u.run_duckdb_stmt(self.logger, local_conn, f"FROM (SHOW DATABASES) WHERE database_name = 'db_{conn_name}'").fetchone()
             if result is None:
@@ -351,7 +356,7 @@ class QueryModel(DataModel):
     def _get_compile_sql_model_args_from_ctx_args(
         self, ctx: dict[str, Any], ctx_args: ContextArgs
     ) -> dict[str, Any]:
-        is_placeholder = lambda placeholder: placeholder in ctx_args.placeholders
+        is_placeholder = lambda placeholder: placeholder in ctx_args._placeholders_copy
         kwargs = {
             "proj_vars": ctx_args.proj_vars, "env_vars": ctx_args.env_vars, "user": ctx_args.user, "prms": ctx_args.prms, 
             "traits": ctx_args.traits, "ctx": ctx, "is_placeholder": is_placeholder, "set_placeholder": ctx_args.set_placeholder,
@@ -424,6 +429,11 @@ class QueryModel(DataModel):
             dependent_model_names.add(self.name)
             for dep_model in self.upstreams.values():
                 dep_model.retrieve_dependent_query_models(dependent_model_names)
+    
+    def _log_sql_to_run(self, sql: str, placeholders: dict[str, Any]) -> None:
+        log_msg = f"SQL to run for model '{self.name}':\n{sql}"
+        log_msg += f"\n\n(with placeholders: {placeholders})"
+        self.logger.info(log_msg)
 
 
 @dataclass
@@ -460,6 +470,7 @@ class DbviewModel(QueryModel):
             return "{{ source(\"" + source_name + "\") }}"
         
         kwargs["source"] = source
+        kwargs["ref"] = source
         return kwargs
 
     def _get_duckdb_query(self, read_dialect: str, query: str) -> str:
@@ -476,17 +487,15 @@ class DbviewModel(QueryModel):
         connection_props = self.conn_set.get_connection(connection_name)
         
         if self.model_config.translate_to_duckdb and isinstance(connection_props, ConnectionProperties):
-            source_func = lambda source_name: "venv." + source_name
-            macros = {
-                "source": source_func, "ref": source_func
+            macros = { 
+                "source": lambda source_name: "venv." + source_name 
             }
             compiled_query2 = self._get_compiled_sql_query_str(compiled_query_str, macros)
             compiled_query_str = self._get_duckdb_query(connection_props.dialect, compiled_query2)
             is_duckdb = True
         else:
-            source_func = lambda source_name: self.sources[source_name].get_table()
-            macros = {
-                "source": source_func, "ref": source_func
+            macros = { 
+                "source": lambda source_name: self.sources[source_name].get_table() 
             }
             compiled_query_str = self._get_compiled_sql_query_str(compiled_query_str, macros)
             is_duckdb = False
@@ -520,19 +529,21 @@ class DbviewModel(QueryModel):
                 if is_duckdb:
                     local_conn = conn.cursor()
                     try:
-                        self.logger.info(f"Running duckdb query: {query}")
+                        self.logger.info(f"Running dbview '{self.name}' on duckdb")
                         return local_conn.sql(query, params=placeholders).pl()
                     except duckdb.CatalogException as e:
-                        raise InvalidInputError(61, f'Model "{self.name}" depends on static data models that cannot be found.')
+                        raise InvalidInputError(409, f'Dependent data model not found.', f'Model "{self.name}" depends on static data models that cannot be found. Trying building the virtual data environment first.')
                     except Exception as e:
                         raise RuntimeError(e)
                     finally:
                         local_conn.close()
                 else:
-                    return self._run_sql_query_on_connection(connection_name, query, placeholders)
+                    self.logger.info(f"Running dbview '{self.name}' on connection: {connection_name}")
+                    return self.conn_set.run_sql_query_from_conn_name(query, connection_name, placeholders)
             except RuntimeError as e:
                 raise FileExecutionError(f'Failed to run dbview sql model "{self.name}"', e)
         
+        self._log_sql_to_run(query, placeholders)
         result = await asyncio.to_thread(run_sql_query_on_connection, is_duckdb, query, placeholders)
         self.result = result.lazy()
 
@@ -582,7 +593,7 @@ class FederateModel(QueryModel):
         connections = self.conn_set.get_connections_as_dict()
         
         def run_external_sql(connection_name: str, sql_query: str) -> pl.DataFrame:
-            return self._run_sql_query_on_connection(connection_name, sql_query, ctx_args.placeholders)
+            return self._run_sql_query_on_connection(connection_name, sql_query, ctx_args._placeholders_copy)
         
         conn_args = ConnectionsArgs(ctx_args.project_path, ctx_args.proj_vars, ctx_args.env_vars)
         build_model_args = BuildModelArgs(conn_args, connections, dependencies, self._ref_for_python, run_external_sql)
@@ -638,17 +649,19 @@ class FederateModel(QueryModel):
             query = compiled_query.query
 
             def create_table(local_conn: duckdb.DuckDBPyConnection):
-                placeholer_exists = lambda key: re.search(r"\$" + key + r"(?!\w)", query)
-                existing_placeholders = {key: value for key, value in placeholders.items() if placeholer_exists(key)}
+                # DuckDB doesn't support specifying named parameters that are not used in the query, so filtering them out
+                placeholder_exists = lambda key: re.search(r"\$" + key + r"(?!\w)", query)
+                existing_placeholders = {key: value for key, value in placeholders.items() if placeholder_exists(key)}
 
                 create_query = self.model_config.get_sql_for_create(self.name, query)
+                self._log_sql_to_run(create_query, existing_placeholders)
                 try:
                     return local_conn.execute(create_query, existing_placeholders)
                 except duckdb.CatalogException as e:
-                    raise InvalidInputError(61, f'Model "{self.name}" depends on static data models that cannot be found.')
+                    raise InvalidInputError(409, f'Dependent data model not found.', f'Model "{self.name}" depends on static data models that cannot be found. Trying building the virtual data environment first.')
                 except Exception as e:
                     if self.name == "__fake_target":
-                        raise InvalidInputError(204, f"Failed to run provided SQL query")
+                        raise InvalidInputError(400, "Invalid SQL query", f"Failed to run provided SQL query")
                     else:
                         raise FileExecutionError(f'Failed to run federate sql model "{self.name}"', e) from e
             
@@ -774,7 +787,7 @@ class BuildModel(StaticModel, QueryModel):
             create_query = self.model_config.get_sql_for_build(self.name, query)
             local_conn = conn.cursor()
             try:
-                return u.run_duckdb_stmt(self.logger, local_conn, create_query)
+                return u.run_duckdb_stmt(self.logger, local_conn, create_query, model_name=self.name)
             except Exception as e:
                 raise FileExecutionError(f'Failed to build static sql model "{self.name}"', e) from e
             finally:
@@ -922,7 +935,7 @@ class DAG:
 
         self._compile_models(context, ctx_args, recurse)
         
-        self.placeholders = ctx_args.placeholders
+        self.placeholders = ctx_args._placeholders_copy
         if runquery:
             await self._run_models()
         
@@ -948,24 +961,24 @@ class DAG:
         
         return G
     
-    def get_all_data_models(self) -> list[arm.DataModelItem]:
+    def get_all_data_models(self) -> list[rm.DataModelItem]:
         data_models = []
         for model_name, model in self.models_dict.items():
             is_queryable = model.is_queryable
-            data_model = arm.DataModelItem(name=model_name, model_type=model.model_type.value, config=model.model_config, is_queryable=is_queryable)
+            data_model = rm.DataModelItem(name=model_name, model_type=model.model_type.value, config=model.model_config, is_queryable=is_queryable)
             data_models.append(data_model)
         return data_models
     
-    def get_all_model_lineage(self) -> list[arm.LineageRelation]:
+    def get_all_model_lineage(self) -> list[rm.LineageRelation]:
         model_lineage = []
         for model_name, model in self.models_dict.items():
             if not isinstance(model, QueryModel):
                 continue
             for dep_model_name in model.model_config.depends_on:
                 edge_type = "buildtime" if isinstance(model, BuildModel) else "runtime"
-                source_model = arm.LineageNode(name=dep_model_name, type="model")
-                target_model = arm.LineageNode(name=model_name, type="model")
-                model_lineage.append(arm.LineageRelation(type=edge_type, source=source_model, target=target_model))
+                source_model = rm.LineageNode(name=dep_model_name, type="model")
+                target_model = rm.LineageNode(name=model_name, type="model")
+                model_lineage.append(rm.LineageRelation(type=edge_type, source=source_model, target=target_model))
         return model_lineage
 
 

squirrels/{package_data → _package_data}/base_project/.env

@@ -10,6 +10,7 @@ SQRL_SECRET__ADMIN_PASSWORD="{{ random_admin_password }}"
 # (default values are shown below)
 SQRL_AUTH__DB_FILE_PATH="target/auth.sqlite"
 SQRL_AUTH__TOKEN_EXPIRE_MINUTES="30"
+SQRL_AUTH__ALLOWED_ORIGINS_FOR_COOKIES="https://squirrels-analytics.github.io"
 
 SQRL_PARAMETERS__CACHE_SIZE="1024"
 SQRL_PARAMETERS__CACHE_TTL_MINUTES="60"

squirrels/{package_data → _package_data}/base_project/.env.example

@@ -10,6 +10,7 @@ SQRL_SECRET__ADMIN_PASSWORD=""
 # (default values are shown below)
 SQRL_AUTH__DB_FILE_PATH="target/auth.sqlite"
 SQRL_AUTH__TOKEN_EXPIRE_MINUTES="30"
+SQRL_AUTH__ALLOWED_ORIGINS_FOR_COOKIES="https://squirrels-analytics.github.io"
 
 SQRL_PARAMETERS__CACHE_SIZE="1024"
 SQRL_PARAMETERS__CACHE_TTL_MINUTES="60"

squirrels/{package_data → _package_data}/base_project/dashboards/dashboard_example.py

@@ -1,10 +1,10 @@
-from squirrels import DashboardArgs, dashboards as d
+from squirrels import arguments as args, dashboards as d
 from matplotlib import pyplot as plt, figure as f, axes as a
 
 
-async def main(sqrl: DashboardArgs) -> d.PngDashboard:
-    spending_by_month_df = await sqrl.dataset("federate_dataset_example", fixed_parameters={"group_by": "g4"})
-    spending_by_subcategory_df = await sqrl.dataset("federate_dataset_example", fixed_parameters={"group_by": "g3"})
+async def main(sqrl: args.DashboardArgs) -> d.PngDashboard:
+    spending_by_month_df = await sqrl.dataset("federate_dataset_example", fixed_parameters={"group_by": "month"})
+    spending_by_subcategory_df = await sqrl.dataset("federate_dataset_example", fixed_parameters={"group_by": "subcat"})
 
     # Create a figure with two subplots
     fig, (ax0, ax1) = plt.subplots(2, 1, figsize=(8, 8), height_ratios=(1, 2))

squirrels/{package_data → _package_data}/base_project/dashboards/dashboard_example.yml

@@ -14,9 +14,9 @@ depends_on:
   - name: dataset_example_month
     dataset: federate_dataset_example
     fixed_parameters:
-      - group_by: g4 (Month)
+      - group_by: month (Month)
   
   - name: dataset_example_subcategory
     dataset: federate_dataset_example
     fixed_parameters:
-      - group_by: g3 (Subcategory)
+      - group_by: subcat (Subcategory)

squirrels/_package_data/base_project/macros/macros_example.sql

@@ -0,0 +1,17 @@
+{%- macro date_and_amount_filters(use_from_range) -%}
+    {%- if use_from_range -%}
+
+    date >= {{ ctx.start_date_from_range | quote }}
+    AND date <= {{ ctx.end_date_from_range | quote }}
+    AND amount >= {{ ctx.min_amount_from_range }}
+    AND amount <= {{ ctx.max_amount_from_range }}
+
+    {%- else -%}
+
+    date >= {{ ctx.start_date | quote }}
+    AND date <= {{ ctx.end_date | quote }}
+    AND amount >= {{ ctx.min_amount }}
+    AND amount <= {{ ctx.max_amount }}
+
+    {%- endif -%}
+{%- endmacro -%}

squirrels/{package_data → _package_data}/base_project/models/builds/build_example.py

@@ -1,8 +1,8 @@
-from squirrels import BuildModelArgs
+from squirrels import arguments as args
 import polars as pl, pandas as pd
 
 
-def main(sqrl: BuildModelArgs) -> pl.LazyFrame | pl.DataFrame | pd.DataFrame:
+def main(sqrl: args.BuildModelArgs) -> pl.LazyFrame | pl.DataFrame | pd.DataFrame:
     """
     Create a build model by joining/processing sources or other build models to form a new
     Python DataFrame (using polars LazyFrame, polars DataFrame, or pandas DataFrame).

squirrels/{package_data → _package_data}/base_project/models/builds/build_example.sql

@@ -1,4 +1,4 @@
-{# DuckDB dialect #}
+{#- DuckDB dialect -#}
 
 SELECT a.id,
     STRFTIME(a.date, '%Y-%m-%d') AS date,

squirrels/{package_data → _package_data}/base_project/models/dbviews/dbview_example.sql

@@ -1,4 +1,4 @@
-{# SQLite dialect (based on connection used) #}
+{#- SQLite dialect (based on connection used) -#}
 
 SELECT STRFTIME('%Y-%m', date) AS month
     , printf('%.2f', SUM(amount)) as total_amount

squirrels/_package_data/base_project/models/federates/federate_example.py

@@ -0,0 +1,41 @@
+from squirrels import arguments as args
+import polars as pl, pandas as pd
+
+
+def main(sqrl: args.ModelArgs) -> pl.LazyFrame | pl.DataFrame | pd.DataFrame:
+    """
+    Create federated models by joining/processing dependent models (sources, seeds, builds, dbviews, other federates, etc.) to
+    form a new Python DataFrame (using polars LazyFrame, polars DataFrame, or pandas DataFrame).
+    """
+    df = sqrl.ref("build_example")
+
+    df = df.filter(
+        (pl.col("date") >= sqrl.ctx["start_date_from_range"]) &
+        (pl.col("date") <= sqrl.ctx["end_date_from_range"]) &
+        (pl.col("amount") >= sqrl.ctx["min_amount_from_range"]) &
+        (pl.col("amount") <= sqrl.ctx["max_amount_from_range"])
+    )
+
+    if sqrl.ctx["has_categories"]:
+        categories: list[str] = sqrl.ctx["categories"]
+        df = df.filter(pl.col("category_id").is_in(categories))
+
+    if sqrl.ctx["has_subcategories"]:
+        subcategories: list[str] = sqrl.ctx["subcategories"]
+        df = df.filter(pl.col("subcategory_id").is_in(subcategories))
+
+    dimension_cols: list[str] = sqrl.ctx["group_by_cols"]
+    df = df.group_by(dimension_cols).agg(
+        pl.sum("amount").cast(pl.Decimal(precision=15, scale=2)).alias("total_amount")
+    )
+    df = df.rename(sqrl.ctx["rename_dict"])
+
+    order_by_cols: list[str] = sqrl.ctx["order_by_cols"]
+    df = df.select(*order_by_cols, "total_amount") \
+        .sort(order_by_cols, descending=True)
+
+    if "limit" in sqrl.ctx:
+        limit: int = sqrl.ctx["limit"]
+        df = df.limit(limit)
+
+    return df

squirrels/_package_data/base_project/models/federates/federate_example.sql

@@ -0,0 +1,25 @@
+{#- DuckDB dialect -#}
+
+SELECT {{ ctx.select_dim_cols | join }}
+    , CAST(SUM(amount) AS DECIMAL(15, 2)) as total_amount
+
+{# ref() can be used on a sources, seeds, builds, dbviews, or other federate models -#}
+FROM {{ ref("build_example") }} AS a
+
+WHERE {{ date_and_amount_filters(use_from_range=true) }}
+{%- if ctx.has_categories %} 
+    AND category_id IN ({{ ctx.categories | quote_and_join }}) 
+{%- endif %}
+{%- if ctx.has_subcategories %} 
+    AND subcategory_id IN ({{ ctx.subcategories | quote_and_join }}) 
+{%- endif %}
+
+GROUP BY {{ ctx.group_by_cols | join }}
+
+ORDER BY {{ ctx.order_by_cols_desc | join }}
+
+{%- if ctx.limit %}
+
+LIMIT {{ ctx.limit }}
+
+{%- endif %}

squirrels/{package_data → _package_data}/base_project/models/federates/federate_example.yml

@@ -11,7 +11,7 @@ eager: false               # optional - defaults to false. Only applies to SQL m
 columns:
   - name: date
     type: string
-    condition: parameter 'group_by' (Group By) is 'g0' (Transaction)
+    condition: parameter 'group_by' (Group By) is 'trans' (Transaction)
     description: The date of the transaction in 'YYYY-MM-DD' format, in descending order
     category: dimension
     depends_on:
@@ -19,7 +19,7 @@ columns:
   
   - name: description
     type: string
-    condition: parameter 'group_by' (Group By) is 'g0' (Transaction)
+    condition: parameter 'group_by' (Group By) is 'trans' (Transaction)
     description: The description of the transaction
     category: dimension
     depends_on:
@@ -27,7 +27,7 @@ columns:
   
   - name: day
     type: string
-    condition: parameter 'group_by' (Group By) is 'g1' (Day)
+    condition: parameter 'group_by' (Group By) is 'day' (Day)
     description: The day for which the amount is aggregated by, in descending order
     category: dimension
     depends_on:
@@ -35,7 +35,7 @@ columns:
   
   - name: month
     type: string
-    condition: parameter 'group_by' (Group By) is 'g4' (Month)
+    condition: parameter 'group_by' (Group By) is 'month' (Month)
     description: The month for which the amount is aggregated by, in descending order
     category: dimension
     depends_on:
@@ -43,7 +43,7 @@ columns:
 
   - name: category
     type: string
-    condition: parameter `group_by` (Group By) is `g0` (Transaction), `g2` (Category), or `g3` (Subcategory)
+    condition: parameter `group_by` (Group By) is `trans` (Transaction), `cat` (Category), or `subcat` (Subcategory)
     description: The category for which the amount is aggregated by
     category: dimension
     depends_on:
@@ -51,7 +51,7 @@ columns:
   
   - name: subcategory
     type: string
-    condition: parameter `group_by` (Group By) is `g0` (Transaction) or `g3` (Subcategory)
+    condition: parameter `group_by` (Group By) is `trans` (Transaction) or `subcat` (Subcategory)
     description: The subcategory for which the amount is aggregated by
     category: dimension
     depends_on: