squirrels 0.4.1__py3-none-any.whl → 0.5.0rc0__py3-none-any.whl

squirrels/package_data/base_project/.env

@@ -0,0 +1,30 @@
+# Custom environment variables
+SQLITE_URI="sqlite:///{project_path}/assets/expenses.db"
+
+# Secrets used by the Squirrels framework that are NOT SAFE TO INCLUDE IN VERSION CONTROL
+# Required if your project uses authentication. Otherwise, optional.
+SQRL_SECRET__KEY="{{ random_secret_key }}"
+SQRL_SECRET__ADMIN_PASSWORD="{{ random_admin_password }}"
+
+# Optional variables used by the Squirrels framework that are safe to include in version control if desired
+# (default values are shown below)
+SQRL_AUTH__DB_FILE_PATH="target/auth.sqlite"
+SQRL_AUTH__TOKEN_EXPIRE_MINUTES="30"
+
+SQRL_PARAMETERS__CACHE_SIZE="1024"
+SQRL_PARAMETERS__CACHE_TTL_MINUTES="60"
+
+SQRL_DATASETS__CACHE_SIZE="128"
+SQRL_DATASETS__CACHE_TTL_MINUTES="60"
+
+SQRL_DASHBOARDS__CACHE_SIZE="128"
+SQRL_DASHBOARDS__CACHE_TTL_MINUTES="60"
+
+SQRL_SEEDS__INFER_SCHEMA="true"
+SQRL_SEEDS__NA_VALUES=["NA"] # must be a JSON list
+
+SQRL_TEST_SETS__DEFAULT_NAME_USED="default"
+
+SQRL_CONNECTIONS__DEFAULT_NAME_USED="default"
+
+SQRL_DUCKDB_VENV__DB_FILE_PATH="target/venv.duckdb"

squirrels/package_data/base_project/.env.example

@@ -0,0 +1,30 @@
+# Custom environment variables
+SQLITE_URI="sqlite:///{project_path}/assets/expenses.db"
+
+# Secrets used by the Squirrels framework that are NOT SAFE TO INCLUDE IN VERSION CONTROL
+# Required if your project uses authentication. Otherwise, optional.
+SQRL_SECRET__KEY="" # a random 32 byte hex string - one way to generate this is by running "openssl rand -hex 32" in bash
+SQRL_SECRET__ADMIN_PASSWORD=""
+
+# Optional variables used by the Squirrels framework that are safe to include in version control if desired
+# (default values are shown below)
+SQRL_AUTH__DB_FILE_PATH="target/auth.sqlite"
+SQRL_AUTH__TOKEN_EXPIRE_MINUTES="30"
+
+SQRL_PARAMETERS__CACHE_SIZE="1024"
+SQRL_PARAMETERS__CACHE_TTL_MINUTES="60"
+
+SQRL_DATASETS__CACHE_SIZE="128"
+SQRL_DATASETS__CACHE_TTL_MINUTES="60"
+
+SQRL_DASHBOARDS__CACHE_SIZE="128"
+SQRL_DASHBOARDS__CACHE_TTL_MINUTES="60"
+
+SQRL_SEEDS__INFER_SCHEMA="true"
+SQRL_SEEDS__NA_VALUES=["NA"] # must be a JSON list
+
+SQRL_TEST_SETS__DEFAULT_NAME_USED="default"
+
+SQRL_CONNECTIONS__DEFAULT_NAME_USED="default"
+
+SQRL_DUCKDB_VENV__DB_FILE_PATH="target/venv.duckdb"

squirrels/package_data/base_project/.gitignore

@@ -1,12 +1,13 @@
 **/__pycache__/
 
 # common virtual environment names
-.env/
 .venv/
 venv/
 
 # squirrels files to ignore
-env.yml
+.env
+.env.local
+duckdb_init.sql
 logs/
 target/
 sqrl_packages/

squirrels/package_data/base_project/connections.yml

@@ -1,7 +1,15 @@
-## Uses SQLAlchemy URLs. More details here: https://docs.sqlalchemy.org/en/latest/core/engines.html
+## Connection URIs are usually in format "dialect://username:password@host:port/database" for database connections
+## However, subtle differences exist depending on the "type" specified. For example, sqlite URIs are slightly different.
+##   sqlalchemy:      sqlite:///relative/path/to/database.db
+##   connectorx/adbc: sqlite://relative/path/to/database.db  (adbc URI format matches connectorx thanks to polars integration)
+## Refer to specific documentation for supported databases by type (with URI examples):
+##   sqlalchemy: https://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls
+##   connectorx: https://sfu-db.github.io/connector-x/databases.html
+##   adbc:       https://arrow.apache.org/adbc/ (see connectorx documentation for URI examples)
 connections:
   - name: default
-    credential: null
-    url: {{ env_vars.sqlite_conn_str }}   ## using Jinja to substitute environment variable from env.yml
+    label: SQLite Expenses Database
+    type: sqlalchemy                      ## one of: sqlalchemy, connectorx, or adbc
+    uri: {{ env_vars.SQLITE_URI }}        ## using Jinja to substitute environment variables
 
 

squirrels/package_data/base_project/dashboards/dashboard_example.py

@@ -3,16 +3,8 @@ from matplotlib import pyplot as plt, figure as f, axes as a
 
 
 async def main(sqrl: DashboardArgs) -> d.PngDashboard:
-    """
-    Create a dashboard by retrieving datasets using "sqrl.dataset" method and transform the datasets to return as a PngDashboard or a HtmlDashboard.
-    - The PngDashboard constructor takes a single argument for either a matplotlib.figure.Figure or io.BytesIO/bytes of PNG data
-    - The HtmlDashboard constructor takes a single argument for a io.StringIO/string of HTML data
-
-    It is imperative to set the correct return type in the function signature for "main" above! It allows Squirrels to provide the correct format to 
-    the data catalog without having to run this function.
-    """
-    spending_by_month_df = await sqrl.dataset("dataset_example", fixed_parameters={"group_by": "g4"})
-    spending_by_subcategory_df = await sqrl.dataset("dataset_example", fixed_parameters={"group_by": "g3"})
+    spending_by_month_df = await sqrl.dataset("federate_dataset_example", fixed_parameters={"group_by": "g4"})
+    spending_by_subcategory_df = await sqrl.dataset("federate_dataset_example", fixed_parameters={"group_by": "g3"})
 
     # Create a figure with two subplots
     fig, (ax0, ax1) = plt.subplots(2, 1, figsize=(8, 8), height_ratios=(1, 2))
@@ -20,13 +12,23 @@ async def main(sqrl: DashboardArgs) -> d.PngDashboard:
     fig.tight_layout(pad=4, h_pad=6)
 
     # Create a bar chart of spending by month
-    spending_by_month_df.sort_values("month").plot(x="month", y="total_amount", ax=ax0)
+
+    # Convert to pandas and ensure total_amount is numeric
+    spending_by_month_pandas = spending_by_month_df.sort("month").to_pandas()
+    spending_by_month_pandas["total_amount"] = spending_by_month_pandas["total_amount"].astype(float)
+
+    spending_by_month_pandas.plot(x="month", y="total_amount", ax=ax0)
     ax0.set_title("Spending by Month")
 
     # Create a pie chart of spending by subcategory
-    df_by_subcategory = spending_by_subcategory_df.set_index("subcategory").sort_values("total_amount", ascending=False)
+
+    # Convert to pandas and ensure total_amount is numeric
+    subcategory_pandas = spending_by_subcategory_df.sort("total_amount", descending=True).to_pandas()
+    subcategory_pandas["total_amount"] = subcategory_pandas["total_amount"].astype(float)
+    subcategory_pandas.set_index("subcategory", inplace=True)
+    
     autopct = lambda pct: ('%.1f%%' % pct) if pct > 6 else ''
-    df_by_subcategory.plot(y="total_amount", kind='pie', ax=ax1, autopct=autopct, legend=False, ylabel="")
+    subcategory_pandas.plot(y="total_amount", kind='pie', ax=ax1, autopct=autopct, legend=False, ylabel="")
     ax1.set_title("Spending by Subcategory")
     
     return d.PngDashboard(fig)

squirrels/package_data/base_project/dashboards/dashboard_example.yml

@@ -0,0 +1,22 @@
+label: Dashboard Example
+
+description: This is an example dashboard
+
+scope: protected
+
+format: png
+
+parameters:
+  - date_range
+  - category
+
+depends_on:
+  - name: dataset_example_month
+    dataset: federate_dataset_example
+    fixed_parameters:
+      - group_by: g4 (Month)
+  
+  - name: dataset_example_subcategory
+    dataset: federate_dataset_example
+    fixed_parameters:
+      - group_by: g3 (Subcategory)

squirrels/package_data/base_project/docker/.dockerignore

@@ -2,12 +2,15 @@
 
 # common virtual environment names
 .venv/
+venv/
 
 # squirrels files to ignore
-env.yml
+.env
+.env.local
+duckdb_init.sql
+logs/
 target/
 sqrl_packages/
 
 # additional files for docker to ignore
-Dockerfile
 .git/

squirrels/package_data/base_project/docker/Dockerfile

@@ -8,9 +8,9 @@ COPY . .
 # "sqrl deps" command if there are packages defined in "squirrels.yml"
 RUN apt-get update && apt-get install -y git
 
-RUN pip install --no-cache-dir -r requirements-lock.txt
+RUN pip install --no-cache-dir -r requirements.txt
 
-RUN squirrels deps
+RUN sqrl deps
 
 EXPOSE 4465
-CMD ["squirrels", "run", "--host", "0.0.0.0", "--port", "4465"]
+CMD ["sqrl", "run", "--build", "--host", "0.0.0.0", "--port", "4465"]

squirrels/package_data/base_project/docker/compose.yml

@@ -4,4 +4,4 @@ services:
     ports:
       - "4465:4465"
     volumes:
-      - ./env.yml:/app/env.yml
+      - ./.env:/app/.env

squirrels/package_data/base_project/duckdb_init.sql

@@ -0,0 +1,9 @@
+-- SQL statements that run at the start of every DuckDB session
+
+-- Example:
+
+-- SET threads = 4;
+
+-- SET temp_directory = '/path/to/tmp/';
+
+-- CREATE SECRET (TYPE S3, PROVIDER CREDENTIAL_CHAIN);

squirrels/package_data/base_project/macros/macros_example.sql

@@ -0,0 +1,15 @@
+{%- macro date_and_amount_filters(use_from_range) -%}
+    
+    {%- if use_from_range -%}
+        date >= {{ ctx.start_date_from_range }}
+        AND date <= {{ ctx.end_date_from_range }}
+        AND amount >= {{ ctx.min_amount_from_range }}
+        AND amount <= {{ ctx.max_amount_from_range }}
+    {%- else -%}
+        date >= {{ ctx.start_date }}
+        AND date <= {{ ctx.end_date }}
+        AND amount >= {{ ctx.min_amount }}
+        AND amount <= {{ ctx.max_amount }}
+    {%- endif -%}
+
+{%- endmacro -%}

squirrels/package_data/base_project/models/builds/build_example.py

@@ -0,0 +1,26 @@
+from squirrels import BuildModelArgs
+import polars as pl, pandas as pd
+
+
+def main(sqrl: BuildModelArgs) -> pl.LazyFrame | pl.DataFrame | pd.DataFrame:
+    """
+    Create a build model by joining/processing sources or other build models to form a new
+    Python DataFrame (using polars LazyFrame, polars DataFrame, or pandas DataFrame).
+    """
+    # sqrl.ref() can be used on a sources, seeds, or other build models
+    expenses_df = sqrl.ref("src_transactions")
+    categories_df = sqrl.ref("seed_categories")
+    subcategories_df = sqrl.ref("seed_subcategories")
+
+    df = expenses_df \
+        .join(subcategories_df, on="subcategory_id", how="left") \
+        .join(categories_df, on="category_id", how="left")
+
+    df = df.with_columns(
+        pl.col("date").dt.strftime("%Y-%m").alias("month"),
+        pl.col("date").dt.strftime("%Y-%m-%d").alias("date"),
+    )
+
+    return df.select(
+        "id", "date", "month", "category_id", "category", "subcategory_id", "subcategory", "amount", "description"
+    )

squirrels/package_data/base_project/models/builds/build_example.sql

@@ -0,0 +1,16 @@
+{# DuckDB dialect #}
+
+SELECT a.id,
+    STRFTIME(a.date, '%Y-%m-%d') AS date,
+    STRFTIME(a.date, '%Y-%m') AS month,
+    c.category_id,
+    c.category,
+    b.subcategory_id,
+    b.subcategory,
+    a.amount,
+    a.description
+
+{# ref() can be used on a sources, seeds, or other build models -#}
+FROM {{ ref("src_transactions") }} AS a 
+    LEFT JOIN {{ ref("seed_subcategories") }} AS b ON a.subcategory_id = b.subcategory_id
+    LEFT JOIN {{ ref("seed_categories") }} AS c ON b.category_id = c.category_id

squirrels/package_data/base_project/models/builds/build_example.yml

@@ -0,0 +1,55 @@
+description: |
+  This is an example of a build model. It adds a new column called "month" to the source table "src_transactions".
+
+depends_on:                # optional for SQL models - the "ref" macro also adds to this set
+  - src_transactions
+  - seed_categories
+  - seed_subcategories
+
+columns:
+  - name: id
+    depends_on:
+      - src_transactions.id
+    pass_through: true
+
+  - name: date
+    type: string
+    description: The day of the transaction as a string in 'YYYY-MM-DD' format
+    depends_on:
+      - src_transactions.date
+
+  - name: month
+    type: string
+    description: The month of the transaction as a string in 'YYYY-MM' format
+    depends_on:
+      - src_transactions.date
+  
+  - name: category_id
+    depends_on:
+      - seed_categories.category_id
+    pass_through: true
+
+  - name: category
+    depends_on:
+      - seed_categories.category
+    pass_through: true
+  
+  - name: subcategory_id
+    depends_on:
+      - seed_subcategories.subcategory_id
+    pass_through: true
+
+  - name: subcategory
+    depends_on:
+      - seed_subcategories.subcategory
+    pass_through: true
+
+  - name: amount
+    depends_on:
+      - src_transactions.amount
+    pass_through: true
+
+  - name: description
+    depends_on:
+      - src_transactions.description
+    pass_through: true

squirrels/package_data/base_project/models/dbviews/dbview_example.sql

@@ -1,22 +1,12 @@
-WITH
-transactions_with_masked_id AS (
-    SELECT *,
-{%- if user.role == "manager" %}
-        id as masked_id
-{%- else %}
-        '***' as masked_id
-{%- endif %},
-        STRFTIME('%Y-%m', date) AS month
-    FROM transactions
-)
-SELECT {{ ctx.select_dim_cols }}
-    , SUM(-amount) as total_amount
-FROM transactions_with_masked_id
-WHERE date >= :start_date
-    AND date <= :end_date
-    AND -amount >= :min_amount
-    AND -amount <= :max_amount
-    {% if is_placeholder("desc_pattern") -%} AND description LIKE :desc_pattern {%- endif %}
-    {% if ctx.has_categories -%} AND category IN ({{ ctx.categories }}) {%- endif %}
-    {% if ctx.has_subcategories -%} AND subcategory IN ({{ ctx.subcategories }}) {%- endif %}
-GROUP BY {{ ctx.group_by_cols }}
+{# SQLite dialect (based on connection used) #}
+
+SELECT STRFTIME('%Y-%m', date) AS month
+    , printf('%.2f', SUM(amount)) as total_amount
+
+FROM {{ source("src_transactions") }}
+
+WHERE {{ date_and_amount_filters(use_from_range=false) }}
+
+GROUP BY 1
+
+ORDER BY 1 DESC

squirrels/package_data/base_project/models/dbviews/dbview_example.yml

@@ -0,0 +1,26 @@
+description: |
+  This is an example of a database view model. It finds the total amount spent by month.
+
+  Parameters are available to filter the date and amount of the transactions.
+
+connection: default        # optional - if not provided, will use default connection specified in the SQRL_CONNECTIONS__DEFAULT_NAME_USED setting
+
+translate_to_duckdb: true  # optional - default is false - if true, then the model will be translated to duckdb for supported dialects
+
+depends_on:                # optional - the "source" macro also adds to this set
+  - src_transactions
+
+columns:
+  - name: month
+    type: string
+    description: The months for which the amount is aggregated by, in descending order
+    category: dimension
+    depends_on:
+      - src_transactions.date
+  
+  - name: total_amount
+    type: float
+    description: The total amount spent by the group-by dimension
+    category: measure
+    depends_on:
+      - src_transactions.amount

squirrels/package_data/base_project/models/federates/federate_example.py

@@ -1,21 +1,44 @@
-from typing import Sequence
-from squirrels import ModelDepsArgs, ModelArgs
-import pandas as pd
+from squirrels import ModelArgs, parameters as p
+import polars as pl, pandas as pd
 
+def dequote(value: str) -> str:
+    return value[1:-1]
 
-def dependencies(sqrl: ModelDepsArgs) -> Sequence[str]:
-    """
-    Define list of dependent models here. This will determine the dependencies first, at compile-time, 
-    before running the model.
-    """
-    return ["dbview_example"]
+def joined_str_to_list(value: str) -> list[str]:
+    return [dequote(category) for category in str(value).split(",")]
 
 
-def main(sqrl: ModelArgs) -> pd.DataFrame:
+def main(sqrl: ModelArgs) -> pl.LazyFrame | pl.DataFrame | pd.DataFrame:
     """
-    Create federated models by joining/processing dependent database views and/or other federated models to
-    form and return the result as a new pandas DataFrame.
+    Create federated models by joining/processing dependent models (sources, seeds, builds, dbviews, other federates, etc.) to
+    form a new Python DataFrame (using polars LazyFrame, polars DataFrame, or pandas DataFrame).
     """
-    (DBVIEW_EXAMPLE,) = dependencies(sqrl)
-    df = sqrl.ref(DBVIEW_EXAMPLE)
-    return df.sort_values(sqrl.ctx["order_by_cols_list"], ascending=False)
+    df = sqrl.ref("build_example")
+
+    df = df.filter(
+        (pl.col("amount") >= sqrl.ctx["min_amount_from_range"]) &
+        (pl.col("amount") <= sqrl.ctx["max_amount_from_range"]) &
+        (pl.col("date") >= dequote(sqrl.ctx["start_date_from_range"])) &
+        (pl.col("date") <= dequote(sqrl.ctx["end_date_from_range"]))
+    )
+
+    if sqrl.ctx["has_categories"]:
+        categories_list = joined_str_to_list(sqrl.ctx["categories"])
+        df = df.filter(pl.col("category_id").is_in(categories_list))
+
+    if sqrl.ctx["has_subcategories"]:
+        subcategories_list = joined_str_to_list(sqrl.ctx["subcategories"])
+        df = df.filter(pl.col("subcategory_id").is_in(subcategories_list))
+
+    dimension_cols = sqrl.ctx["group_by_cols_list"]
+    df = df.group_by(dimension_cols).agg(
+        pl.sum("amount").cast(pl.Decimal(precision=15, scale=2)).alias("total_amount")
+    )
+    df = df.sort(dimension_cols, descending=True)
+
+    if sqrl.param_exists("limit"):
+        assert isinstance(limit := sqrl.prms["limit"], p.NumberParameter)
+        df = df.limit(int(limit.get_selected_value()))
+
+    df = df.select(*dimension_cols, "total_amount")
+    return df.rename(sqrl.ctx["rename_dict"])

squirrels/package_data/base_project/models/federates/federate_example.sql

@@ -1,3 +1,17 @@
-SELECT *
-FROM {{ ref("dbview_example") }}
+{# DuckDB dialect #}
+
+SELECT {{ ctx.select_dim_cols }}
+    , CAST(SUM(amount) AS DECIMAL(15, 2)) as total_amount
+
+{# ref() can be used on a sources, seeds, builds, dbviews, or other federate models -#}
+FROM {{ ref("build_example") }} AS a
+
+WHERE {{ date_and_amount_filters(use_from_range=true) }}
+    {% if ctx.has_categories -%} AND category_id IN ({{ ctx.categories }}) {%- endif %}
+    {% if ctx.has_subcategories -%} AND subcategory_id IN ({{ ctx.subcategories }}) {%- endif %}
+
+GROUP BY {{ ctx.group_by_cols }}
+
 ORDER BY {{ ctx.order_by_cols }}
+
+{{ ctx.limit_clause }}

squirrels/package_data/base_project/models/federates/federate_example.yml

@@ -0,0 +1,65 @@
+description: |
+  This is an example of a federate view model. It takes the build example model and groups or filters the results based on the parameter selections provided.
+
+  Parameters are available to specify the group by dimension and filter by date, amount of the transaction, category, and subcategory.
+
+depends_on:                # optional for SQL models - the "ref" macro also adds to this set
+  - build_example
+
+eager: false               # optional - defaults to false. Only applies to SQL models.
+
+columns:
+  - name: date
+    type: string
+    condition: parameter 'group_by' (Group By) is 'g0' (Transaction)
+    description: The date of the transaction in 'YYYY-MM-DD' format, in descending order
+    category: dimension
+    depends_on:
+      - build_example.date
+  
+  - name: description
+    type: string
+    condition: parameter 'group_by' (Group By) is 'g0' (Transaction)
+    description: The description of the transaction
+    category: dimension
+    depends_on:
+      - build_example.description
+  
+  - name: day
+    type: string
+    condition: parameter 'group_by' (Group By) is 'g1' (Day)
+    description: The day for which the amount is aggregated by, in descending order
+    category: dimension
+    depends_on:
+      - build_example.date
+  
+  - name: month
+    type: string
+    condition: parameter 'group_by' (Group By) is 'g4' (Month)
+    description: The month for which the amount is aggregated by, in descending order
+    category: dimension
+    depends_on:
+      - build_example.month
+
+  - name: category
+    type: string
+    condition: parameter `group_by` (Group By) is `g0` (Transaction), `g2` (Category), or `g3` (Subcategory)
+    description: The category for which the amount is aggregated by
+    category: dimension
+    depends_on:
+      - build_example.category
+  
+  - name: subcategory
+    type: string
+    condition: parameter `group_by` (Group By) is `g0` (Transaction) or `g3` (Subcategory)
+    description: The subcategory for which the amount is aggregated by
+    category: dimension
+    depends_on:
+      - build_example.subcategory
+
+  - name: total_amount
+    type: float
+    description: The total amount spent by the group by dimension
+    category: measure
+    depends_on:
+      - build_example.total_amount

squirrels/package_data/base_project/models/sources.yml

@@ -0,0 +1,39 @@
+sources:
+  - name: src_transactions
+    description: "The source table for transactions"  # optional
+    connection: default           # optional - if not provided, will use the connection named "default" or the default connection specified in settings
+    table: expenses               # optional - if not provided, will use the "name" field of the source
+    load_to_duckdb: true          # optional - default is false - outside of dbview models that have translate_to_duckdb as false, other models can only reference this source if load_to_duckdb is true
+
+    # For performance reasons, avoid specifying primary_key for large tables if upserts are not required
+    primary_key: [id]             # optional - if not provided, then this is an insert-only table for incremental loads - otherwise, this uses upsert
+    
+    update_hints:
+      increasing_column: date     # optional - if not provided, will always do full refresh, otherwise uses this column for incremental loads
+      strictly_increasing: false  # optional - default is true - if false, then maximum value of column is removed before incremental loads
+
+    columns:                      # optional - if load_to_duckdb is true, then only the columns listed here are loaded to duckdb
+      - name: id
+        type: string
+        description: The unique identifier for the transaction
+        category: dimension
+
+      - name: date
+        type: date
+        description: The date of the transaction
+        category: dimension
+
+      - name: subcategory_id
+        type: string
+        description: The ID of the subcategory of the transaction
+        category: dimension
+
+      - name: amount
+        type: float
+        description: The amount of the transaction
+        category: measure
+
+      - name: description
+        type: string
+        description: The description of the transaction
+        category: dimension