squirrels 0.4.1__py3-none-any.whl → 0.5.0__py3-none-any.whl

squirrels/_package_data/base_project/.env

@@ -0,0 +1,42 @@
+# Custom environment variables
+SQLITE_URI="sqlite:///{project_path}/assets/expenses.db"
+
+# Secrets used by the Squirrels framework that are NOT SAFE TO INCLUDE IN VERSION CONTROL
+# Required if your project uses authentication. Otherwise, optional.
+SQRL_SECRET__KEY="{{ random_secret_key }}"
+SQRL_SECRET__ADMIN_PASSWORD="{{ random_admin_password }}"
+
+# Optional variables used by the Squirrels framework that are safe to include in version control if desired
+# (default values are shown below)
+SQRL_AUTH__DB_FILE_PATH="{project_path}/target/auth.sqlite"
+SQRL_AUTH__TOKEN_EXPIRE_MINUTES="30"
+SQRL_AUTH__ALLOWED_ORIGINS_FOR_COOKIES="https://squirrels-analytics.github.io"
+
+SQRL_PARAMETERS__CACHE_SIZE="1024"
+SQRL_PARAMETERS__CACHE_TTL_MINUTES="60"
+SQRL_PARAMETERS__DATASOURCE_REFRESH_MINUTES="60"
+
+SQRL_DATASETS__CACHE_SIZE="128"
+SQRL_DATASETS__CACHE_TTL_MINUTES="60"
+SQRL_DATASETS__MAX_ROWS_FOR_AI="100"
+
+SQRL_DASHBOARDS__CACHE_SIZE="128"
+SQRL_DASHBOARDS__CACHE_TTL_MINUTES="60"
+
+SQRL_PERMISSIONS__ELEVATED_ACCESS_LEVEL="admin" # one of "admin", "member", "guest"
+
+SQRL_SEEDS__INFER_SCHEMA="true"
+SQRL_SEEDS__NA_VALUES=["NA"] # must be a JSON list
+
+SQRL_CONNECTIONS__DEFAULT_NAME_USED="default"
+
+SQRL_VDL__CATALOG_DB_PATH="ducklake:{project_path}/target/vdl_catalog.duckdb"
+SQRL_VDL__DATA_PATH="{project_path}/target/vdl_data/"
+
+SQRL_STUDIO__BASE_URL="https://squirrels-analytics.github.io/squirrels-studio-v1"
+
+SQRL_LOGGING__LOG_LEVEL="INFO" # one of "DEBUG", "INFO", "WARNING"
+SQRL_LOGGING__LOG_FORMAT="text"
+SQRL_LOGGING__LOG_TO_FILE="false"
+SQRL_LOGGING__LOG_FILE_SIZE_MB="50"
+SQRL_LOGGING__LOG_FILE_BACKUP_COUNT="1"

squirrels/_package_data/base_project/.env.example

@@ -0,0 +1,42 @@
+# Custom environment variables
+SQLITE_URI="sqlite:///{project_path}/assets/expenses.db"
+
+# Secrets used by the Squirrels framework that are NOT SAFE TO INCLUDE IN VERSION CONTROL
+# Required if your project uses authentication. Otherwise, optional.
+SQRL_SECRET__KEY="" # a random 32 byte hex string - one way to generate this is by running "openssl rand -hex 32" in bash
+SQRL_SECRET__ADMIN_PASSWORD=""
+
+# Optional variables used by the Squirrels framework that are safe to include in version control if desired
+# (default values are shown below)
+SQRL_AUTH__DB_FILE_PATH="target/auth.sqlite"
+SQRL_AUTH__TOKEN_EXPIRE_MINUTES="30"
+SQRL_AUTH__ALLOWED_ORIGINS_FOR_COOKIES="https://squirrels-analytics.github.io"
+
+SQRL_PARAMETERS__CACHE_SIZE="1024"
+SQRL_PARAMETERS__CACHE_TTL_MINUTES="60"
+SQRL_PARAMETERS__DATASOURCE_REFRESH_MINUTES="60"
+
+SQRL_DATASETS__CACHE_SIZE="128"
+SQRL_DATASETS__CACHE_TTL_MINUTES="60"
+SQRL_DATASETS__MAX_ROWS_FOR_AI="100"
+
+SQRL_DASHBOARDS__CACHE_SIZE="128"
+SQRL_DASHBOARDS__CACHE_TTL_MINUTES="60"
+
+SQRL_PERMISSIONS__ELEVATED_ACCESS_LEVEL="admin" # one of "admin", "member", "guest"
+
+SQRL_SEEDS__INFER_SCHEMA="true"
+SQRL_SEEDS__NA_VALUES=["NA"] # must be a JSON list
+
+SQRL_CONNECTIONS__DEFAULT_NAME_USED="default"
+
+SQRL_VDL__CATALOG_DB_PATH="ducklake:{project_path}/target/vdl_catalog.duckdb"
+SQRL_VDL__DATA_PATH="{project_path}/target/vdl_data/"
+
+SQRL_STUDIO__BASE_URL="https://squirrels-analytics.github.io/squirrels-studio-v1"
+
+SQRL_LOGGING__LOG_LEVEL="INFO" # one of "DEBUG", "INFO", "WARNING"
+SQRL_LOGGING__LOG_FORMAT="text"
+SQRL_LOGGING__LOG_TO_FILE="false"
+SQRL_LOGGING__LOG_FILE_SIZE_MB="50"
+SQRL_LOGGING__LOG_FILE_BACKUP_COUNT="1"

squirrels/_package_data/base_project/connections.yml

@@ -0,0 +1,16 @@
+## Connection URIs are usually in format "dialect://username:password@host:port/database" for database connections
+## However, subtle differences exist depending on the "type" specified. For example, sqlite URIs are slightly different.
+##   sqlalchemy:      sqlite:///{project_path}/relative/path/to/database.db
+##   connectorx/adbc: sqlite://{project_path}/relative/path/to/database.db  (adbc URI format matches connectorx)
+##   duckdb:          sqlite:{project_path}/relative/path/to/database.db
+## Refer to specific documentation for supported databases by type (with URI examples):
+##   sqlalchemy: https://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls
+##   connectorx: https://sfu-db.github.io/connector-x/databases.html
+##   adbc:       https://arrow.apache.org/adbc/ (see connectorx documentation for URI examples)
+connections:
+  - name: default
+    label: SQLite Expenses Database
+    type: sqlalchemy                      ## one of: sqlalchemy, connectorx, adbc, or duckdb
+    uri: {{ env_vars.SQLITE_URI }}        ## using Jinja to substitute environment variables
+
+

squirrels/_package_data/base_project/dashboards/dashboard_example.py

@@ -0,0 +1,34 @@
+from squirrels import arguments as args, dashboards as d
+from matplotlib import pyplot as plt, figure as f, axes as a
+
+
+async def main(sqrl: args.DashboardArgs) -> d.PngDashboard:
+    spending_by_month_df = await sqrl.dataset("federate_dataset_example", fixed_parameters={"group_by": "month"})
+    spending_by_subcategory_df = await sqrl.dataset("federate_dataset_example", fixed_parameters={"group_by": "subcat"})
+
+    # Create a figure with two subplots
+    fig, (ax0, ax1) = plt.subplots(2, 1, figsize=(8, 8), height_ratios=(1, 2))
+    fig: f.Figure; ax0: a.Axes; ax1: a.Axes
+    fig.tight_layout(pad=4, h_pad=6)
+
+    # Create a bar chart of spending by month
+
+    # Convert to pandas and ensure total_amount is numeric
+    spending_by_month_pandas = spending_by_month_df.sort("month").to_pandas()
+    spending_by_month_pandas["total_amount"] = spending_by_month_pandas["total_amount"].astype(float)
+
+    spending_by_month_pandas.plot(x="month", y="total_amount", ax=ax0)
+    ax0.set_title("Spending by Month")
+
+    # Create a pie chart of spending by subcategory
+
+    # Convert to pandas and ensure total_amount is numeric
+    subcategory_pandas = spending_by_subcategory_df.sort("total_amount", descending=True).to_pandas()
+    subcategory_pandas["total_amount"] = subcategory_pandas["total_amount"].astype(float)
+    subcategory_pandas.set_index("subcategory", inplace=True)
+    
+    autopct = lambda pct: ('%.1f%%' % pct) if pct > 6 else ''
+    subcategory_pandas.plot(y="total_amount", kind='pie', ax=ax1, autopct=autopct, legend=False, ylabel="")
+    ax1.set_title("Spending by Subcategory")
+    
+    return d.PngDashboard(fig)

squirrels/_package_data/base_project/dashboards/dashboard_example.yml

@@ -0,0 +1,22 @@
+label: Dashboard Example
+
+description: This is an example dashboard
+
+scope: protected
+
+format: png
+
+parameters:
+  - date_range
+  - category
+
+depends_on:
+  - name: dataset_example_month
+    dataset: federate_dataset_example
+    fixed_parameters:
+      - group_by: month (Month)
+  
+  - name: dataset_example_subcategory
+    dataset: federate_dataset_example
+    fixed_parameters:
+      - group_by: subcat (Subcategory)

squirrels/{package_data → _package_data}/base_project/docker/.dockerignore

@@ -2,12 +2,15 @@
 
 # common virtual environment names
 .venv/
+venv/
 
 # squirrels files to ignore
-env.yml
+.env
+.env.local
+duckdb_init.sql
+logs/
 target/
 sqrl_packages/
 
 # additional files for docker to ignore
-Dockerfile
 .git/

squirrels/{package_data → _package_data}/base_project/docker/Dockerfile

@@ -8,9 +8,9 @@ COPY . .
 # "sqrl deps" command if there are packages defined in "squirrels.yml"
 RUN apt-get update && apt-get install -y git
 
-RUN pip install --no-cache-dir -r requirements-lock.txt
+RUN pip install --no-cache-dir -r requirements.txt
 
-RUN squirrels deps
+RUN sqrl deps
 
 EXPOSE 4465
-CMD ["squirrels", "run", "--host", "0.0.0.0", "--port", "4465"]
+CMD ["sqrl", "run", "--build", "--host", "0.0.0.0", "--port", "4465"]

squirrels/{package_data → _package_data}/base_project/docker/compose.yml

@@ -4,4 +4,4 @@ services:
     ports:
       - "4465:4465"
     volumes:
-      - ./env.yml:/app/env.yml
+      - ./.env:/app/.env

squirrels/_package_data/base_project/duckdb_init.sql

@@ -0,0 +1,10 @@
+-- SQL statements that run at the start of every DuckDB session
+-- If the VDL catalog path is provided, the project will ATTACH it as 'vdl' (READ_ONLY)
+
+-- Example:
+
+-- SET threads = 4;
+
+-- SET temp_directory = '/path/to/tmp/';
+
+-- CREATE SECRET (TYPE S3, PROVIDER CREDENTIAL_CHAIN);

squirrels/{package_data/base_project/.gitignore → _package_data/base_project/gitignore}

@@ -1,12 +1,13 @@
 **/__pycache__/
 
 # common virtual environment names
-.env/
 .venv/
 venv/
 
 # squirrels files to ignore
-env.yml
+.env
+.env.local
+duckdb_init.sql
 logs/
 target/
 sqrl_packages/

squirrels/_package_data/base_project/macros/macros_example.sql

@@ -0,0 +1,17 @@
+{%- macro date_and_amount_filters(use_from_range) -%}
+    {%- if use_from_range -%}
+
+    date >= {{ ctx.start_date_from_range | quote }}
+    AND date <= {{ ctx.end_date_from_range | quote }}
+    AND amount >= {{ ctx.min_amount_from_range }}
+    AND amount <= {{ ctx.max_amount_from_range }}
+
+    {%- else -%}
+
+    date >= {{ ctx.start_date | quote }}
+    AND date <= {{ ctx.end_date | quote }}
+    AND amount >= {{ ctx.min_amount }}
+    AND amount <= {{ ctx.max_amount }}
+
+    {%- endif -%}
+{%- endmacro -%}

squirrels/_package_data/base_project/models/builds/build_example.py

@@ -0,0 +1,26 @@
+from squirrels import arguments as args
+import polars as pl, pandas as pd
+
+
+def main(sqrl: args.BuildModelArgs) -> pl.LazyFrame | pl.DataFrame | pd.DataFrame:
+    """
+    Create a build model by joining/processing sources or other build models to form a new
+    Python DataFrame (using polars LazyFrame, polars DataFrame, or pandas DataFrame).
+    """
+    # sqrl.ref() can be used on a sources, seeds, or other build models
+    expenses_df = sqrl.ref("src_transactions")
+    categories_df = sqrl.ref("seed_categories")
+    subcategories_df = sqrl.ref("seed_subcategories")
+
+    df = expenses_df \
+        .join(subcategories_df, on="subcategory_id", how="left") \
+        .join(categories_df, on="category_id", how="left")
+
+    df = df.with_columns(
+        pl.col("date").dt.strftime("%Y-%m").alias("month"),
+        pl.col("date").dt.strftime("%Y-%m-%d").alias("date"),
+    )
+
+    return df.select(
+        "id", "date", "month", "category_id", "category", "subcategory_id", "subcategory", "amount", "description"
+    )

squirrels/_package_data/base_project/models/builds/build_example.sql

@@ -0,0 +1,16 @@
+{#- DuckDB dialect -#}
+
+SELECT a.id,
+    STRFTIME(a.date, '%Y-%m-%d') AS date,
+    STRFTIME(a.date, '%Y-%m') AS month,
+    c.category_id,
+    c.category,
+    b.subcategory_id,
+    b.subcategory,
+    a.amount,
+    a.description
+
+{# ref() can be used on a sources, seeds, or other build models -#}
+FROM {{ ref("src_transactions") }} AS a 
+    LEFT JOIN {{ ref("seed_subcategories") }} AS b ON a.subcategory_id = b.subcategory_id
+    LEFT JOIN {{ ref("seed_categories") }} AS c ON b.category_id = c.category_id

squirrels/_package_data/base_project/models/builds/build_example.yml

@@ -0,0 +1,57 @@
+description: |
+  This is an example of a build model. It adds a new column called "month" to the source table "src_transactions".
+
+materialization: TABLE     # optional - defaults to "VIEW" for SQL models, ignored and always a "TABLE" for Python models
+
+depends_on:                # optional for SQL models - the "ref" macro also adds to this set
+  - src_transactions
+  - seed_categories
+  - seed_subcategories
+
+columns:
+  - name: id
+    depends_on:
+      - src_transactions.id
+    pass_through: true
+
+  - name: date
+    type: string
+    description: The day of the transaction as a string in 'YYYY-MM-DD' format
+    depends_on:
+      - src_transactions.date
+
+  - name: month
+    type: string
+    description: The month of the transaction as a string in 'YYYY-MM' format
+    depends_on:
+      - src_transactions.date
+  
+  - name: category_id
+    depends_on:
+      - seed_categories.category_id
+    pass_through: true
+
+  - name: category
+    depends_on:
+      - seed_categories.category
+    pass_through: true
+  
+  - name: subcategory_id
+    depends_on:
+      - seed_subcategories.subcategory_id
+    pass_through: true
+
+  - name: subcategory
+    depends_on:
+      - seed_subcategories.subcategory
+    pass_through: true
+
+  - name: amount
+    depends_on:
+      - src_transactions.amount
+    pass_through: true
+
+  - name: description
+    depends_on:
+      - src_transactions.description
+    pass_through: true

squirrels/_package_data/base_project/models/dbviews/dbview_example.sql

@@ -0,0 +1,12 @@
+{#- SQLite dialect (based on connection used) -#}
+
+SELECT STRFTIME('%Y-%m', date) AS month
+    , printf('%.2f', SUM(amount)) as total_amount
+
+FROM {{ source("src_transactions") }}
+
+WHERE {{ date_and_amount_filters(use_from_range=false) }}
+
+GROUP BY 1
+
+ORDER BY 1 DESC

squirrels/_package_data/base_project/models/dbviews/dbview_example.yml

@@ -0,0 +1,26 @@
+description: |
+  This is an example of a database view model. It finds the total amount spent by month.
+
+  Parameters are available to filter the date and amount of the transactions.
+
+connection: default        # optional - if not provided, will use default connection specified in the SQRL_CONNECTIONS__DEFAULT_NAME_USED setting
+
+translate_to_duckdb: true  # optional - default is false - if true, then the model will be translated to duckdb for supported dialects
+
+depends_on:                # optional - the "source" macro also adds to this set
+  - src_transactions
+
+columns:
+  - name: month
+    type: string
+    description: The months for which the amount is aggregated by, in descending order
+    category: dimension
+    depends_on:
+      - src_transactions.date
+  
+  - name: total_amount
+    type: float
+    description: The total amount spent by month
+    category: measure
+    depends_on:
+      - src_transactions.amount

squirrels/_package_data/base_project/models/federates/federate_example.py

@@ -0,0 +1,37 @@
+from squirrels import arguments as args
+import polars as pl, pandas as pd
+
+
+def main(sqrl: args.ModelArgs) -> pl.LazyFrame | pl.DataFrame | pd.DataFrame:
+    """
+    Create federated models by joining/processing dependent models (sources, seeds, builds, dbviews, other federates, etc.) to
+    form a new Python DataFrame (using polars LazyFrame, polars DataFrame, or pandas DataFrame).
+    """
+    df = sqrl.ref("build_example")
+
+    df = df.filter(
+        (pl.col("date") >= sqrl.ctx["start_date_from_range"]) &
+        (pl.col("date") <= sqrl.ctx["end_date_from_range"]) &
+        (pl.col("amount") >= sqrl.ctx["min_amount_from_range"]) &
+        (pl.col("amount") <= sqrl.ctx["max_amount_from_range"])
+    )
+
+    if sqrl.ctx["has_categories"]:
+        categories: list[str] = sqrl.ctx["categories"]
+        df = df.filter(pl.col("category_id").is_in(categories))
+
+    if sqrl.ctx["has_subcategories"]:
+        subcategories: list[str] = sqrl.ctx["subcategories"]
+        df = df.filter(pl.col("subcategory_id").is_in(subcategories))
+
+    dimension_cols: list[str] = sqrl.ctx["group_by_cols"]
+    df = df.group_by(dimension_cols).agg(
+        pl.sum("amount").cast(pl.Decimal(precision=15, scale=2)).alias("total_amount")
+    )
+    df = df.rename(sqrl.ctx["column_to_alias_mapping"])
+
+    order_by_cols: list[str] = sqrl.ctx["order_by_cols"]
+    df = df.select(*order_by_cols, "total_amount") \
+        .sort(order_by_cols, descending=True)
+
+    return df

squirrels/_package_data/base_project/models/federates/federate_example.sql

@@ -0,0 +1,19 @@
+{#- DuckDB dialect -#}
+
+SELECT {{ ctx.select_dim_cols | join }}
+    , CAST(SUM(amount) AS DECIMAL(15, 2)) as total_amount
+
+{# ref() can be used on a sources, seeds, builds, dbviews, or other federate models -#}
+FROM {{ ref("build_example") }} AS a
+
+WHERE {{ date_and_amount_filters(use_from_range=true) }}
+{%- if ctx.has_categories %} 
+    AND category_id IN ({{ ctx.categories | quote_and_join }}) 
+{%- endif %}
+{%- if ctx.has_subcategories %} 
+    AND subcategory_id IN ({{ ctx.subcategories | quote_and_join }}) 
+{%- endif %}
+
+GROUP BY {{ ctx.group_by_cols | join }}
+
+ORDER BY {{ ctx.order_by_cols_desc | join }}

squirrels/_package_data/base_project/models/federates/federate_example.yml

@@ -0,0 +1,65 @@
+description: |
+  This is an example of a federate view model. It takes the build example model and groups or filters the results based on the parameter selections provided.
+
+  Parameters are available to specify the group by dimension and filter by date, amount of the transaction, category, and subcategory.
+
+depends_on:                # optional for SQL models - the "ref" macro also adds to this set
+  - build_example
+
+eager: false               # optional - defaults to false. Only applies to SQL models.
+
+columns:
+  - name: date
+    type: string
+    condition: parameter 'group_by' (Group By) is 'trans' (Transaction)
+    description: The date of the transaction in 'YYYY-MM-DD' format, in descending order
+    category: dimension
+    depends_on:
+      - build_example.date
+  
+  - name: description
+    type: string
+    condition: parameter 'group_by' (Group By) is 'trans' (Transaction)
+    description: The description of the transaction
+    category: dimension
+    depends_on:
+      - build_example.description
+  
+  - name: day
+    type: string
+    condition: parameter 'group_by' (Group By) is 'day' (Day)
+    description: The day for which the amount is aggregated by, in descending order
+    category: dimension
+    depends_on:
+      - build_example.date
+  
+  - name: month
+    type: string
+    condition: parameter 'group_by' (Group By) is 'month' (Month)
+    description: The month for which the amount is aggregated by, in descending order
+    category: dimension
+    depends_on:
+      - build_example.month
+
+  - name: category
+    type: string
+    condition: parameter `group_by` (Group By) is `trans` (Transaction), `cat` (Category), or `subcat` (Subcategory)
+    description: The category for which the amount is aggregated by
+    category: dimension
+    depends_on:
+      - build_example.category
+  
+  - name: subcategory
+    type: string
+    condition: parameter `group_by` (Group By) is `trans` (Transaction) or `subcat` (Subcategory)
+    description: The subcategory for which the amount is aggregated by
+    category: dimension
+    depends_on:
+      - build_example.subcategory
+
+  - name: total_amount
+    type: float
+    description: The total amount spent by the group by dimension
+    category: measure
+    depends_on:
+      - build_example.total_amount

squirrels/_package_data/base_project/models/sources.yml

@@ -0,0 +1,38 @@
+sources:
+  - name: src_transactions
+    description: "The source table for transactions"  # optional
+    connection: default           # optional - if not provided, will use the connection named "default" or the default connection specified in settings
+    table: expenses               # optional - if not provided, will use the "name" field of the source
+    load_to_vdl: true             # optional - default is false - other than dbview models (with translate_to_duckdb set to false), other models can only reference this source if load_to_vdl is true or connection type is duckdb
+    
+    update_hints:
+      increasing_column: date     # optional - if not provided, will always do full refresh, otherwise uses this column for incremental loads
+      strictly_increasing: false  # optional - default is true - if false, then maximum value of column is removed before incremental load is performed
+
+    primary_key: [id]             # optional - if not provided, then this is an insert-only table for incremental loads
+    
+    columns:                      # optional - if load_to_vdl is true, then only the columns listed here are loaded to the Virtual Data Lake (VDL)
+      - name: id
+        type: string
+        description: The unique identifier for the transaction
+        category: dimension
+
+      - name: date
+        type: date
+        description: The date of the transaction
+        category: dimension
+
+      - name: subcategory_id
+        type: string
+        description: The ID of the subcategory of the transaction
+        category: dimension
+
+      - name: amount
+        type: float
+        description: The amount of the transaction
+        category: measure
+
+      - name: description
+        type: string
+        description: The description of the transaction
+        category: dimension