pointblank 0.12.1__tar.gz → 0.13.0__tar.gz

{pointblank-0.12.1 → pointblank-0.13.0}/.gitignore

@@ -133,3 +133,4 @@ benchmark/data/*.json
 .swp
 
 uv.lock
+memory

{pointblank-0.12.1 → pointblank-0.13.0}/CITATION.cff

@@ -3,7 +3,7 @@ message: 'If you wish to cite the "Pointblank" package use:'
 type: software
 license: MIT
 title: "Pointblank: Find out if your data is what you think it is"
-version: 0.11.3
+version: 0.12.2
 abstract: Validate data in Polars and Pandas DataFrames and database tables.
   Validation pipelines can be made using easily-readable, consecutive validation
   steps. Upon execution of the validation plan, several reporting options are available.

{pointblank-0.12.1 → pointblank-0.13.0}/Makefile

@@ -13,11 +13,19 @@ test:
 test-update:
 	pytest --snapshot-update
 
-
+.PHONY: lint
 lint: ## Run ruff formatter and linter
 	@uv run ruff format
 	@uv run ruff check --fix
 
+.PHONY: install-pre-commit
+install-pre-commit: # Install pre-commit hooks
+	@uvx pre-commit install
+
+.PHONY: run-pre-commit
+run-pre-commit: # Run pre-commit hooks
+	@uvx pre-commit run --all-files
+
 check:
 	pyright --pythonversion 3.8 pointblank
 	pyright --pythonversion 3.9 pointblank

{pointblank-0.12.1 → pointblank-0.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pointblank
-Version: 0.12.1
+Version: 0.13.0
 Summary: Find out if your data is what you think it is.
 Author-email: Richard Iannone <riannone@me.com>
 License: MIT License
@@ -60,6 +60,12 @@ Requires-Dist: chatlas>=0.3.0; extra == "generate"
 Requires-Dist: anthropic[bedrock]>=0.45.2; extra == "generate"
 Requires-Dist: openai>=1.63.0; extra == "generate"
 Requires-Dist: shiny>=1.3.0; extra == "generate"
+Provides-Extra: mcp
+Requires-Dist: mcp[cli]>=1.10.1; extra == "mcp"
+Requires-Dist: fastmcp>=2.11.3; extra == "mcp"
+Requires-Dist: pytest-asyncio>=1.0.0; extra == "mcp"
+Provides-Extra: excel
+Requires-Dist: openpyxl>=3.0.0; extra == "excel"
 Provides-Extra: bigquery
 Requires-Dist: ibis-framework[bigquery]>=9.5.0; extra == "bigquery"
 Provides-Extra: databricks
@@ -84,6 +90,7 @@ Requires-Dist: quartodoc>=0.8.1; python_version >= "3.9" and extra == "docs"
 Requires-Dist: pandas>=2.2.3; extra == "docs"
 Requires-Dist: polars>=1.17.1; extra == "docs"
 Requires-Dist: pyspark==3.5.6; extra == "docs"
+Requires-Dist: openpyxl>=3.0.0; extra == "docs"
 Dynamic: license-file
 
 <div align="center">

{pointblank-0.12.1 → pointblank-0.13.0}/docs/_quarto.yml

@@ -98,6 +98,9 @@ website:
             - user-guide/cli-data-inspection.qmd
             - user-guide/cli-data-validation.qmd
             - user-guide/cli-reference.qmd
+        - section: "MCP Server"
+          contents:
+            - user-guide/mcp-quick-start.qmd
 
   page-footer:
     left: 'Proudly supported by <a href="https://www.posit.co/" class="no-icon"><img src="/assets/posit-logo-black.svg" alt="Posit" width="80" style="padding-left: 3px;vertical-align:text-top;"></a>'
@@ -194,6 +197,7 @@ quartodoc:
         can split the data based on the validation results (with `get_sundered_data()`).
       contents:
         - name: Validate.interrogate
+        - name: Validate.set_tbl
         - name: Validate.get_tabular_report
         - name: Validate.get_step_report
         - name: Validate.get_json_report

{pointblank-0.12.1 → pointblank-0.13.0}/docs/user-guide/installation.qmd

@@ -32,6 +32,15 @@ uv pip install pointblank
 ```bash
 conda install -c conda-forge pointblank
 ```
+
+## pixi
+
+```bash
+# add pointblank to project
+pixi init name-of-project
+cd name-of-project
+pixi add pointblank
+```
 :::
 
 ## DataFrame Libraries
@@ -122,6 +131,23 @@ conda install -c conda-forge pointblank-snowflake   # Snowflake
 # Example of installing multiple backends
 conda install -c conda-forge pointblank-duckdb pointblank-postgres pointblank-sqlite
 ```
+
+## pixi
+
+```bash
+pixi add pointblank-sqlite      # SQLite
+pixi add pointblank-duckdb      # DuckDB
+pixi add pointblank-postgres    # PostgreSQL
+pixi add pointblank-mysql       # MySQL
+pixi add pointblank-mssql       # Microsoft SQL Server
+pixi add pointblank-bigquery    # BigQuery
+pixi add pointblank-pyspark     # Apache Spark
+pixi add pointblank-databricks  # Databricks
+pixi add pointblank-snowflake   # Snowflake
+
+# Example of installing multiple backends
+pixi add pointblank-duckdb pointblank-postgres pointblank-sqlite
+```
 :::
 
 ::: {.callout-note}

pointblank-0.13.0/docs/user-guide/mcp-quick-start.qmd

@@ -0,0 +1,248 @@
+---
+title: "MCP Quick Start"
+jupyter: python3
+toc-expand: 2
+html-table-processing: none
+---
+
+## Getting Started in 5 Minutes
+
+Transform your data validation workflow with conversational AI in VS Code or Positron IDE. Here are three simple steps to start validating data through conversation (and no complex configuration required).
+
+### 1. Install
+
+```bash
+pip install pointblank[mcp,pd,excel]
+```
+
+**What this installs:**
+
+- `mcp` - Model Context Protocol server dependencies
+- `pd` - pandas backend for data processing
+- `excel` - Excel file support (`openpyxl`)
+
+**Alternative installs based on your needs:**
+
+```bash
+# Minimal MCP server only
+pip install pointblank[mcp]
+
+# Add Polars for faster data processing
+pip install pointblank[mcp,pd,pl]
+
+# Full installation with all backends
+pip install pointblank[mcp,pd,pl,excel]
+```
+
+### 2. Configure Your IDE
+
+**For VS Code**:
+
+**Option 1: Workspace Configuration (Recommended for teams)**
+
+1. Create a `.vscode/mcp.json` file in your project folder
+2. Add this configuration:
+
+```json
+{
+  "servers": {
+    "pointblank": {
+      "command": "python",
+      "args": ["-m", "pointblank_mcp_server.pointblank_server"]
+    }
+  }
+}
+```
+
+**Option 2: User Configuration (Personal use)**
+
+1. Run command: `MCP: Open User Configuration` (Cmd/Ctrl + Shift + P)
+2. Add the same JSON configuration above
+
+> ⚠️ **Security Note**: Only add MCP servers from trusted sources. VS Code will ask you to confirm trust when starting the server for the first time.
+
+**For Positron**:
+
+1. Open Positron Settings
+2. Navigate to MCP Server configuration
+3. Add the configuration (format may vary)
+
+> **Note**: If you don't see MCP settings, you may need to install an MCP extension first. Search for "MCP" in the Extensions marketplace.
+
+### 3. Start Chatting
+
+```
+"Load my sales data and check its quality"
+```
+
+That's basically how you get started.
+
+## Essential Commands
+
+Master these four command patterns and you'll be able to handle most data validation scenarios. Think of these as your fundamental vocabulary for talking to Pointblank.
+
+### Load Data
+
+```
+"Load the file /path/to/data.csv"
+"Open my customer data from Downloads"
+"Load the Excel file with sales metrics"
+```
+
+### Check Quality
+
+```
+"Analyze the data quality"
+"What issues should I worry about?"
+"Check for missing values and duplicates"
+```
+
+### Create Data Validations
+
+```
+"Set up validation rules for this data"
+"Check that all emails are valid"
+"Make sure amounts are positive"
+```
+
+### Run and Export
+
+```
+"Run the validation"
+"Show me what failed"
+"Export problem rows to CSV"
+```
+
+---
+
+## Top Three Use Cases
+
+Once you have Pointblank set up, these scenarios show the most common and powerful ways to use natural language data validation. Each example demonstrates the conversational flow and immediate value you'll experience.
+
+### 1. 📊 Quick Data Check
+
+**When**: You get a new dataset
+
+```
+👤 "I just got customer_data.csv. Is it clean?"
+🤖 "Loading... Found 15K customers. Issues:
+    - ✅ No duplicate IDs
+    - ⚠️ 2% missing phones
+    - ❌ 234 invalid emails
+    Want me to fix the emails?"
+```
+
+### 2. 🔄 Daily Validation
+
+**When**: Regular data quality checks
+
+```
+👤 "Run our standard sales validation"
+🤖 "Running daily_sales template...
+    ✅ All checks passed!
+    3,247 transactions validated
+    Ready for dashboard update"
+```
+
+### 3. 🚨 Issue Investigation
+
+**When**: Something looks wrong
+
+```
+👤 "Conversion rates jumped to 95%. That's impossible"
+🤖 "Found the problem! Test data contamination:
+    - 1,247 test emails with 100% conversion
+    - Real rate is 4.2% (normal)
+    Should I clean the data?"
+```
+
+These examples show how natural conversation can quickly identify and resolve data quality issues that might take hours to diagnose manually.
+
+## Common Validation Rules
+
+Understanding what validation rules to ask for will help you quickly build comprehensive data quality checks. These examples cover the most frequent validation scenarios across different industries and data types.
+
+### Data Integrity
+
+- "Check for duplicate IDs"
+- "Ensure no missing required fields"
+- "Validate that dates are reasonable"
+
+### Business Logic
+
+- "Amounts must be positive"
+- "Email addresses must be valid format"
+- "Status must be active, inactive, or pending"
+
+### Cross-Field Validation
+
+- "End date must be after start date"
+- "Discount percentage between 0 and 100"
+- "Age must match birth date"
+
+These rule patterns can be combined and customized for your specific data and business requirements. The natural language interface makes it easy to express complex validation logic without learning technical syntax.
+
+## Some Tips and Tricks
+
+These recommendations will help you get more value from your Pointblank MCP server and avoid some common pitfalls.
+
+### Talk Naturally
+
+✅ **Good:** "Check if customer emails look valid"
+
+❌ **Avoid:** "Execute col_vals_regex on email column"
+
+### Provide Context
+
+✅ **Good:** "This is for the board presentation"
+
+❌ **Avoid:** Just asking for validation without explanation
+
+### Build Incrementally
+
+1. Start with data profiling
+2. Add basic validation rules
+3. Create templates for reuse
+4. Set up automated checks
+
+### Save Templates
+
+```
+"Save these rules as 'customer_validation'"
+"Apply the financial_data template"
+"Use our standard survey validation"
+```
+
+These practices help you build data quality workflows that scale with your needs while remaining accessible to those with varying technical backgrounds.
+
+## File Support
+
+Pointblank works with many major data file formats, making it easy to validate data regardless of how it's stored. This support means you can maintain consistent validation practices across your entire data ecosystem.
+
+| Type | Extensions | Example |
+|------|------------|---------|
+| **CSV** | `.csv` | `sales_data.csv` |
+| **Excel** | `.xlsx`, `.xls` | `monthly_report.xlsx` |
+| **Parquet** | `.parquet` | `big_data.parquet` |
+| **JSON** | `.json`, `.jsonl` | `api_response.json` |
+
+The consistent natural language interface works the same regardless of file format, so you can focus on validation logic rather than technical details.
+
+## Quick Troubleshooting
+
+When you encounter issues, these quick fixes resolve the most common problems. Furthermore, the natural language interface means you can always ask for help and explanations.
+
+| Problem | Quick Fix |
+|---------|-----------|
+| "File not found" | Use full file path: `/Users/name/data.csv` |
+| Validation too slow | "Use a sample for testing" |
+| Don't understand error | "Explain why validation failed" |
+| Need help | "Show me examples of data quality checks" |
+
+Remember, you can always ask the AI to explain what's happening or suggest solutions when you run into problems.
+
+## Now You're Ready!
+
+You now have everything needed to start validating data through conversation. The beauty of Pointblank's MCP server is that it grows with your expertise: start simple and gradually build more sophisticated validation workflows as you become comfortable with the interface.
+
+Start with simple commands and build up to more complex validation workflows. The AI will guide you through the process and help you create robust data quality checks!

{pointblank-0.12.1 → pointblank-0.13.0}/docs/user-guide/thresholds.qmd

@@ -86,6 +86,20 @@ thresholds_proportional = pb.Thresholds(error=0.1)
 With this configuration, the 'error' threshold would be triggered if 10% or more of the test units
 fail.
 
+### Boolean Shorthand
+
+For cases where you want to allow exactly 1 failing test unit, you can use `True` as a convenient
+shorthand:
+
+```python
+# Critical threshold of exactly 1 failing test unit
+thresholds_boolean = pb.Thresholds(critical=True)
+```
+
+This is equivalent to setting `critical=1` but provides a more intuitive way to express "allow at
+most one failure". This shorthand is particularly useful for strict validations where any failure
+beyond a single edge case should trigger immediate attention.
+
 ## Understanding Severity Levels
 
 The three threshold levels in Pointblank ('warning', 'error', and 'critical') are inspired by

{pointblank-0.12.1 → pointblank-0.13.0}/docs/user-guide/yaml-reference.qmd

@@ -56,6 +56,40 @@ tbl:
     pl.scan_csv("data.csv").filter(pl.col("date") >= "2024-01-01")
 ```
 
+#### Using Templates with `set_tbl=`
+
+For reusable validation templates that will always use a custom data source via the `set_tbl=`
+parameter in `yaml_interrogate()`, the `tbl` field is still required but its value doesn't matter
+since it will be overridden. Recommended approaches:
+
+```yaml
+# Option 1: Use a valid dataset name (gets overridden anyway)
+tbl: small_table  # Will be ignored when `set_tbl=` is used
+
+# Option 2: Use YAML null (clearest semantic intent)
+tbl: null  # Indicates table will be provided via `set_tbl=`
+```
+
+When using `yaml_interrogate()` with `set_tbl=`, the validation template becomes fully reusable:
+
+```python
+# Define reusable template
+template = """
+tbl: null  # Will be overridden
+tbl_name: "Sales Validation"
+steps:
+  - col_exists:
+      columns: [customer_id, revenue, region]
+  - col_vals_gt:
+      columns: [revenue]
+      value: 0
+"""
+
+# Apply to different datasets
+q1_result = pb.yaml_interrogate(template, set_tbl=q1_data)
+q2_result = pb.yaml_interrogate(template, set_tbl=q2_data)
+```
+
 ### DataFrame Library (`df_library`)
 
 The `df_library` key controls which DataFrame library is used to load data sources. This parameter
@@ -117,7 +151,7 @@ thresholds:
   critical: 0.15   # 15% failure rate triggers critical
 ```
 
-- values: numbers between 0 and 1 (percentages) or integers (row counts)
+- values: numbers between `0` and `1` (percentages) or integers (row counts)
 - levels: `warning`, `error`, `critical`
 
 ### Global Actions
@@ -477,6 +511,7 @@ For Pandas DataFrames (when using `df_library: pandas`):
 ```yaml
 - specially:
     expr: "lambda df: df.assign(is_valid=df['a'] + df['d'] > 0)"
+```
 
 ## Column Selection Patterns
 

{pointblank-0.12.1 → pointblank-0.13.0}/docs/user-guide/yaml-validation-workflows.qmd

@@ -167,6 +167,136 @@ tbl:
     )
 ```
 
+## Reusable Templates with `set_tbl=`
+
+One of the most powerful features of YAML validation workflows is the ability to create reusable
+templates that can be applied to different datasets. Using the `set_tbl=` parameter with
+`yaml_interrogate()`, you can define validation logic once and apply it to multiple data sources.
+
+### Creating Validation Templates
+
+When creating templates for use with `set_tbl=`, the `tbl` field is still required but its value
+will be overridden. The recommended approach is to use `tbl: null`:
+
+```yaml
+tbl: null
+tbl_name: "Sales Data Validation Template"
+label: "Standard validation checks for sales data"
+steps:
+  - col_exists:
+      columns: [customer_id, revenue, region, date]
+  - col_vals_not_null:
+      columns: [customer_id, revenue]
+  - col_vals_gt:
+      columns: [revenue]
+      value: 0
+  - col_vals_in_set:
+      columns: [region]
+      set: [North, South, East, West]
+```
+
+### Applying Templates to Multiple Datasets
+
+Here's a practical example showing how to apply the same validation template to multiple quarterly
+datasets, demonstrating the power of reusable YAML configurations:
+
+```{python}
+import pointblank as pb
+import polars as pl
+
+# Define the template once
+sales_template = """
+tbl: null  # Will be overridden
+tbl_name: "Sales Data Validation"
+label: "Standard sales validation checks"
+thresholds:
+  warning: 0.05
+  error: 0.1
+steps:
+  - col_exists:
+      columns: [customer_id, revenue, region]
+  - col_vals_not_null:
+      columns: [customer_id, revenue]
+  - col_vals_gt:
+      columns: [revenue]
+      value: 0
+  - col_vals_in_set:
+      columns: [region]
+      set: [North, South, East, West]
+"""
+
+# Create different datasets
+q1_data = pl.DataFrame({
+    "customer_id": [1, 2, 3, 4],
+    "revenue": [100, 200, 150, 300],
+    "region": ["North", "South", "East", "West"]
+})
+
+q2_data = pl.DataFrame({
+    "customer_id": [5, 6, 7, 8],
+    "revenue": [250, 180, 220, 350],
+    "region": ["South", "North", "West", "East"]
+})
+
+# Apply the same template to both datasets
+q1_result = pb.yaml_interrogate(sales_template, set_tbl=q1_data)
+q2_result = pb.yaml_interrogate(sales_template, set_tbl=q2_data)
+
+print(f"Q1 validation: {all(v.all_passed for v in q1_result.validation_info)}")
+print(f"Q2 validation: {all(v.all_passed for v in q2_result.validation_info)}")
+```
+
+### Template Best Practices
+
+1. **Use `tbl: null`**: this clearly indicates the template expects a data source to be provided
+2. **Include comprehensive metadata**: use `tbl_name`, `label`, and `brief` to make results
+self-documenting
+3. **Set appropriate thresholds**: define warning/error levels that make sense for your use case
+4. **Version control templates**: store templates in your repository alongside your data processing
+code
+5. **Test with sample data**: validate your templates work with representative datasets
+
+### Common Template Patterns
+
+For API response validation, you can ensure that responses have the expected structure and valid
+status codes:
+
+```yaml
+tbl: null
+tbl_name: "API Response Validation"
+brief: "Standard checks for API response data"
+steps:
+  - col_exists:
+      columns: [user_id, status, timestamp]
+  - col_vals_in_set:
+      columns: [status]
+      set: [success, error, pending]
+  - col_vals_not_null:
+      columns: [user_id, timestamp]
+```
+
+For file upload validation, you can check file sizes and formats to ensure they meet your
+requirements:
+
+```yaml
+tbl: null
+tbl_name: "File Upload Validation"
+steps:
+  - col_vals_gt:
+      columns: [file_size]
+      value: 0
+  - col_vals_lt:
+      columns: [file_size]
+      value: 10485760  # 10MB limit
+  - col_vals_in_set:
+      columns: [file_type]
+      set: [csv, json, xlsx, parquet]
+```
+
+This template approach is particularly valuable in data pipelines, ETL processes, and automated
+testing scenarios where you need to apply consistent validation logic across multiple similar
+datasets.
+
 ## Validation Steps
 
 YAML supports all of Pointblank's validation methods. Here are some common patterns:

{pointblank-0.12.1 → pointblank-0.13.0}/pointblank/_interrogation.py

@@ -1388,6 +1388,17 @@ class RowsDistinct:
     def get_test_results(self):
         return self.test_unit_res
 
+    def test(self):
+        # Get the number of failing test units by counting instances of `False` in the `pb_is_good_`
+        # column and then determine if the test passes overall by comparing the number of failing
+        # test units to the threshold for failing test units
+
+        results_list = nw.from_native(self.test_unit_res)["pb_is_good_"].to_list()
+
+        return _threshold_check(
+            failing_test_units=results_list.count(False), threshold=self.threshold
+        )
+
 
 @dataclass
 class RowsComplete:
@@ -2029,23 +2040,6 @@ def _column_has_null_values(table: FrameT, column: str) -> bool:
     return True
 
 
-def _check_nulls_across_columns_ibis(table, columns_subset):
-    # Get all column names from the table
-    column_names = columns_subset if columns_subset else table.columns
-
-    # Build the expression by combining each column's isnull() with OR operations
-    null_expr = functools.reduce(
-        lambda acc, col: acc | table[col].isnull() if acc is not None else table[col].isnull(),
-        column_names,
-        None,
-    )
-
-    # Add the expression as a new column to the table
-    result = table.mutate(_any_is_null_=null_expr)
-
-    return result
-
-
 def _check_nulls_across_columns_nw(table, columns_subset):
     # Get all column names from the table
     column_names = columns_subset if columns_subset else table.columns