pointblank 0.9.0__tar.gz → 0.9.1__tar.gz

{pointblank-0.9.0 → pointblank-0.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pointblank
-Version: 0.9.0
+Version: 0.9.1
 Summary: Find out if your data is what you think it is.
 Author-email: Richard Iannone <riannone@me.com>
 License: MIT License

{pointblank-0.9.0 → pointblank-0.9.1}/docs/_quarto.yml

@@ -61,10 +61,12 @@ website:
           contents:
             - user-guide/types.qmd
             - user-guide/columns.qmd
-            - user-guide/across.qmd
             - user-guide/preprocessing.qmd
+            - user-guide/segmentation.qmd
+            - user-guide/briefs.qmd
         - section: "Post Interrogation"
           contents:
+            - user-guide/step-reports.qmd
             - user-guide/extracts.qmd
             - user-guide/sundering.qmd
         - section: "Data Inspection"
@@ -126,6 +128,7 @@ quartodoc:
         - name: Validate.col_vals_expr
         - name: Validate.col_exists
         - name: Validate.rows_distinct
+        - name: Validate.rows_complete
         - name: Validate.col_schema_match
         - name: Validate.row_count_match
         - name: Validate.col_count_match

{pointblank-0.9.0 → pointblank-0.9.1}/docs/user-guide/actions.qmd

@@ -107,9 +107,18 @@ validation_2 = (
         thresholds=pb.Thresholds(warning=0.05, error=0.10, critical=0.15),
         actions=pb.Actions(critical=action_str),
     )
-    .col_vals_regex(columns="player_id", pattern=r"[A-Z]{12}\d{3}")
-    .col_vals_gt(columns="item_revenue", value=0.10)
-    .col_vals_ge(columns="session_duration", value=15)
+    .col_vals_regex(
+        columns="player_id",
+        pattern=r"[A-Z]{12}\d{3}"
+    )
+    .col_vals_gt(
+        columns="item_revenue",
+        value=0.10
+    )
+    .col_vals_ge(
+        columns="session_duration",
+        value=15
+    )
     .interrogate()
 )
 
@@ -139,8 +148,14 @@ validation_3 = (
         data=pb.load_dataset(dataset="game_revenue", tbl_type="duckdb"),
         thresholds=pb.Thresholds(warning=0.05, error=0.10, critical=0.15),
     )
-    .col_vals_regex(columns="player_id", pattern=r"[A-Z]{12}\d{3}")
-    .col_vals_gt(columns="item_revenue", value=0.05)
+    .col_vals_regex(
+        columns="player_id",
+        pattern=r"[A-Z]{12}\d{3}"
+    )
+    .col_vals_gt(
+        columns="item_revenue",
+        value=0.05
+    )
     .col_vals_gt(
         columns="session_duration",
         value=15,
@@ -191,8 +206,14 @@ validation = (
         actions=pb.Actions(default=print_problem),
         brief=True,
     )
-    .col_vals_regex(columns="player_id", pattern=r"[A-Z]{12}\d{3}")
-    .col_vals_gt(columns="item_revenue", value=0.05)
+    .col_vals_regex(
+        columns="player_id",
+        pattern=r"[A-Z]{12}\d{3}"
+    )
+    .col_vals_gt(
+        columns="item_revenue",
+        value=0.05
+    )
     .col_vals_gt(
         columns="session_duration",
         value=15,
@@ -253,8 +274,14 @@ validation_with_final = (
             send_alert               # a callable function
         )
     )
-    .col_vals_regex(columns="player_id", pattern=r"[A-Z]{12}\d{3}")
-    .col_vals_gt(columns="item_revenue", value=0.10)
+    .col_vals_regex(
+        columns="player_id",
+        pattern=r"[A-Z]{12}\d{3}"
+    )
+    .col_vals_gt(
+        columns="item_revenue",
+        value=0.10
+    )
     .interrogate()
 )
 
@@ -320,8 +347,14 @@ validation = (
         tbl_name="game_revenue",
         final_actions=pb.FinalActions(comprehensive_report),
     )
-    .col_vals_regex(columns="player_id", pattern=r"[A-Z]{12}\d{3}")
-    .col_vals_gt(columns="item_revenue", value=0.05)
+    .col_vals_regex(
+        columns="player_id",
+        pattern=r"[A-Z]{12}\d{3}"
+    )
+    .col_vals_gt(
+        columns="item_revenue",
+        value=0.05
+    )
     .interrogate()
 )
 
@@ -336,8 +369,14 @@ validation = (
         data=pb.load_dataset(dataset="game_revenue", tbl_type="duckdb"),
         tbl_name="game_revenue",
     )
-    .col_vals_regex(columns="player_id", pattern=r"[A-Z]{12}\d{3}")
-    .col_vals_gt(columns="item_revenue", value=0.05)
+    .col_vals_regex(
+        columns="player_id",
+        pattern=r"[A-Z]{12}\d{3}"
+    )
+    .col_vals_gt(
+        columns="item_revenue",
+        value=0.05
+    )
     .interrogate()
 )
 
@@ -376,8 +415,14 @@ validation_combined = (
         actions=pb.Actions(default=log_step_failure),
         final_actions=pb.FinalActions(generate_summary),
     )
-    .col_vals_regex(columns="player_id", pattern=r"[A-Z]{12}\d{3}")
-    .col_vals_gt(columns="item_revenue", value=0.05)
+    .col_vals_regex(
+        columns="player_id",
+        pattern=r"[A-Z]{12}\d{3}"
+    )
+    .col_vals_gt(
+        columns="item_revenue",
+        value=0.05
+    )
     .interrogate()
 )
 ```

pointblank-0.9.1/docs/user-guide/briefs.qmd

@@ -0,0 +1,330 @@
+---
+title: Enhancing Validation Reports with Briefs
+jupyter: python3
+html-table-processing: none
+---
+
+```{python}
+#| echo: false
+#| output: false
+import pointblank as pb
+pb.config(report_incl_footer=False)
+```
+
+When validating data with Pointblank, it's often helpful to have descriptive labels for each
+validation step. This is where *briefs* come in. A brief is a short description of what a validation
+step is checking and it appears in the `STEP` column of the validation report table. Briefs make
+your validation reports more readable and they help others understand what each step is verifying
+without needing to look at the code.
+
+Briefs can be set in two ways:
+
+1. **Globally**: applied to all validation steps via the `brief=` parameter in
+[`Validate()`](https://posit-dev.github.io/pointblank/reference/Validate.html)
+2. **Locally**: set for individual validation steps via the `brief=` parameter in each validation
+method
+
+Understanding these two approaches to adding briefs gives you flexibility in how you document your
+validation process. Global briefs provide consistency across all steps and save time when you want
+similar descriptions throughout, while step-level briefs allow for precise customization when
+specific validations need more detailed or unique explanations. In practice, many validation
+workflows will combine both approaches (i.e., setting a useful global brief template while
+overriding it for steps that require special attention).
+
+## Global Briefs
+
+To set a global brief that applies to all validation steps, use the `Validate(brief=)` parameter
+when creating a `Validate` object:
+
+```{python}
+import pointblank as pb
+import polars as pl
+
+# Sample data
+data = pl.DataFrame({
+    "id": [1, 2, 3, 4, 5],
+    "value": [10, 20, 30, 40, 50],
+    "category": ["A", "B", "C", "A", "B"]
+})
+
+# Create a validation with a global brief
+validation_1 = (
+    pb.Validate(
+        data=data,
+        tbl_name="example_data",
+        brief="Step {step}: {auto}"  # Global brief template
+    )
+    .col_vals_gt(
+        columns="value",
+        value=5
+    )
+    .col_vals_in_set(
+        columns="category",
+        set=["A", "B", "C"]
+    )
+    .interrogate()
+)
+
+validation_1
+```
+
+In this example, every validation step will have a brief description that follows the pattern
+`"Step X: [auto-generated description]"`.
+
+This is a simple example of template-based briefs. Later in this guide, we'll explore the full range
+of templating elements available for creating custom brief descriptions that precisely communicate
+what each validation step is checking.
+
+## Step-level Briefs
+
+You can also set briefs for individual validation steps:
+
+```{python}
+validation_2 = (
+    pb.Validate(data=data, tbl_name="example_data")
+    .col_vals_gt(
+        columns="value",
+        value=5,
+        brief="Check if values exceed minimum threshold of 5"
+    )
+    .col_vals_in_set(
+        columns="category",
+        set=["A", "B", "C"],
+        brief="Verify categories are valid"
+    )
+    .interrogate()
+)
+
+validation_2
+```
+
+Local briefs override any global briefs that might be set.
+
+## Brief Templating
+
+Briefs support templating elements that get replaced with specific values:
+
+- `{auto}`: an auto-generated description of the validation
+- `{step}`: the step number in the validation plan
+- `{col}`: the column name(s) being validated
+- `{value}`: the comparison value used in the validation (when applicable)
+- `{set}`: the set of values used in set-based validations
+- `{operator}`: the logical operator used in the validation (e.g., `>`, `<`, etc.)
+- `{threshold}`: the threshold values defined for the validation step (if any)
+
+Here's how to use these templates:
+
+```{python}
+validation_3 = (
+    pb.Validate(data=data, tbl_name="example_data")
+    .col_vals_gt(
+        columns="value",
+        value=5,
+        brief="Step {step}: Checking column '{col}' for values `> 5`"
+    )
+    .col_vals_in_set(
+        columns="category",
+        set=["A", "B", "C"],
+        brief="{auto} **(Step {step})**"
+    )
+    .interrogate()
+)
+
+validation_3
+```
+
+These template elements make briefs highly flexible and customizable. You can combine multiple
+templating elements in a single brief to create descriptive yet concise validation step
+descriptions. The templates help maintain consistency across your validation reports while providing
+enough detail to understand what each step is checking.
+
+Note that not all templating elements will be relevant for every validation step. For instance,
+`{set}` is only applicable to validation functions like
+[`col_vals_in_set()`](https://posit-dev.github.io/pointblank/reference/Validate.col_vals_in_set.html)
+that use a set of values, while `{value}` would be used in comparisons like
+[`col_vals_gt()`](https://posit-dev.github.io/pointblank/reference/Validate.col_vals_gt.html). If
+you include a templating element that isn't relevant to a particular step, it will simply be
+ignored.
+
+Briefs support the use of Markdown formatting, allowing you to add emphasis with **bold** or
+_italic_ text, include `inline code` formatting, or other Markdown elements to make your briefs more
+visually distinctive and informative. This can be especially helpful when you want certain parts of
+your briefs to stand out in the validation report.
+
+## Automatic Briefs
+
+If you want Pointblank to generate briefs for you automatically, you can set `brief=True`. Here,
+we'll make that setting at the global level (by using `Validate(brief=True)`):
+
+```{python}
+validation_4 = (
+    pb.Validate(
+        data=data,
+        tbl_name="example_data",
+        brief=True
+    )
+    .col_vals_gt(
+        columns="value",
+        value=5
+    )
+    .col_vals_in_set(
+        columns="category",
+        set=["A", "B", "C"]
+    )
+    .interrogate()
+)
+
+validation_4
+```
+
+Automatic briefs are descriptive and include information about what's being validated, including the
+column names and the validation conditions.
+
+## Briefs Localized to a Specified Language
+
+When using the `lang=` parameter in `Validate()`, automatically generated briefs will be created in
+the specified language (along with other elements of the validation report table):
+
+```{python}
+validation_5 = (
+    pb.Validate(
+        data=data,
+        tbl_name="example_data",
+        lang="es",  # Spanish
+        brief=True  # Auto-generate all briefs in Spanish
+    )
+    .col_vals_gt(
+        columns="value",
+        value=5
+    )
+    .col_vals_in_set(
+        columns="category",
+        set=["A", "B", "C"]
+    )
+    .interrogate()
+)
+
+validation_5
+```
+
+When using the `lang=` parameter in combination with the `{auto}` templating element, the
+auto-generated portion of the brief will also be translated to the specified language. This makes it
+possible to create fully localized validation reports where both custom text and auto-generated
+descriptions appear in the same language.
+
+Pointblank supports several languages for localized briefs, including French (`"fr"`), German
+(`"de"`), Spanish (`"es"`), Italian (`"it"`), and Portuguese (`"pt"`). For the complete list of
+supported languages, refer to the
+[`Validate()`](https://posit-dev.github.io/pointblank/reference/Validate.html) documentation.
+
+## Disabling Briefs
+
+If you've set a global brief but want to disable it for specific validation steps, you can set
+`brief=False`:
+
+```{python}
+validation_6 = (
+    pb.Validate(
+        data=data,
+        tbl_name="example_data",
+        brief="Step {step}: {auto}"  # Global brief template
+    )
+    .col_vals_gt(columns="value", value=5)  # Uses global brief
+    .col_vals_in_set(
+        columns="category",
+        set=["A", "B", "C"],
+        brief=False  # No brief for this step
+    )
+    .interrogate()
+)
+
+validation_6
+```
+
+## Best Practices for Using Briefs
+
+Well-crafted briefs can significantly enhance the readability and usefulness of your validation
+reports. Here are some guidelines to follow:
+
+1. **Be concise**: briefs should be short and to the point; they're meant to quickly communicate the
+purpose of a validation step
+
+2. **Be specific**: include relevant details or conditions that make the validation meaningful
+
+3. **Use templates consistently**: if you're using template elements like `"{step}"` or `"{col}"`,
+try to use them consistently across all briefs for a cleaner look
+
+4. **Use auto-generated briefs as a starting point**: you can start with `Validate(brief=True)` to
+see what Pointblank generates automatically, then customize as needed
+
+5. **Add custom briefs for complex validations**: custom briefs are especially useful for complex
+validations where the purpose might not be immediately obvious from the code
+
+Following these best practices will help ensure your validation reports are easy to understand for
+everyone who needs to review them.
+
+## Example: Comprehensive Validation with Briefs
+
+In real-world data validation scenarios, you'll likely work with more complex datasets and apply
+various types of validation checks. This final example brings together many of the brief-generating
+techniques we've covered, showing how you can mix different approaches in a single validation
+workflow.
+
+The example below demonstrates:
+
+- custom text briefs with direct messages
+- template-based briefs with Markdown formatting
+- automatically generated briefs
+
+By combining these different brief styles, you can create validation reports that are informative,
+consistent, and tailored to your specific data quality requirements.
+
+```{python}
+# Create a more complex dataset
+complex_data = pl.DataFrame({
+    "id": [1, 2, 3, 4, 5, 6, 7, 8],
+    "value": [10, 20, 30, 40, 50, 60, 70, 80],
+    "ratio": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8],
+    "category": ["A", "B", "C", "A", "B", "C", "A", "B"],
+    "date": ["2023-01-01", "2023-01-02", "2023-01-03", "2023-01-04",
+             "2023-01-05", "2023-01-06", "2023-01-07", "2023-01-08"]
+})
+
+# Create a comprehensive validation
+validation_7 = (
+    pb.Validate(
+        data=complex_data,
+        tbl_name="complex_data",
+        label="Complex Data Validation"
+    )
+    .col_vals_gt(
+        columns="value",
+        value=0,
+        brief="All values must be positive."
+    )
+    .col_vals_between(
+        columns="ratio",
+        left=0,
+        right=1,
+        brief="**Step {step}**: Ratios should be between `0` and `1`."
+    )
+    .col_vals_in_set(
+        columns="category",
+        set=["A", "B", "C"],
+        brief=True  # Auto-generate brief
+    )
+    .interrogate()
+)
+
+validation_7
+```
+
+## Conclusion
+
+Briefs help make validation reports more readable and understandable. By using global briefs,
+step-level briefs, or a combination of both, you can create validation reports that clearly
+communicate what each validation step is checking.
+
+Whether you want automatically generated descriptions or precisely tailored custom messages, the
+brief system provides the flexibility to make your data validation work more transparent and easier
+to interpret for all stakeholders.

{pointblank-0.9.0 → pointblank-0.9.1}/docs/user-guide/col-summary-tbl.qmd

@@ -32,6 +32,7 @@ start us off:
 import pointblank as pb
 
 small_table = pb.load_dataset(dataset="small_table", tbl_type="polars")
+
 pb.col_summary_tbl(small_table)
 ```