csv-refine 0.1.0__py3-none-any.whl

csv_refine/build_summary.py

@@ -0,0 +1,152 @@
+import datetime
+import logging
+
+from csv_refine.contract_models import Contract
+
+logger = logging.getLogger(__name__)
+
+
+def count_total_errors(validated_classified_rows: dict) -> tuple:
+    """Return the total error count and the number of rows with multiple errors."""
+    count_errors = 0
+    rows_with_multiple_errors = 0
+
+    invalid_rows = validated_classified_rows["invalid_rows"]
+
+    for invalid_row in invalid_rows:
+        count_errors += len(invalid_row["errors"])
+        if len(invalid_row["errors"]) > 1:
+            rows_with_multiple_errors += 1
+
+    return count_errors, rows_with_multiple_errors
+
+
+def collect_errors(validated_classified_rows: dict):
+    """Return the errors collected from all invalid rows."""
+    errors = []
+    invalid_rows = validated_classified_rows["invalid_rows"]
+    for invalid_row in invalid_rows:
+        errors.extend(invalid_row["errors"])
+    return errors
+
+
+def count_rules_violations(errors) -> list[tuple]:
+    """Count rule violations and return them sorted by frequency."""
+    rules_violations = {
+        "max_length": 0,
+        "min_length": 0,
+        "max": 0,
+        "min": 0,
+        "starts_with": 0,
+        "ends_with": 0,
+        "regex": 0,
+        "allowed_values": 0,
+    }
+    for error in errors:
+        if "length max" in error:
+            rules_violations["max_length"] = rules_violations["max_length"] + 1
+        if "length min" in error:
+            rules_violations["min_length"] = rules_violations["min_length"] + 1
+        if "minimum autorized" in error:
+            rules_violations["min"] = rules_violations["min"] + 1
+        if "maximum autorized" in error:
+            rules_violations["max"] = rules_violations["max"] + 1
+        if "regex" in error:
+            rules_violations["regex"] = rules_violations["regex"] + 1
+        if "startwith" in error:
+            rules_violations["starts_with"] = rules_violations["starts_with"] + 1
+        if "endswith" in error:
+            rules_violations["ends_with"] = rules_violations["ends_with"] + 1
+        if "allowed values" in error:
+            rules_violations["allowed_values"] = rules_violations["allowed_values"] + 1
+
+    return sorted(rules_violations.items(), key=lambda item: item[1], reverse=True)
+
+
+def count_errors_by_category(errors: list, rules_violations: list[tuple]):
+    """Return error counts grouped by category."""
+    count_errors_by_category = {
+        "rules_violations": sum(values[1] for values in rules_violations),
+        "conversion_errors": 0,
+        "unique_errors": 0,
+        "nullable_errors": 0,
+    }
+    for error in errors:
+        if "cannot be null" in error:
+            count_errors_by_category["nullable_errors"] = (
+                count_errors_by_category["nullable_errors"] + 1
+            )
+        if "converted or validated" in error:
+            count_errors_by_category["conversion_errors"] = (
+                count_errors_by_category["conversion_errors"] + 1
+            )
+        if "unique" in error:
+            count_errors_by_category["unique_errors"] = (
+                count_errors_by_category["unique_errors"] + 1
+            )
+
+    return count_errors_by_category
+
+
+def build_report(
+    validated_classified_rows: dict,
+    mode: str,
+    contract: Contract,
+    csv_name: str,
+    duration: float,
+):
+    """Build the validation summary report."""
+    logger.info("Start building validation report.")
+    valid_rows = len(validated_classified_rows["valid_rows"])
+    invalid_rows = len(validated_classified_rows["invalid_rows"])
+
+    total_rows = valid_rows + invalid_rows
+
+    count_errors, rows_with_multiple_errors = count_total_errors(
+        validated_classified_rows
+    )
+    collected_errors = collect_errors(validated_classified_rows)
+    rules_violations = count_rules_violations(collected_errors)
+    errors_by_category = count_errors_by_category(collected_errors, rules_violations)
+
+    if total_rows == 0:
+        valid_rows_percentage = 0
+    else:
+        valid_rows_percentage = valid_rows / total_rows * 100
+
+    if invalid_rows == 0:
+        average_errors_per_invalid_row = 0
+        status = "passed"
+    else:
+        average_errors_per_invalid_row = count_errors / invalid_rows
+        status = "completed_with_errors"
+
+    report = {
+        "status": status,
+        "input_file": csv_name,
+        "mode": mode,
+        "generated_at": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+        "columns_count": len(contract.headers),
+        "delimiter": contract.delimiter,
+        "encoding": contract.encoding,
+        "total_rows": valid_rows + invalid_rows,
+        "valid_rows": valid_rows,
+        "invalid_rows": invalid_rows,
+        "valid_rows_percentage": round(valid_rows_percentage, 2),
+        "total_errors": count_errors,
+        "rows_with_multiple_errors": rows_with_multiple_errors,
+        "average_errors_per_invalid_row": round(average_errors_per_invalid_row, 2),
+        "rules_violations": dict(rules_violations),
+        "errors_by_category": errors_by_category,
+        "processing_duration_ms": round(duration, 2),
+    }
+    logger.info(
+        logger.info(
+            f"Validation report built successfully: "
+            f"status={status}, "
+            f"valid_rows={valid_rows}, "
+            f"invalid_rows={invalid_rows}, "
+            f"total_errors={count_errors}"
+        )
+    )
+    return report

csv_refine/cli.py

@@ -0,0 +1,80 @@
+import argparse
+import logging
+import webbrowser
+from pathlib import Path
+
+from csv_refine.exceptions import YAMLContractError, CSVError
+from csv_refine.logging_config import configure_logging
+from csv_refine.orchestration import orchestration
+
+logger = logging.getLogger(__name__)
+
+
+def create_parser() -> argparse.ArgumentParser:
+    """Create and configure the CLI argument parser."""
+    parser = argparse.ArgumentParser(
+        description="A Python CLI for validating and normalizing CSV files using YAML data contracts.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "--contract", type=Path, required=True, help="Path to the YAML contract."
+    )
+
+    parser.add_argument(
+        "--csv", type=Path, required=True, help="Path to the CSV file to process."
+    )
+
+    parser.add_argument(
+        "--output",
+        help="Output directory.",
+        default="output",
+    )
+    parser.add_argument(
+        "--mode",
+        type=str,
+        choices=["strict", "permissive"],
+        help="Validation mode.",
+        default="permissive",
+    )
+    return parser
+
+
+def run_csv_refine() -> int:
+    """Run the CSV Refine CLI and return the appropriate exit code."""
+    configure_logging()
+    parser = create_parser()
+    args = parser.parse_args()
+
+    try:
+        report, html_output = orchestration(
+            yaml_contract_path=args.contract,
+            csv_path=args.csv,
+            mode=args.mode,
+            output=Path(args.output),
+        )
+
+    except YAMLContractError as error:
+        logger.error(f"Error(s) caused by contract, error(s): '{error}'.")
+        return 1
+
+    except CSVError as error:
+        logger.error(f"Error(s) caused by csv, error(s): '{error}'.")
+        return 2
+
+    webbrowser.open(html_output.resolve().as_uri())
+    print_summary(report=report)
+    return 0
+
+
+def print_summary(report: dict) -> None:
+    """Print a concise processing summary for the user."""
+    print(
+        f"Columns counted: {report['columns_count']}\n"
+        f"Total rows: {report['total_rows']}\n"
+        f"Total valid rows: {report['valid_rows']}\n"
+        f"Errors: {report['total_errors']}."
+    )
+
+
+if __name__ == "__main__":
+    raise SystemExit(run_csv_refine())