focus-validator 2.1.0__tar.gz → 2.2.0__tar.gz

{focus_validator-2.1.0 → focus_validator-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: focus_validator
-Version: 2.1.0
+Version: 2.2.0
 Summary: FOCUS spec validator.
 License-File: LICENSE
 Requires-Python: >=3.12,<4.0
@@ -11,6 +11,7 @@ Classifier: Programming Language :: Python :: 3.14
 Requires-Dist: ddt (>=1.7.1,<2.0.0)
 Requires-Dist: duckdb (>=1.4.1,<2.0.0)
 Requires-Dist: graphviz (>=0.21,<0.22)
+Requires-Dist: jsonschema (>=4.25.1,<5.0.0)
 Requires-Dist: multimethod (>=2.0,<2.1)
 Requires-Dist: numpy (>=1.26,<2.0)
 Requires-Dist: pandas (>=2,<3)

{focus_validator-2.1.0 → focus_validator-2.2.0}/focus_validator/config_objects/focus_to_duckdb_converter.py

@@ -548,7 +548,9 @@ class TypeStringCheckGenerator(DuckDBCheckGenerator):
         """
 
         # Predicate SQL (for condition mode)
-        predicate_sql = f"{col} IS NOT NULL AND typeof({col}) = 'VARCHAR'"
+        predicate_sql = self._apply_condition(
+            f"{col} IS NOT NULL AND typeof({col}) = 'VARCHAR'"
+        )
 
         return SQLQuery(
             requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql
@@ -558,6 +560,42 @@ class TypeStringCheckGenerator(DuckDBCheckGenerator):
         return "type_string"
 
 
+class TypeJSONCheckGenerator(DuckDBCheckGenerator):
+    REQUIRED_KEYS = {"ColumnName"}
+
+    def generateSql(self) -> SQLQuery:
+        col = self.params.ColumnName
+        keyword = self._get_validation_keyword()
+        message = self.errorMessage or f"{col} {keyword} be of type JSON."
+        msg_sql = message.replace("'", "''")
+
+        condition = f"{col} IS NOT NULL AND NOT json_valid(CAST({col} AS VARCHAR))"
+        condition = self._apply_condition(condition)
+
+        requirement_sql = f"""
+        WITH invalid AS (
+            SELECT 1
+            FROM {{table_name}}
+            WHERE {condition}
+        )
+        SELECT
+            COUNT(*) AS violations,
+            CASE WHEN COUNT(*) > 0 THEN '{msg_sql}' END AS error_message
+        FROM invalid
+        """
+
+        predicate_sql = self._apply_condition(
+            f"{col} IS NOT NULL AND typeof({col}) = 'JSON'"
+        )
+
+        return SQLQuery(
+            requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql
+        )
+
+    def getCheckType(self) -> str:
+        return "type_json"
+
+
 class TypeDecimalCheckGenerator(DuckDBCheckGenerator):
     REQUIRED_KEYS = {"ColumnName"}
 
@@ -590,7 +628,7 @@ class TypeDecimalCheckGenerator(DuckDBCheckGenerator):
         """
 
         # Predicate SQL (for condition mode)
-        predicate_sql = (
+        predicate_sql = self._apply_condition(
             f"{col} IS NOT NULL AND typeof({col}) IN ('DECIMAL', 'DOUBLE', 'FLOAT')"
         )
 
@@ -639,7 +677,7 @@ class TypeDateTimeGenerator(DuckDBCheckGenerator):
         """
 
         # Predicate SQL (for condition mode)
-        predicate_sql = (
+        predicate_sql = self._apply_condition(
             f"{col} IS NOT NULL "
             f"AND (typeof({col}) IN ('TIMESTAMP', 'TIMESTAMP_NS', 'TIMESTAMP WITH TIME ZONE', 'DATE') "
             f"OR ({col}::TEXT ~ '^[0-9]{{4}}-[0-1][0-9]-[0-3][0-9]T[0-2][0-9]:[0-5][0-9]:[0-5][0-9]Z$'))"
@@ -684,7 +722,9 @@ class FormatNumericGenerator(DuckDBCheckGenerator):
         """
 
         # Predicate SQL (for condition mode)
-        predicate_sql = f"{col} IS NOT NULL AND (TRIM({col}::TEXT) ~ '^[+-]?([0-9]*[.])?[0-9]+([eE][+-]?[0-9]+)?$')"
+        predicate_sql = self._apply_condition(
+            f"{col} IS NOT NULL AND (TRIM({col}::TEXT) ~ '^[+-]?([0-9]*[.])?[0-9]+([eE][+-]?[0-9]+)?$')"
+        )
 
         return SQLQuery(
             requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql
@@ -741,7 +781,9 @@ class FormatStringGenerator(DuckDBCheckGenerator):
         """
 
         # Predicate SQL (for condition mode)
-        predicate_sql = f"{col} IS NOT NULL AND ({col}::TEXT ~ '^[\\x00-\\x7F]*$')"
+        predicate_sql = self._apply_condition(
+            f"{col} IS NOT NULL AND ({col}::TEXT ~ '^[\\x00-\\x7F]*$')"
+        )
 
         return SQLQuery(
             requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql
@@ -806,7 +848,7 @@ class FormatDateTimeGenerator(DuckDBCheckGenerator):
         """
 
         # Predicate SQL (for condition mode)
-        predicate_sql = (
+        predicate_sql = self._apply_condition(
             f"{col} IS NOT NULL "
             f"AND (typeof({col}) IN ('TIMESTAMP', 'TIMESTAMP_NS', 'TIMESTAMP WITH TIME ZONE', 'DATE') "
             f"OR (typeof({col}) = 'VARCHAR' AND {col}::TEXT ~ '^[0-9]{{4}}-[0-1][0-9]-[0-3][0-9]T[0-2][0-9]:[0-5][0-9]:[0-5][0-9]Z?$' "
@@ -857,7 +899,9 @@ class FormatBillingCurrencyCodeGenerator(DuckDBCheckGenerator):
         """
 
         # Predicate SQL (for condition mode)
-        predicate_sql = f"{col} IS NOT NULL AND TRIM({col}::TEXT) IN ('{codes_list}')"
+        predicate_sql = self._apply_condition(
+            f"{col} IS NOT NULL AND TRIM({col}::TEXT) IN ('{codes_list}')"
+        )
 
         return SQLQuery(
             requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql
@@ -897,7 +941,9 @@ class FormatCurrencyGenerator(DuckDBCheckGenerator):
         """
 
         # Predicate SQL (for condition mode)
-        predicate_sql = f"{col} IS NOT NULL AND (TRIM({col}::TEXT) ~ '^[A-Z]{{3}}$')"
+        predicate_sql = self._apply_condition(
+            f"{col} IS NOT NULL AND (TRIM({col}::TEXT) ~ '^[A-Z]{{3}}$')"
+        )
 
         return SQLQuery(
             requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql
@@ -1035,7 +1081,7 @@ class FormatUnitGenerator(DuckDBCheckGenerator):
         """
 
         # Predicate SQL (for condition mode)
-        predicate_sql = (
+        predicate_sql = self._apply_condition(
             f"{col} IS NOT NULL AND regexp_matches({col}, '{combined_pattern}')"
         )
 
@@ -1081,16 +1127,10 @@ class FormatJSONGenerator(DuckDBCheckGenerator):
         message = self.errorMessage or f"{col} {keyword} be valid JSON format"
         msg_sql = message.replace("'", "''")
 
-        # Requirement SQL (finds violations)
-        # Check if column is not null and either:
-        # 1. Cannot be cast to JSON, or
-        # 2. Is not a valid JSON string when treated as text
-        condition = (
-            f"{col} IS NOT NULL "
-            f"AND (TRY_CAST({col} AS JSON) IS NULL "
-            f"OR (typeof({col}) = 'VARCHAR' AND NOT json_valid({col}::TEXT)))"
+        invalid_predicate = (
+            f"{col} IS NOT NULL AND NOT json_valid(CAST({col} AS VARCHAR))"
         )
-        condition = self._apply_condition(condition)
+        condition = self._apply_condition(invalid_predicate)
 
         requirement_sql = f"""
         WITH invalid AS (
@@ -1104,11 +1144,8 @@ class FormatJSONGenerator(DuckDBCheckGenerator):
         FROM invalid
         """
 
-        # Predicate SQL (for condition mode)
-        predicate_sql = (
-            f"{col} IS NOT NULL "
-            f"AND (TRY_CAST({col} AS JSON) IS NOT NULL "
-            f"OR (typeof({col}) = 'VARCHAR' AND json_valid({col}::TEXT)))"
+        predicate_sql = self._apply_condition(
+            f"{col} IS NOT NULL AND json_valid(CAST({col} AS VARCHAR))"
         )
 
         return SQLQuery(
@@ -1119,6 +1156,178 @@ class FormatJSONGenerator(DuckDBCheckGenerator):
         return "format_json"
 
 
+class CheckJSONSchemaGenerator(DuckDBCheckGenerator):
+    REQUIRED_KEYS = {"ColumnName", "SchemaId"}
+    DEFAULTS = {"Path": "$"}
+
+    def getCheckType(self) -> str:
+        return "json_schema"
+
+    def generateSql(self) -> SQLQuery:
+        col = self.params.ColumnName
+        schema_id = self.params.SchemaId
+        keyword = self._get_validation_keyword()
+        self.errorMessage = (
+            self.errorMessage or f"{col} {keyword} conform to JSON Schema '{schema_id}'"
+        )
+        return SQLQuery(requirement_sql="SELECT 0 AS violations")
+
+    def _extract_path_value(self, payload: Any, path: str) -> Any:
+        """Extract a value from a JSON payload using a limited JSONPath subset.
+
+        Supported: '$', '$.key', '$.key.nested', '$.key[0]'.
+        Not supported: chained indices ('$.foo[0][1]'), bracket-key access
+        ('$["foo bar"]'), wildcards, or filters.
+        """
+        if path == "$":
+            return payload
+
+        if not path.startswith("$."):
+            raise InvalidRuleException(
+                f"Unsupported JSON path '{path}' for CheckJSONSchema in rule {self.rule_id}"
+            )
+
+        current = payload
+        for segment in path[2:].split("."):
+            if current is None:
+                return None
+
+            token = segment
+            while token:
+                array_match = re.match(
+                    r"^([A-Za-z_][A-Za-z0-9_]*)(\[(\d+)\])?(.*)$", token
+                )
+                if not array_match:
+                    raise InvalidRuleException(
+                        f"Unsupported JSON path segment '{segment}' for CheckJSONSchema in rule {self.rule_id}"
+                    )
+
+                key_name, _, array_idx, remainder = array_match.groups()
+                if not isinstance(current, dict):
+                    return None
+                current = current.get(key_name)
+
+                if array_idx is not None:
+                    if not isinstance(current, list):
+                        return None
+                    idx = int(array_idx)
+                    if idx >= len(current):
+                        return None
+                    current = current[idx]
+
+                token = remainder or ""
+
+        return current
+
+    def generateCheck(self) -> DuckDBColumnCheck:
+        chk = super().generateCheck()
+
+        schema_map = getattr(self.params, "schemas", None) or {}
+        schema_id = self.params.SchemaId
+        schema_entry = schema_map.get(schema_id)
+
+        if not isinstance(schema_entry, dict) or "Schema" not in schema_entry:
+            raise InvalidRuleException(
+                f"SchemaId '{schema_id}' referenced by rule {self.rule_id} was not found in model Schemas"
+            )
+
+        schema = schema_entry["Schema"]
+        path = getattr(self.params, "Path", "$")
+        col = self.params.ColumnName
+        where_clauses = [f"{col} IS NOT NULL"]
+        row_condition = (self.row_condition_sql or "").strip()
+        if row_condition:
+            where_clauses.append(f"({row_condition})")
+
+        query = f"SELECT {col} FROM {{table_name}} WHERE " + " AND ".join(where_clauses)
+
+        def _exec_json_schema(conn):
+            try:
+                from jsonschema import (  # type: ignore[import-untyped]
+                    Draft202012Validator,
+                )
+            except ModuleNotFoundError as exc:
+                raise RuntimeError(
+                    "CheckJSONSchema requires the 'jsonschema' package to be installed"
+                ) from exc
+
+            Draft202012Validator.check_schema(schema)
+            validator = Draft202012Validator(schema)
+            table_name = getattr(self.params, "table_name", "focus_data")
+            sql = query.replace("{table_name}", table_name)
+            sql = sql.replace("{table_name}", table_name)
+            try:
+                rows = conn.execute(sql).fetchall()
+            except (duckdb.BinderException, duckdb.CatalogException) as exc:
+                msg = str(exc)
+                missing = []
+                patterns = [
+                    r'Column with name ([A-Za-z0-9_"]+) does not exist',
+                    r'Referenced column "([A-Za-z0-9_]+)" not found',
+                    r'Binder Error: .*? column ([A-Za-z0-9_"]+)',
+                    r'"([A-Za-z0-9_]+)" not found',
+                ]
+                for pattern in patterns:
+                    for match in re.finditer(pattern, msg):
+                        col_name = match.group(1).strip('"')
+                        if col_name and col_name not in missing:
+                            missing.append(col_name)
+
+                missing_msg = (
+                    f"Missing columns: {', '.join(missing)}"
+                    if missing
+                    else "Missing required column(s)"
+                )
+                return False, {
+                    "violations": 1,
+                    "schema_id": schema_id,
+                    "message": f"{self.errorMessage}. {missing_msg}",
+                    "failure_reason": missing_msg,
+                    "error_type": "missing_columns",
+                }
+
+            failure_messages: list[str] = []
+            violations = 0
+            for row_num, row in enumerate(rows, start=1):
+                raw_value = row[0] if isinstance(row, (tuple, list)) else row
+                try:
+                    payload = (
+                        json.loads(raw_value)
+                        if isinstance(raw_value, str)
+                        else raw_value
+                    )
+                except Exception as exc:
+                    violations += 1
+                    failure_messages.append(f"row {row_num}: invalid JSON ({exc})")
+                    continue
+
+                instance = self._extract_path_value(payload, path)
+                errors = sorted(
+                    validator.iter_errors(instance), key=lambda err: list(err.path)
+                )
+                if errors:
+                    violations += 1
+                    failure_messages.append(f"row {row_num}: {errors[0].message}")
+
+            ok = violations == 0
+            details = {
+                "violations": violations,
+                "schema_id": schema_id,
+                "message": (
+                    self.errorMessage
+                    if ok
+                    else f"{self.errorMessage}. First error: {failure_messages[0]}"
+                ),
+            }
+            if failure_messages:
+                details["failure_messages"] = failure_messages[:5]
+            return ok, details
+
+        chk.special_executor = _exec_json_schema
+        chk.meta["special_executor_kind"] = "json_schema"
+        return chk
+
+
 class CheckValueGenerator(DuckDBCheckGenerator):
     REQUIRED_KEYS = {"ColumnName", "Value"}
 
@@ -1157,7 +1366,8 @@ class CheckValueGenerator(DuckDBCheckGenerator):
         """
 
         return SQLQuery(
-            requirement_sql=requirement_sql.strip(), predicate_sql=predicate
+            requirement_sql=requirement_sql.strip(),
+            predicate_sql=self._apply_condition(predicate),
         )
 
     def get_sample_sql(self) -> str:
@@ -1246,7 +1456,8 @@ class CheckNotValueGenerator(DuckDBCheckGenerator):
         """
 
         return SQLQuery(
-            requirement_sql=requirement_sql.strip(), predicate_sql=predicate
+            requirement_sql=requirement_sql.strip(),
+            predicate_sql=self._apply_condition(predicate),
         )
 
     def get_sample_sql(self) -> str:
@@ -1286,6 +1497,118 @@ class CheckNotValueGenerator(DuckDBCheckGenerator):
         return sql_query.get_predicate_sql()
 
 
+class CheckRegexMatchGenerator(DuckDBCheckGenerator):
+    REQUIRED_KEYS = {"ColumnName", "Pattern"}
+
+    def generateSql(self) -> SQLQuery:
+        col = self.params.ColumnName
+        pattern = self.params.Pattern
+        keyword = self._get_validation_keyword()
+        pattern_sql = str(pattern).replace("'", "''")
+        message = self.errorMessage or f"{col} {keyword} match regex '{pattern}'."
+        msg_sql = message.replace("'", "''")
+
+        condition = f"{col} IS NOT NULL AND NOT regexp_matches(CAST({col} AS VARCHAR), '{pattern_sql}')"
+        condition = self._apply_condition(condition)
+
+        requirement_sql = f"""
+        WITH invalid AS (
+            SELECT 1
+            FROM {{table_name}}
+            WHERE {condition}
+        )
+        SELECT
+            COUNT(*) AS violations,
+            CASE WHEN COUNT(*) > 0 THEN '{msg_sql}' END AS error_message
+        FROM invalid
+        """
+
+        predicate_sql = self._apply_condition(
+            f"{col} IS NOT NULL AND regexp_matches(CAST({col} AS VARCHAR), '{pattern_sql}')"
+        )
+
+        return SQLQuery(
+            requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql
+        )
+
+    def get_sample_sql(self) -> str:
+        col = self.params.ColumnName
+        pattern = self.params.Pattern
+        pattern_sql = str(pattern).replace("'", "''")
+
+        condition = f"{col} IS NOT NULL AND NOT regexp_matches(CAST({col} AS VARCHAR), '{pattern_sql}')"
+        condition = self._apply_condition(condition)
+
+        return f"""
+        SELECT {col}
+        FROM {{table_name}}
+        WHERE {condition}
+        """
+
+    @property
+    def sample_sql(self) -> str:
+        return self.get_sample_sql()
+
+    def getCheckType(self) -> str:
+        return "check_regex_match"
+
+
+class CheckStringEndsWithGenerator(DuckDBCheckGenerator):
+    REQUIRED_KEYS = {"ColumnName", "Value"}
+
+    def generateSql(self) -> SQLQuery:
+        col = self.params.ColumnName
+        value = self.params.Value
+        keyword = self._get_validation_keyword()
+        value_sql = str(value).replace("'", "''")
+        message = self.errorMessage or f"{col} {keyword} end with '{value}'."
+        msg_sql = message.replace("'", "''")
+
+        condition = f"{col} IS NOT NULL AND NOT ends_with(CAST({col} AS VARCHAR), '{value_sql}')"
+        condition = self._apply_condition(condition)
+
+        requirement_sql = f"""
+        WITH invalid AS (
+            SELECT 1
+            FROM {{table_name}}
+            WHERE {condition}
+        )
+        SELECT
+            COUNT(*) AS violations,
+            CASE WHEN COUNT(*) > 0 THEN '{msg_sql}' END AS error_message
+        FROM invalid
+        """
+
+        predicate_sql = self._apply_condition(
+            f"{col} IS NOT NULL AND ends_with(CAST({col} AS VARCHAR), '{value_sql}')"
+        )
+
+        return SQLQuery(
+            requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql
+        )
+
+    def get_sample_sql(self) -> str:
+        col = self.params.ColumnName
+        value = self.params.Value
+        value_sql = str(value).replace("'", "''")
+
+        condition = f"{col} IS NOT NULL AND NOT ends_with(CAST({col} AS VARCHAR), '{value_sql}')"
+        condition = self._apply_condition(condition)
+
+        return f"""
+        SELECT {col}
+        FROM {{table_name}}
+        WHERE {condition}
+        """
+
+    @property
+    def sample_sql(self) -> str:
+        return self.get_sample_sql()
+
+    def getCheckType(self) -> str:
+        return "check_string_ends_with"
+
+
 class CheckSameValueGenerator(DuckDBCheckGenerator):
     REQUIRED_KEYS = {"ColumnAName", "ColumnBName"}
 
@@ -1317,7 +1640,7 @@ class CheckSameValueGenerator(DuckDBCheckGenerator):
         """
 
         # Predicate SQL (for condition mode)
-        predicate_sql = (
+        predicate_sql = self._apply_condition(
             f"{col_a} IS NOT NULL AND {col_b} IS NOT NULL AND {col_a} = {col_b}"
         )
 
@@ -1395,7 +1718,7 @@ class CheckNotSameValueGenerator(DuckDBCheckGenerator):
         """
 
         # Predicate SQL (for condition mode)
-        predicate_sql = (
+        predicate_sql = self._apply_condition(
             f"{col_a} IS NOT NULL AND {col_b} IS NOT NULL AND {col_a} <> {col_b}"
         )
 
@@ -1469,7 +1792,9 @@ class ColumnByColumnEqualsColumnValueGenerator(DuckDBCheckGenerator):
         """
 
         # Predicate SQL (for condition mode)
-        predicate_sql = f"{a} IS NOT NULL AND {b} IS NOT NULL AND {r} IS NOT NULL AND ({a} * {b}) = {r}"
+        predicate_sql = self._apply_condition(
+            f"{a} IS NOT NULL AND {b} IS NOT NULL AND {r} IS NOT NULL AND ({a} * {b}) = {r}"
+        )
 
         return SQLQuery(
             requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql
@@ -1486,21 +1811,38 @@ class ColumnByColumnEqualsColumnValueGenerator(DuckDBCheckGenerator):
         return sql_query.get_predicate_sql()
 
 
-class CheckGreaterOrEqualGenerator(DuckDBCheckGenerator):
+class _CheckScalarComparisonGenerator(DuckDBCheckGenerator):
+    """Base for single-column scalar comparison checks (>=, >, <=, ...).
+
+    Subclasses differ only by operator and wording, so they set:
+      - PASS_OPERATOR: operator a valid value satisfies (e.g. ">=")
+      - VIOLATION_OPERATOR: its negation, used to find violating rows (e.g. "<")
+      - MESSAGE_PHRASE: human-readable phrase (e.g. "greater than or equal to")
+      - CHECK_TYPE: value returned by getCheckType()
+    """
+
     REQUIRED_KEYS = {"ColumnName", "Value"}
+    PASS_OPERATOR: ClassVar[str]
+    VIOLATION_OPERATOR: ClassVar[str]
+    MESSAGE_PHRASE: ClassVar[str]
+    CHECK_TYPE: ClassVar[str]
+
+    def _violation_condition(self) -> str:
+        col = self.params.ColumnName
+        val = self.params.Value
+        return f"{col} IS NOT NULL AND {col} {self.VIOLATION_OPERATOR} {self._lit(val)}"
 
     def generateSql(self) -> SQLQuery:
         col = self.params.ColumnName
         val = self.params.Value
         keyword = self._get_validation_keyword()
         message = (
-            self.errorMessage or f"{col} {keyword} be greater than or equal to {val}."
+            self.errorMessage or f"{col} {keyword} be {self.MESSAGE_PHRASE} {val}."
         )
         msg_sql = message.replace("'", "''")
 
         # Requirement SQL (finds violations)
-        condition = f"{col} IS NOT NULL AND {col} < {val}"
-        condition = self._apply_condition(condition)
+        condition = self._apply_condition(self._violation_condition())
 
         requirement_sql = f"""
         WITH invalid AS (
@@ -1515,7 +1857,9 @@ class CheckGreaterOrEqualGenerator(DuckDBCheckGenerator):
         """
 
         # Predicate SQL (for condition mode)
-        predicate_sql = f"{col} IS NOT NULL AND {col} >= {self._lit(val)}"
+        predicate_sql = self._apply_condition(
+            f"{col} IS NOT NULL AND {col} {self.PASS_OPERATOR} {self._lit(val)}"
+        )
 
         return SQLQuery(
             requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql
@@ -1524,11 +1868,7 @@ class CheckGreaterOrEqualGenerator(DuckDBCheckGenerator):
     def get_sample_sql(self) -> str:
         """Return SQL to fetch sample violating rows for display"""
         col = self.params.ColumnName
-        val = self.params.Value
-
-        # Build condition to find violating rows (values less than the required minimum)
-        condition = f"{col} IS NOT NULL AND {col} < {val}"
-        condition = self._apply_condition(condition)
+        condition = self._apply_condition(self._violation_condition())
 
         return f"""
         SELECT {col}
@@ -1542,7 +1882,7 @@ class CheckGreaterOrEqualGenerator(DuckDBCheckGenerator):
         return self.get_sample_sql()
 
     def getCheckType(self) -> str:
-        return "check_greater_equal"
+        return self.CHECK_TYPE
 
     def generatePredicate(self) -> str | None:
         """Backward compatibility wrapper"""
@@ -1552,6 +1892,88 @@ class CheckGreaterOrEqualGenerator(DuckDBCheckGenerator):
         return sql_query.get_predicate_sql()
 
 
+class CheckGreaterOrEqualGenerator(_CheckScalarComparisonGenerator):
+    PASS_OPERATOR = ">="
+    VIOLATION_OPERATOR = "<"
+    MESSAGE_PHRASE = "greater than or equal to"
+    CHECK_TYPE = "check_greater_equal"
+
+
+class CheckGreaterThanGenerator(_CheckScalarComparisonGenerator):
+    PASS_OPERATOR = ">"
+    VIOLATION_OPERATOR = "<="
+    MESSAGE_PHRASE = "greater than"
+    CHECK_TYPE = "check_greater_than"
+
+
+class CheckLessOrEqualGenerator(_CheckScalarComparisonGenerator):
+    PASS_OPERATOR = "<="
+    VIOLATION_OPERATOR = ">"
+    MESSAGE_PHRASE = "less than or equal to"
+    CHECK_TYPE = "check_less_or_equal"
+
+
+class CheckColumnComparisonGenerator(DuckDBCheckGenerator):
+    REQUIRED_KEYS = {"ColumnAName", "ColumnBName", "Comparator"}
+
+    _VALID_COMPARATORS: ClassVar[Set[str]] = {"=", "!=", "<>", ">", ">=", "<", "<="}
+
+    def generateSql(self) -> SQLQuery:
+        col_a = self.params.ColumnAName
+        col_b = self.params.ColumnBName
+        comparator = self.params.Comparator
+        keyword = self._get_validation_keyword()
+
+        if comparator not in self._VALID_COMPARATORS:
+            raise InvalidRuleException(
+                f"Unsupported comparator for {self.rule_id}: {comparator}"
+            )
+
+        message = self.errorMessage or f"{col_a} {keyword} be {comparator} {col_b}."
+        msg_sql = message.replace("'", "''")
+
+        pass_predicate = f"{col_a} IS NOT NULL AND {col_b} IS NOT NULL AND {col_a} {comparator} {col_b}"
+        condition = f"{col_a} IS NOT NULL AND {col_b} IS NOT NULL AND NOT ({col_a} {comparator} {col_b})"
+        condition = self._apply_condition(condition)
+
+        requirement_sql = f"""
+        WITH invalid AS (
+            SELECT 1
+            FROM {{table_name}}
+            WHERE {condition}
+        )
+        SELECT
+            COUNT(*) AS violations,
+            CASE WHEN COUNT(*) > 0 THEN '{msg_sql}' END AS error_message
+        FROM invalid
+        """
+
+        return SQLQuery(
+            requirement_sql=requirement_sql.strip(),
+            predicate_sql=self._apply_condition(pass_predicate),
+        )
+
+    def get_sample_sql(self) -> str:
+        col_a = self.params.ColumnAName
+        col_b = self.params.ColumnBName
+        comparator = self.params.Comparator
+        condition = f"{col_a} IS NOT NULL AND {col_b} IS NOT NULL AND NOT ({col_a} {comparator} {col_b})"
+        condition = self._apply_condition(condition)
+
+        return f"""
+        SELECT {col_a}, {col_b}
+        FROM {{table_name}}
+        WHERE {condition}
+        """
+
+    @property
+    def sample_sql(self) -> str:
+        return self.get_sample_sql()
+
+    def getCheckType(self) -> str:
+        return "check_column_comparison"
+
+
 class CheckDistinctCountGenerator(DuckDBCheckGenerator):
     REQUIRED_KEYS = {"ColumnAName", "ColumnBName", "ExpectedCount"}
 
@@ -1605,6 +2027,66 @@ class CheckDistinctCountGenerator(DuckDBCheckGenerator):
         return "distinct_count"
 
 
+class CheckNoDuplicatesGenerator(DuckDBCheckGenerator):
+    REQUIRED_KEYS = {"ColumnName"}
+
+    def generateSql(self) -> SQLQuery:
+        col = self.params.ColumnName
+        keyword = self._get_validation_keyword()
+        message = self.errorMessage or f"{col} {keyword} contain no duplicate values."
+        msg_sql = message.replace("'", "''")
+
+        where_clause = f"WHERE {col} IS NOT NULL"
+        if self.row_condition_sql and self.row_condition_sql.strip():
+            where_clause = f"WHERE ({col} IS NOT NULL) AND ({self.row_condition_sql})"
+
+        requirement_sql = f"""
+        WITH counts AS (
+            SELECT {col} AS value, COUNT(*) AS occurrences
+            FROM {{table_name}}
+            {where_clause}
+            GROUP BY {col}
+        ),
+        invalid AS (
+            SELECT value, occurrences
+            FROM counts
+            WHERE occurrences > 1
+        )
+        SELECT
+            COUNT(*) AS violations,
+            CASE WHEN COUNT(*) > 0 THEN '{msg_sql}' END AS error_message
+        FROM invalid
+        """
+
+        return SQLQuery(requirement_sql=requirement_sql.strip(), predicate_sql=None)
+
+    def get_sample_sql(self) -> str:
+        col = self.params.ColumnName
+        where_clause = f"WHERE {col} IS NOT NULL"
+        if self.row_condition_sql and self.row_condition_sql.strip():
+            where_clause = f"WHERE ({col} IS NOT NULL) AND ({self.row_condition_sql})"
+
+        return f"""
+        WITH dupes AS (
+            SELECT {col} AS value
+            FROM {{table_name}}
+            {where_clause}
+            GROUP BY {col}
+            HAVING COUNT(*) > 1
+        )
+        SELECT t.{col}
+        FROM {{table_name}} t
+        JOIN dupes d ON t.{col} = d.value
+        """
+
+    @property
+    def sample_sql(self) -> str:
+        return self.get_sample_sql()
+
+    def getCheckType(self) -> str:
+        return "check_no_duplicates"
+
+
 class CheckModelRuleGenerator(DuckDBCheckGenerator):
     REQUIRED_KEYS = {"ModelRuleId"}
 
@@ -1825,7 +2307,8 @@ class JSONCheckPathTypeGenerator(DuckDBCheckGenerator):
             """
 
         return SQLQuery(
-            requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip()
+            requirement_sql=requirement_sql.strip(),
+            predicate_sql=self._apply_condition(predicate_sql.strip()),
         )
 
     def getCheckType(self) -> str:
@@ -1947,7 +2430,8 @@ class JSONCheckPathKeyValueFormatGenerator(DuckDBCheckGenerator):
             """
 
         return SQLQuery(
-            requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip()
+            requirement_sql=requirement_sql.strip(),
+            predicate_sql=self._apply_condition(predicate_sql.strip()),
         )
 
     def getCheckType(self) -> str:
@@ -2080,7 +2564,8 @@ class JSONCheckPathKeyStartsWithGenerator(DuckDBCheckGenerator):
             """
 
         return SQLQuery(
-            requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip()
+            requirement_sql=requirement_sql.strip(),
+            predicate_sql=self._apply_condition(predicate_sql.strip()),
         )
 
     def getCheckType(self) -> str:
@@ -2213,7 +2698,8 @@ class JSONCheckPathKeyExistsGenerator(DuckDBCheckGenerator):
             """
 
         return SQLQuery(
-            requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip()
+            requirement_sql=requirement_sql.strip(),
+            predicate_sql=self._apply_condition(predicate_sql.strip()),
         )
 
     def getCheckType(self) -> str:
@@ -2384,7 +2870,8 @@ class JSONCheckPathValueGenerator(DuckDBCheckGenerator):
                 """
 
         return SQLQuery(
-            requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip()
+            requirement_sql=requirement_sql.strip(),
+            predicate_sql=self._apply_condition(predicate_sql.strip()),
         )
 
     def getCheckType(self) -> str:
@@ -2548,7 +3035,8 @@ class JSONCheckPathNotValueGenerator(DuckDBCheckGenerator):
                 """
 
         return SQLQuery(
-            requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip()
+            requirement_sql=requirement_sql.strip(),
+            predicate_sql=self._apply_condition(predicate_sql.strip()),
         )
 
     def getCheckType(self) -> str:
@@ -2843,7 +3331,8 @@ class JSONCheckPathSameValueGenerator(DuckDBCheckGenerator):
                 """
 
         return SQLQuery(
-            requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip()
+            requirement_sql=requirement_sql.strip(),
+            predicate_sql=self._apply_condition(predicate_sql.strip()),
         )
 
     def getCheckType(self) -> str:
@@ -2948,7 +3437,8 @@ class JSONCheckPathNumericFormatGenerator(DuckDBCheckGenerator):
             """
 
         return SQLQuery(
-            requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip()
+            requirement_sql=requirement_sql.strip(),
+            predicate_sql=self._apply_condition(predicate_sql.strip()),
         )
 
     def getCheckType(self) -> str:
@@ -3127,7 +3617,8 @@ class JSONCheckPathUnitFormatGenerator(DuckDBCheckGenerator):
             """
 
         return SQLQuery(
-            requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip()
+            requirement_sql=requirement_sql.strip(),
+            predicate_sql=self._apply_condition(predicate_sql.strip()),
         )
 
     def getCheckType(self) -> str:
@@ -3225,7 +3716,8 @@ class JSONCheckPathDistinctParentGenerator(DuckDBCheckGenerator):
         """
 
         return SQLQuery(
-            requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip()
+            requirement_sql=requirement_sql.strip(),
+            predicate_sql=self._apply_condition(predicate_sql.strip()),
         )
 
     def getCheckType(self) -> str:
@@ -3281,7 +3773,7 @@ class FormatJSONFormatGenerator(DuckDBCheckGenerator):
 
             return SQLQuery(
                 requirement_sql=requirement_sql.strip(),
-                predicate_sql=predicate_sql.strip(),
+                predicate_sql=self._apply_condition(predicate_sql.strip()),
             )
 
         # Path provided - validate elements at that path
@@ -3360,7 +3852,8 @@ class FormatJSONFormatGenerator(DuckDBCheckGenerator):
             """
 
         return SQLQuery(
-            requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip()
+            requirement_sql=requirement_sql.strip(),
+            predicate_sql=self._apply_condition(predicate_sql.strip()),
         )
 
     def getCheckType(self) -> str:
@@ -3465,7 +3958,8 @@ class JSONFormatStringGenerator(DuckDBCheckGenerator):
             """
 
         return SQLQuery(
-            requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip()
+            requirement_sql=requirement_sql.strip(),
+            predicate_sql=self._apply_condition(predicate_sql.strip()),
         )
 
     def getCheckType(self) -> str:
@@ -3652,7 +4146,8 @@ class JSONFormatUnitGenerator(DuckDBCheckGenerator):
             """
 
         return SQLQuery(
-            requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip()
+            requirement_sql=requirement_sql.strip(),
+            predicate_sql=self._apply_condition(predicate_sql.strip()),
         )
 
     def getCheckType(self) -> str:
@@ -3764,7 +4259,8 @@ class JSONFormatNumericGenerator(DuckDBCheckGenerator):
             """
 
         return SQLQuery(
-            requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip()
+            requirement_sql=requirement_sql.strip(),
+            predicate_sql=self._apply_condition(predicate_sql.strip()),
         )
 
     def getCheckType(self) -> str:
@@ -4343,6 +4839,10 @@ class FocusToDuckDBSchemaConverter:
             "generator": TypeStringCheckGenerator,
             "factory": lambda args: "ColumnName",
         },
+        "TypeJSON": {
+            "generator": TypeJSONCheckGenerator,
+            "factory": lambda args: "ColumnName",
+        },
         "TypeDecimal": {
             "generator": TypeDecimalCheckGenerator,
             "factory": lambda args: "ColumnName",
@@ -4367,6 +4867,10 @@ class FocusToDuckDBSchemaConverter:
             "generator": FormatBillingCurrencyCodeGenerator,
             "factory": lambda args: "ColumnName",
         },
+        "FormatJSON": {
+            "generator": FormatJSONGenerator,
+            "factory": lambda args: "ColumnName",
+        },
         "FormatKeyValue": {
             "generator": FormatJSONGenerator,
             "factory": lambda args: "ColumnName",
@@ -4375,10 +4879,18 @@ class FocusToDuckDBSchemaConverter:
             "generator": FormatCurrencyGenerator,
             "factory": lambda args: "ColumnName",
         },
+        "CheckColumnComparison": {
+            "generator": CheckColumnComparisonGenerator,
+            "factory": lambda args: "ColumnAName",
+        },
         "CheckNationalCurrency": {
             "generator": FormatBillingCurrencyCodeGenerator,
             "factory": lambda args: "ColumnName",
         },
+        "CheckGreaterThanValue": {
+            "generator": CheckGreaterThanGenerator,
+            "factory": lambda args: "ColumnName",
+        },
         "FormatUnit": {
             "generator": FormatUnitGenerator,
             "factory": lambda args: "ColumnName",
@@ -4387,10 +4899,26 @@ class FocusToDuckDBSchemaConverter:
             "generator": CheckValueGenerator,
             "factory": lambda args: "ColumnName",
         },
+        "CheckLessOrEqualThanValue": {
+            "generator": CheckLessOrEqualGenerator,
+            "factory": lambda args: "ColumnName",
+        },
         "CheckNotValue": {
             "generator": CheckNotValueGenerator,
             "factory": lambda args: "ColumnName",
         },
+        "CheckNoDuplicates": {
+            "generator": CheckNoDuplicatesGenerator,
+            "factory": lambda args: "ColumnName",
+        },
+        "CheckRegexMatch": {
+            "generator": CheckRegexMatchGenerator,
+            "factory": lambda args: "ColumnName",
+        },
+        "CheckStringEndsWith": {
+            "generator": CheckStringEndsWithGenerator,
+            "factory": lambda args: "ColumnName",
+        },
         "CheckSameValue": {
             "generator": CheckSameValueGenerator,
             "factory": lambda args: "ColumnAName",
@@ -4415,6 +4943,10 @@ class FocusToDuckDBSchemaConverter:
             "generator": CheckModelRuleGenerator,
             "factory": lambda args: "ModelRuleId",
         },
+        "CheckJSONSchema": {
+            "generator": CheckJSONSchemaGenerator,
+            "factory": lambda args: "ColumnName",
+        },
         "AND": {
             "generator": CompositeANDRuleGenerator,
             "factory": lambda args: "Items",
@@ -4595,6 +5127,7 @@ class FocusToDuckDBSchemaConverter:
         transpile_dialect: Optional[str] = None,
         show_violations: bool = False,
         rules_version: Optional[str] = None,
+        schemas: Optional[Dict[str, Any]] = None,
     ) -> None:
         self.log = logging.getLogger(f"{__name__}.{self.__class__.__qualname__}")
         self.conn: duckdb.DuckDBPyConnection | None = None
@@ -4608,6 +5141,7 @@ class FocusToDuckDBSchemaConverter:
         )
         self.show_violations = show_violations
         self.rules_version = rules_version
+        self.schemas = schemas or {}
 
         # Build the effective CHECK_GENERATORS mapping for this version
         self.CHECK_GENERATORS = self._build_check_generators_for_version(rules_version)
@@ -5304,7 +5838,7 @@ class FocusToDuckDBSchemaConverter:
         gen_cls = reg["generator"]
 
         # Strip reserved + 'CheckFunction' and pass as-is (no aliasing)
-        reserved = getattr(DuckDBCheckGenerator, "RESERVED", set()) or set()
+        reserved: set = getattr(DuckDBCheckGenerator, "RESERVED", set()) or set()
         params = {
             k: v
             for k, v in requirement.items()
@@ -5374,6 +5908,8 @@ class FocusToDuckDBSchemaConverter:
             rule_id=rule_id,
             plan=self.plan,
             conn=self.conn,
+            schemas=self.schemas,
+            table_name=self.table_name,
             parent_results_by_idx=parent_results_by_idx or {},
             parent_edges=parent_edges or (),
             row_condition_sql=row_condition_sql,
@@ -5683,7 +6219,7 @@ class FocusToDuckDBSchemaConverter:
         gen_cls = reg["generator"]
 
         # Basic required-key validation (optional)
-        required = getattr(gen_cls, "REQUIRED_KEYS", set()) or set()
+        required: set = getattr(gen_cls, "REQUIRED_KEYS", set()) or set()
         missing = [k for k in required if k not in spec]
         if missing:
             # For conditions, you can choose to return None or raise
@@ -5911,15 +6447,20 @@ class FocusToDuckDBSchemaConverter:
         # Conformance reference / special executor (no SQL)
         special = getattr(check, "special_executor", None)
         if callable(special):
+            special_kind = meta.get("special_executor_kind")
             return {
                 "rule_id": rid,
-                "type": "reference",
+                "type": "special",
                 "check_type": ctype,
                 "generator": meta.get("generator"),
                 "row_condition_sql": meta.get("row_condition_sql"),
                 "referenced": getattr(check, "referenced_rule_id", None),
                 "sql": None,  # executed by reference, not SQL
-                "note": "mirrors referenced rule outcome (no SQL)",
+                "note": (
+                    "mirrors referenced rule outcome (no SQL)"
+                    if special_kind == "reference"
+                    else "executed via special executor (no SQL)"
+                ),
                 "must_satisfy": must_satisfy,
             }
 

{focus_validator-2.1.0 → focus_validator-2.2.0}/focus_validator/data_loaders/parquet_data_loader.py

@@ -58,14 +58,23 @@ class ParquetDataLoader:
                     # Try multiple datetime parsing strategies
                     converted = None
 
+                    # A strategy succeeds only if parsing introduces no nulls
+                    # beyond those already present (so nullable columns convert),
+                    # and the column has at least one real value (an all-null
+                    # column carries no evidence of being a datetime, so it is
+                    # left as a string rather than coerced).
+                    original_null_count = series.null_count()
+                    has_values = original_null_count < series.len()
+
                     # Strategy 1: Try ISO format with timezone
                     try:
                         candidate = series.str.to_datetime(
                             format="%Y-%m-%dT%H:%M:%S%z",  # ISO with timezone like -05:00
                             strict=False,
                         )
-                        # Check if conversion was successful (all values converted)
-                        if candidate.null_count() == 0:
+                        # Accept if parsing added no new nulls (nullable columns)
+                        # and at least one value actually parsed
+                        if has_values and candidate.null_count() == original_null_count:
                             converted = candidate
                     except Exception:
                         pass
@@ -77,8 +86,12 @@ class ParquetDataLoader:
                                 format="%Y-%m-%dT%H:%M:%SZ",  # ISO with Z timezone
                                 strict=False,
                             )
-                            # Check if conversion was successful (all values converted)
-                            if candidate.null_count() == 0:
+                            # Accept if parsing added no new nulls (nullable
+                            # columns) and at least one value actually parsed
+                            if (
+                                has_values
+                                and candidate.null_count() == original_null_count
+                            ):
                                 converted = candidate
                         except Exception:
                             pass
@@ -90,8 +103,12 @@ class ParquetDataLoader:
                                 format="%Y-%m-%d %H:%M:%S",  # Space-separated format
                                 strict=False,
                             )
-                            # Check if conversion was successful (all values converted)
-                            if candidate.null_count() == 0:
+                            # Accept if parsing added no new nulls (nullable
+                            # columns) and at least one value actually parsed
+                            if (
+                                has_values
+                                and candidate.null_count() == original_null_count
+                            ):
                                 converted = candidate
                         except Exception:
                             pass
@@ -102,8 +119,12 @@ class ParquetDataLoader:
                             candidate = series.str.to_datetime(
                                 format="%Y-%m-%d", strict=False  # Simple date format
                             )
-                            # Check if conversion was successful (all values converted)
-                            if candidate.null_count() == 0:
+                            # Accept if parsing added no new nulls (nullable
+                            # columns) and at least one value actually parsed
+                            if (
+                                has_values
+                                and candidate.null_count() == original_null_count
+                            ):
                                 converted = candidate
                         except Exception:
                             pass
@@ -147,14 +168,21 @@ class ParquetDataLoader:
                                 series.name, converted_values, dtype=pl.Datetime("us")
                             )
 
-                            # Check if we successfully converted all values
-                            if candidate.null_count() == 0:
+                            # Accept if parsing added no new nulls (nullable
+                            # columns) and at least one value actually parsed
+                            if (
+                                has_values
+                                and candidate.null_count() == original_null_count
+                            ):
                                 converted = candidate
 
                         except Exception:
                             pass
 
-                    # Strategy 6: Let Polars infer format (for fallback cases)
+                    # Strategy 6: Let Polars infer the format (fallback for any
+                    # single format strategies 1-5 did not match). Format inference
+                    # cannot parse timezone-qualified ISO (trailing 'Z' or offsets),
+                    # but strategies 1-2 already cover those.
                     if converted is None:
                         try:
                             candidate = series.str.to_datetime(
@@ -163,8 +191,12 @@ class ParquetDataLoader:
                                 exact=False,  # Allow partial matches
                                 cache=True,  # Cache format inference
                             )
-                            # For auto-inference, allow some nulls but require most values to convert
-                            if candidate.null_count() < len(candidate):
+                            # Accept if parsing added no new nulls (nullable columns)
+                            # and at least one value actually parsed
+                            if (
+                                has_values
+                                and candidate.null_count() == original_null_count
+                            ):
                                 converted = candidate
                         except Exception:
                             pass

{focus_validator-2.1.0 → focus_validator-2.2.0}/focus_validator/rules/spec_rules.py

@@ -442,6 +442,7 @@ class SpecRules:
 
         self.plan = val_plan
         self.column_types = column_types
+        self.model_data = model_data
         self._meta = {
             "json_rule_file": self.json_rule_file,
             "focus_dataset": self.focus_dataset,
@@ -482,6 +483,7 @@ class SpecRules:
             transpile_dialect=self.transpile_dialect,
             show_violations=show_violations,
             rules_version=self.rules_version,
+            schemas=getattr(self, "model_data", {}).get("Schemas", {}),
         )
         # 1) Let the converter prepare schemas, UDFs, temp views, etc.
         if connection is None:
@@ -620,6 +622,7 @@ class SpecRules:
             transpile_dialect=self.transpile_dialect,
             show_violations=False,  # Not relevant for explain mode
             rules_version=self.rules_version,
+            schemas=getattr(self, "model_data", {}).get("Schemas", {}),
         )
 
         # Create a minimal connection for explain mode (converter needs it for initialization)

{focus_validator-2.1.0 → focus_validator-2.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "focus_validator"
-version = "2.1.0"
+version = "2.2.0"
 description = "FOCUS spec validator."
 authors = []
 readme = "README.md"
@@ -26,6 +26,7 @@ requests = "*"
 pandera = { version = "^0.26.1" }
 multimethod = ">=2.0,<2.1"
 sqlglot = "^27.28.1"
+jsonschema = "^4.25.1"
 numpy = { version = "^1.26"}
 pytz = "^2025.2"
 pandasql = "^0.7.3"