phlo-core-plugins 0.2.3__tar.gz → 0.2.4__tar.gz

{phlo_core_plugins-0.2.3 → phlo_core_plugins-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: phlo-core-plugins
-Version: 0.2.3
+Version: 0.2.4
 Summary: Core plugin package for Phlo
 Author-email: Phlo Team <team@phlo.dev>
 License: MIT

{phlo_core_plugins-0.2.3 → phlo_core_plugins-0.2.4}/pyproject.toml

@@ -13,7 +13,7 @@ dependencies = [
 description = "Core plugin package for Phlo"
 name = "phlo-core-plugins"
 requires-python = ">=3.11"
-version = "0.2.3"
+version = "0.2.4"
 
 [[project.authors]]
 email = "team@phlo.dev"

phlo_core_plugins-0.2.4/src/phlo_core/__init__.py

@@ -0,0 +1,42 @@
+"""Core plugins for Phlo.
+
+This package provides the foundational set of plugins bundled with Phlo,
+including quality check plugins and source connector plugins. These plugins
+are automatically available when Phlo is installed.
+
+Quality Checks:
+    - NullCheckPlugin: Validates column completeness by checking for null values.
+    - UniquenessCheckPlugin: Validates primary key uniqueness.
+    - FreshnessCheckPlugin: Validates data freshness based on timestamps.
+    - SchemaCheckPlugin: Validates column presence and data types.
+
+Source Connectors:
+    - RestAPIPlugin: Generic REST API connector for fetching data.
+
+Example:
+    To use these plugins in your Phlo project::
+
+        from phlo_core import NullCheckPlugin, RestAPIPlugin
+
+        null_check = NullCheckPlugin()
+        rest_source = RestAPIPlugin()
+
+Attributes:
+    __version__: The version string for the phlo-core-plugins package.
+
+"""
+
+from phlo_core.quality.freshness_check import FreshnessCheckPlugin
+from phlo_core.quality.null_check import NullCheckPlugin
+from phlo_core.quality.schema_check import SchemaCheckPlugin
+from phlo_core.quality.uniqueness_check import UniquenessCheckPlugin
+from phlo_core.sources.rest_api import RestAPIPlugin
+
+__all__ = [
+    "NullCheckPlugin",
+    "UniquenessCheckPlugin",
+    "FreshnessCheckPlugin",
+    "SchemaCheckPlugin",
+    "RestAPIPlugin",
+]
+__version__ = "0.2.3"

phlo_core_plugins-0.2.4/src/phlo_core/quality/__init__.py

@@ -0,0 +1,52 @@
+"""Quality check plugins bundled with Phlo.
+
+This module provides a collection of quality check plugins that can be used
+to validate data integrity, completeness, freshness, and schema conformance.
+These plugins integrate with the Phlo quality framework and can be applied to
+Pandera schemas or used directly in data pipelines.
+
+Available Plugins:
+    - NullCheckPlugin: Checks for null values in specified columns with
+        configurable thresholds.
+    - UniquenessCheckPlugin: Validates that specified columns contain unique
+        values, with optional tolerance for duplicates.
+    - FreshnessCheckPlugin: Validates that timestamped data is within an
+        acceptable age range.
+    - SchemaCheckPlugin: Validates that data conforms to an expected schema
+        with correct columns and types.
+
+Each plugin follows the QualityCheckPlugin interface and provides a
+``create_check()`` method to instantiate the actual check object.
+
+Example:
+    Import and use quality plugins::
+
+        from phlo_core.quality import NullCheckPlugin, FreshnessCheckPlugin
+
+        # Create a null check for required columns
+        null_plugin = NullCheckPlugin()
+        null_check = null_plugin.create_check(
+            columns=["id", "name", "email"],
+            allow_threshold=0.01  # Allow up to 1% nulls
+        )
+
+        # Create a freshness check for data age
+        freshness_plugin = FreshnessCheckPlugin()
+        freshness_check = freshness_plugin.create_check(
+            timestamp_column="created_at",
+            max_age_hours=24
+        )
+
+"""
+
+from phlo_core.quality.freshness_check import FreshnessCheckPlugin
+from phlo_core.quality.null_check import NullCheckPlugin
+from phlo_core.quality.schema_check import SchemaCheckPlugin
+from phlo_core.quality.uniqueness_check import UniquenessCheckPlugin
+
+__all__ = [
+    "NullCheckPlugin",
+    "UniquenessCheckPlugin",
+    "FreshnessCheckPlugin",
+    "SchemaCheckPlugin",
+]

phlo_core_plugins-0.2.4/src/phlo_core/quality/freshness_check.py

@@ -0,0 +1,155 @@
+"""Freshness check plugin for validating data timeliness.
+
+This module provides the FreshnessCheckPlugin, which enables validation of
+data freshness based on timestamp columns. It helps ensure that data is
+being updated within acceptable timeframes, critical for time-sensitive
+analytics and operational dashboards.
+
+Example:
+    Using the freshness check plugin::
+
+        from datetime import datetime, timedelta
+        from phlo_core.quality.freshness_check import FreshnessCheckPlugin
+
+        # Create the plugin
+        plugin = FreshnessCheckPlugin()
+
+        # Check that data is no more than 24 hours old
+        check = plugin.create_check(
+            timestamp_column="updated_at",
+            max_age_hours=24.0
+        )
+
+        # Check against a specific reference time
+        reference = datetime.now() - timedelta(hours=12)
+        check_with_ref = plugin.create_check(
+            timestamp_column="created_at",
+            max_age_hours=6.0,
+            reference_time=reference
+        )
+
+"""
+
+from datetime import datetime
+from typing import Any
+
+from phlo.plugins import PluginMetadata, QualityCheckPlugin
+
+
+class FreshnessCheckPlugin(QualityCheckPlugin[Any]):
+    """Plugin for performing freshness validation on timestamped data.
+
+    This plugin creates freshness check instances that validate whether data
+    is within an acceptable age based on a timestamp column. It compares
+    the maximum timestamp value in the data against a reference time (defaults
+    to current time) to ensure data is fresh enough for use.
+
+    The freshness check is particularly useful for:
+        - Monitoring data pipeline latency
+        - Ensuring dashboards show current data
+        - Detecting stale data sources
+        - Validating ETL job success
+
+    Attributes:
+        metadata: PluginMetadata containing name, version, description,
+            author, and tags for this plugin.
+
+    Example:
+        Basic freshness check with current time as reference::
+
+            from phlo_core.quality.freshness_check import FreshnessCheckPlugin
+
+            plugin = FreshnessCheckPlugin()
+            check = plugin.create_check(
+                timestamp_column="event_time",
+                max_age_hours=2.0
+            )
+
+        Freshness check with custom reference time::
+
+            from datetime import datetime, timedelta
+
+            yesterday = datetime.now() - timedelta(days=1)
+            check = plugin.create_check(
+                timestamp_column="ingested_at",
+                max_age_hours=1.0,
+                reference_time=yesterday
+            )
+
+    """
+
+    @property
+    def metadata(self) -> PluginMetadata:
+        """Return plugin metadata for the freshness-check plugin.
+
+        Returns:
+            PluginMetadata: Metadata including name ("freshness_check"),
+                version ("0.1.0"), description ("Freshness checks for timestamped data"),
+                author ("Phlo Team"), and tags (["quality", "freshness"]).
+
+        """
+        return PluginMetadata(
+            name="freshness_check",
+            version="0.1.0",
+            description="Freshness checks for timestamped data",
+            author="Phlo Team",
+            tags=["quality", "freshness"],
+        )
+
+    def create_check(
+        self,
+        timestamp_column: str,
+        max_age_hours: float,
+        reference_time: datetime | None = None,
+    ) -> Any:
+        """Create a freshness check instance.
+
+        Creates and returns a configured FreshnessCheck instance from phlo_pandera
+        that validates data freshness based on timestamps.
+
+        Args:
+            timestamp_column: Name of the timestamp column used for freshness
+                calculations. This column must exist in the data and contain
+                datetime values.
+            max_age_hours: Maximum allowed age of data in hours. If the data's
+                newest timestamp is older than this threshold relative to the
+                reference time, the check fails.
+            reference_time: Optional reference datetime for age evaluation.
+                If None, uses the current time. Useful for testing or when
+                validating against a specific point in time.
+
+        Returns:
+            Any: Configured FreshnessCheck instance ready to validate data.
+            The returned object can be used with Pandera schemas or called
+            directly with DataFrames.
+
+        Example:
+            Create a freshness check for recent data::
+
+                from phlo_core.quality.freshness_check import FreshnessCheckPlugin
+
+                plugin = FreshnessCheckPlugin()
+                check = plugin.create_check(
+                    timestamp_column="created_at",
+                    max_age_hours=24.0
+                )
+
+            Create a freshness check with custom reference::
+
+                from datetime import datetime, timedelta
+
+                check_time = datetime.now() - timedelta(hours=12)
+                check = plugin.create_check(
+                    timestamp_column="updated_at",
+                    max_age_hours=6.0,
+                    reference_time=check_time
+                )
+
+        """
+        from phlo_pandera.checks import FreshnessCheck
+
+        return FreshnessCheck(
+            timestamp_column=timestamp_column,
+            max_age_hours=max_age_hours,
+            reference_time=reference_time,
+        )

phlo_core_plugins-0.2.4/src/phlo_core/quality/null_check.py

@@ -0,0 +1,155 @@
+"""Null check plugin for validating column completeness.
+
+This module provides the NullCheckPlugin, which enables validation of
+null value presence in specified columns. It helps ensure data completeness
+by detecting missing values and enforcing thresholds for acceptable null rates.
+
+Example:
+    Using the null check plugin::
+
+        from phlo_core.quality.null_check import NullCheckPlugin
+
+        # Create the plugin
+        plugin = NullCheckPlugin()
+
+        # Strict null check (no nulls allowed)
+        strict_check = plugin.create_check(
+            columns=["id", "email", "created_at"],
+            allow_threshold=0.0
+        )
+
+        # Lenient null check (allow up to 10% nulls in optional fields)
+        lenient_check = plugin.create_check(
+            columns=["middle_name", "phone_number"],
+            allow_threshold=0.10
+        )
+
+        # Mixed columns with different requirements
+        mixed_check = plugin.create_check(
+            columns=["required_field", "optional_field"],
+            allow_threshold=0.0  # Applies to all columns
+        )
+
+"""
+
+from typing import Any
+
+from phlo.plugins import PluginMetadata, QualityCheckPlugin
+
+
+class NullCheckPlugin(QualityCheckPlugin[Any]):
+    """Plugin for performing null value validation on data columns.
+
+        This plugin creates null check instances that validate whether specified
+    columns contain null values within acceptable thresholds. It supports both
+    strict validation (no nulls allowed) and lenient validation (allows a
+    configurable percentage of nulls per column).
+
+        The null check is particularly useful for:
+            - Validating required field completeness
+            - Detecting data quality issues in ETL pipelines
+            - Enforcing data completeness SLAs
+            - Identifying sparse columns that may need attention
+
+    Attributes:
+            metadata: PluginMetadata containing name, version, description,
+                author, and tags for this plugin.
+
+    Example:
+            Strict null check for required fields::
+
+                from phlo_core.quality.null_check import NullCheckPlugin
+
+                plugin = NullCheckPlugin()
+                check = plugin.create_check(
+                    columns=["user_id", "email", "registration_date"],
+                    allow_threshold=0.0  # No nulls allowed
+                )
+
+            Lenient null check for optional fields::
+
+                check = plugin.create_check(
+                    columns=["phone", "address_line_2"],
+                    allow_threshold=0.20  # Allow up to 20% nulls
+                )
+
+            Using with Pandera schema::
+
+                import pandera as pa
+
+                schema = pa.DataFrameSchema(
+                    columns={
+                        "id": pa.Column(pa.Int64, checks=check),
+                        "name": pa.Column(pa.String, checks=check),
+                    }
+                )
+
+    """
+
+    @property
+    def metadata(self) -> PluginMetadata:
+        """Return plugin metadata for the null-check quality plugin.
+
+        Returns:
+            PluginMetadata: Metadata including name ("null_check"),
+                version ("0.1.0"), description ("Null checks for column completeness"),
+                author ("Phlo Team"), and tags (["quality", "nulls"]).
+
+        """
+        return PluginMetadata(
+            name="null_check",
+            version="0.1.0",
+            description="Null checks for column completeness",
+            author="Phlo Team",
+            tags=["quality", "nulls"],
+        )
+
+    def create_check(self, columns: list[str], allow_threshold: float = 0.0) -> Any:
+        """Create a null check instance.
+
+        Creates and returns a configured NullCheck instance from phlo_pandera
+        that validates null value presence in specified columns.
+
+        Args:
+            columns: List of column names to validate for null values.
+                Each column in the list will be checked individually for
+                null value presence against the threshold.
+            allow_threshold: Maximum allowed null ratio per column as a float
+                between 0.0 and 1.0. Defaults to 0.0 (strict validation, no
+                nulls allowed). A threshold of 0.10 allows up to 10% of values
+                in each column to be null.
+
+        Returns:
+            Any: Configured NullCheck instance ready to validate data.
+            The returned object can be used with Pandera schemas or called
+            directly with DataFrames.
+
+        Raises:
+            ValueError: If allow_threshold is not between 0.0 and 1.0.
+
+        Example:
+            Create a strict null check::
+
+                from phlo_core.quality.null_check import NullCheckPlugin
+
+                plugin = NullCheckPlugin()
+                check = plugin.create_check(
+                    columns=["customer_id", "order_date"],
+                    allow_threshold=0.0
+                )
+
+            Create a lenient null check for optional fields::
+
+                check = plugin.create_check(
+                    columns=["middle_name", "secondary_email"],
+                    allow_threshold=0.15  # 15% tolerance
+                )
+
+            Apply to DataFrame directly::
+
+                result = check.validate(df)
+
+        """
+        from phlo_pandera.checks import NullCheck
+
+        return NullCheck(columns=columns, allow_threshold=allow_threshold)

phlo_core_plugins-0.2.4/src/phlo_core/quality/schema_check.py

@@ -0,0 +1,130 @@
+"""Schema check plugin for validating data structure and types.
+
+This module provides the SchemaCheckPlugin, which enables validation of
+data against expected schemas including column presence and data type checks.
+It integrates with Pandera to provide comprehensive schema validation
+capabilities.
+
+Example:
+    Using the schema check plugin with a Pandera schema::
+
+        import pandera as pa
+        from phlo_core.quality.schema_check import SchemaCheckPlugin
+
+        # Define expected schema
+        schema = pa.DataFrameSchema({
+            "id": pa.Column(pa.Int64, nullable=False),
+            "name": pa.Column(pa.String, nullable=False),
+            "email": pa.Column(pa.String, nullable=False),
+            "created_at": pa.Column(pa.DateTime, nullable=False),
+        })
+
+        # Create the check
+        plugin = SchemaCheckPlugin()
+        check = plugin.create_check(schema=schema, lazy=True)
+
+        # Apply to data
+        validated_df = check.validate(dataframe)
+
+"""
+
+from typing import Any
+
+from phlo.plugins import PluginMetadata, QualityCheckPlugin
+
+
+class SchemaCheckPlugin(QualityCheckPlugin[Any]):
+    """Plugin for performing schema validation on data.
+
+    This plugin creates schema check instances that validate data against
+    expected column structures and data types. It supports both strict
+    validation (fails immediately) and lazy validation (collects all errors).
+
+    The schema check is particularly useful for:
+        - Validating column presence in incoming data
+        - Ensuring correct data types before processing
+        - Detecting schema drift in data pipelines
+        - Enforcing contracts between data producers and consumers
+
+    Attributes:
+        metadata: PluginMetadata containing name, version, description,
+            author, and tags for this plugin.
+
+    Example:
+        Create and use a schema check::
+
+            from phlo_core.quality.schema_check import SchemaCheckPlugin
+            import pandera as pa
+
+            plugin = SchemaCheckPlugin()
+            schema = pa.DataFrameSchema({
+                "user_id": pa.Column(pa.Int64),
+                "username": pa.Column(pa.String)
+            })
+
+            check = plugin.create_check(schema=schema, lazy=True)
+            result = check.validate(df)
+
+    """
+
+    @property
+    def metadata(self) -> PluginMetadata:
+        """Return plugin metadata for the schema-check plugin.
+
+        Returns:
+            PluginMetadata: Metadata including name ("schema_check"),
+                version ("0.1.0"), description, author ("Phlo Team"),
+                and tags (["quality", "schema"]).
+
+        """
+        return PluginMetadata(
+            name="schema_check",
+            version="0.1.0",
+            description="Schema validation for expected columns and types",
+            author="Phlo Team",
+            tags=["quality", "schema"],
+        )
+
+    def create_check(self, schema: Any, lazy: bool = True) -> Any:
+        """Create a schema check instance.
+
+        Creates and returns a configured SchemaCheck instance from phlo_pandera
+        that validates data against the provided schema.
+
+        Args:
+            schema: Expected schema object for validation. This is typically
+                a Pandera DataFrameSchema or similar schema definition that
+                defines expected columns, types, and constraints.
+            lazy: Whether to collect all validation errors before failing.
+                When True, all validation errors are collected and reported
+                together. When False, validation fails on the first error.
+                Defaults to True.
+
+        Returns:
+            Any: Configured SchemaCheck instance ready to validate data.
+            The returned object has a ``validate()`` method that accepts
+            a DataFrame and returns the validated data or raises a
+            SchemaError if validation fails.
+
+        Example:
+            Create a schema check with lazy validation::
+
+                from phlo_core.quality.schema_check import SchemaCheckPlugin
+                import pandera as pa
+
+                plugin = SchemaCheckPlugin()
+                schema = pa.DataFrameSchema({
+                    "id": pa.Column(pa.Int64, nullable=False),
+                    "value": pa.Column(pa.Float, nullable=True)
+                })
+
+                check = plugin.create_check(schema=schema, lazy=True)
+
+            Create a schema check with strict validation::
+
+                strict_check = plugin.create_check(schema=schema, lazy=False)
+
+        """
+        from phlo_pandera.checks_extra import SchemaCheck
+
+        return SchemaCheck(schema=schema, lazy=lazy)