pycharter 0.0.24__py3-none-any.whl → 0.0.26__py3-none-any.whl

pycharter/worker/backends/spark.py

@@ -0,0 +1,233 @@
+"""
+Spark-based validation backend for large datasets.
+
+Supports:
+- Local mode: Single machine, no cluster needed (default)
+- Remote mode: Submit to external Spark service
+- Cluster mode: Connect to Spark cluster (future)
+"""
+
+import json
+import os
+import pickle
+from typing import Any, Dict, Optional
+
+from pyspark.sql import DataFrame, SparkSession
+from pyspark.sql.functions import col, udf
+from pyspark.sql.types import (
+    ArrayType,
+    BooleanType,
+    StringType,
+    StructField,
+    StructType,
+)
+
+from pycharter import build_contract_from_store, get_model_from_contract
+from pycharter.metadata_store import MetadataStoreClient
+from pycharter.runtime_validator.validator_core import ValidationResult, validate
+
+from pycharter.worker.backends.base import ValidationBackend
+
+
+class SparkValidationBackend(ValidationBackend):
+    """
+    Spark-compatible validation backend.
+
+    Works in three modes:
+    1. Local mode (default) - Single machine, no cluster needed
+    2. Remote mode - Submit to external Spark service
+    3. Cluster mode - Connect to Spark cluster (future)
+    """
+
+    def __init__(
+        self,
+        mode: str = "local",  # "local", "remote", "cluster"
+        master: Optional[str] = None,  # "local[*]", "spark://host:port", "yarn", etc.
+        app_name: str = "pycharter-validation",
+    ):
+        """
+        Initialize Spark validation backend.
+
+        Args:
+            mode: Spark mode ("local", "remote", "cluster")
+            master: Spark master URL (auto-detected if not provided)
+            app_name: Spark application name
+        """
+        self.mode = mode
+        self.master = master or self._get_default_master()
+        self.app_name = app_name
+        self.spark: Optional[SparkSession] = None
+
+    def _get_default_master(self) -> str:
+        """Get default Spark master based on mode."""
+        if self.mode == "local":
+            return "local[*]"  # Use all CPU cores locally
+        elif self.mode == "remote":
+            # For remote Spark service (e.g., Spark on Kubernetes)
+            return os.getenv("SPARK_MASTER", "spark://localhost:7077")
+        else:  # cluster
+            return os.getenv("SPARK_MASTER", "yarn")
+
+    def get_spark_session(self) -> SparkSession:
+        """Get or create Spark session."""
+        if self.spark is None:
+            self.spark = (
+                SparkSession.builder.appName(self.app_name)
+                .master(self.master)
+                .config("spark.sql.adaptive.enabled", "true")
+                .config("spark.sql.adaptive.coalescePartitions.enabled", "true")
+                .getOrCreate()
+            )
+        return self.spark
+
+    def validate(
+        self,
+        schema_id: str,
+        data_source: str,  # S3 path, file path, database table, etc.
+        store: MetadataStoreClient,
+        options: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, Any]:
+        """
+        Validate data using Spark.
+
+        This method:
+        1. Loads contract from metadata store
+        2. Reads data from source
+        3. Validates using PyCharter validation logic
+        4. Returns aggregated results
+
+        Args:
+            schema_id: Schema identifier
+            data_source: Data source path or identifier
+            store: Metadata store client
+            options: Validation options
+
+        Returns:
+            Dictionary with validation results
+        """
+        spark = self.get_spark_session()
+        options = options or {}
+
+        # Load contract from metadata store
+        contract = build_contract_from_store(store, schema_title=schema_id)
+        model = get_model_from_contract(contract)
+
+        # Serialize model for Spark UDF
+        # Note: Pydantic models need to be pickled and broadcast
+        model_pickle = pickle.dumps(model)
+        model_broadcast = spark.sparkContext.broadcast(model_pickle)
+
+        # Read data
+        df = self._read_data(spark, data_source)
+
+        # Define validation UDF
+        @udf(
+            returnType=StructType(
+                [
+                    StructField("is_valid", BooleanType()),
+                    StructField("errors", ArrayType(StringType())),
+                    StructField("data_json", StringType()),
+                ]
+            )
+        )
+        def validate_record(record_json: str) -> Dict[str, Any]:
+            """Validate a single record using PyCharter."""
+            import pickle
+
+            # Unpickle model
+            model = pickle.loads(model_broadcast.value)
+
+            # Parse record
+            record = json.loads(record_json)
+
+            # Validate using PyCharter
+            result = validate(model, record, strict=False)
+
+            # Return result
+            return {
+                "is_valid": result.is_valid,
+                "errors": result.errors or [],
+                "data_json": json.dumps(result.data.model_dump())
+                if result.data
+                else None,
+            }
+
+        # Convert DataFrame to JSON strings for validation
+        # Assuming data is already in a format we can work with
+        # If data is in columns, we need to convert to JSON
+        if "value" in df.columns:
+            # Data is already in JSON format
+            validation_df = df.withColumn(
+                "validation_result", validate_record(col("value"))
+            )
+        else:
+            # Convert row to JSON
+            from pyspark.sql.functions import to_json, struct
+
+            json_cols = [col(c) for c in df.columns]
+            df_with_json = df.withColumn("value", to_json(struct(*json_cols)))
+            validation_df = df_with_json.withColumn(
+                "validation_result", validate_record(col("value"))
+            )
+
+        # Aggregate results
+        results = validation_df.select(
+            col("validation_result.is_valid").alias("is_valid"),
+            col("validation_result.errors").alias("errors"),
+        ).collect()
+
+        # Calculate metrics
+        total_count = len(results)
+        valid_count = sum(1 for r in results if r.is_valid)
+        invalid_count = total_count - valid_count
+
+        # Collect violations
+        violations = []
+        for result in results:
+            if not result.is_valid and result.errors:
+                violations.extend(result.errors)
+
+        # Calculate quality score
+        quality_score = valid_count / total_count if total_count > 0 else 0.0
+
+        return {
+            "total_count": total_count,
+            "valid_count": valid_count,
+            "invalid_count": invalid_count,
+            "violations": violations,
+            "quality_score": quality_score,
+            "data_source": data_source,
+        }
+
+    def _read_data(self, spark: SparkSession, data_source: str) -> DataFrame:
+        """Read data from various sources."""
+        if data_source.startswith("s3://") or data_source.startswith("s3a://"):
+            # Read from S3
+            return spark.read.parquet(data_source)
+        elif data_source.startswith("hdfs://"):
+            # Read from HDFS
+            return spark.read.parquet(data_source)
+        elif data_source.endswith(".parquet"):
+            # Local parquet file
+            return spark.read.parquet(data_source)
+        elif data_source.endswith(".json"):
+            # JSON file
+            return spark.read.json(data_source)
+        elif data_source.endswith(".csv"):
+            # CSV file
+            return spark.read.csv(data_source, header=True, inferSchema=True)
+        else:
+            # Assume it's a database table or try to infer format
+            # For now, try JSON first
+            try:
+                return spark.read.json(data_source)
+            except Exception:
+                # Fallback to parquet
+                return spark.read.parquet(data_source)
+
+    def close(self):
+        """Close Spark session."""
+        if self.spark:
+            self.spark.stop()
+            self.spark = None
+

{worker → pycharter/worker}/cli.py

@@ -10,7 +10,7 @@ from typing import Optional
 
 from pycharter.config import get_database_url
 
-from worker.processor import ValidationProcessor
+from pycharter.worker.processor import ValidationProcessor
 
 logger = logging.getLogger(__name__)
 

{worker → pycharter/worker}/processor.py

@@ -16,8 +16,8 @@ from pycharter.metadata_store import (
 from pycharter.db.models.base import get_session
 from sqlalchemy.orm import Session
 
-from worker.backends.spark import SparkValidationBackend
-from worker.queue.redis_queue import ValidationJobQueue
+from pycharter.worker.backends.spark import SparkValidationBackend
+from pycharter.worker.queue.redis_queue import ValidationJobQueue
 
 logger = logging.getLogger(__name__)
 

pycharter/worker/queue/__init__.py

@@ -0,0 +1,8 @@
+"""
+Message queue implementations for worker component.
+"""
+
+from pycharter.worker.queue.redis_queue import ValidationJobQueue
+
+__all__ = ["ValidationJobQueue"]
+

pycharter/worker/queue/redis_queue.py

@@ -0,0 +1,147 @@
+"""
+Redis-based message queue for async validation jobs.
+"""
+
+import asyncio
+import json
+from datetime import datetime
+from typing import Dict, Any, Optional
+import uuid
+
+try:
+    import redis.asyncio as redis
+    REDIS_AVAILABLE = True
+except ImportError:
+    REDIS_AVAILABLE = False
+    redis = None
+
+
+class ValidationJobQueue:
+    """Message queue for async validation jobs using Redis."""
+
+    def __init__(self, redis_url: str = "redis://localhost:6379"):
+        """
+        Initialize Redis job queue.
+
+        Args:
+            redis_url: Redis connection URL
+        """
+        if not REDIS_AVAILABLE:
+            raise ImportError(
+                "redis package is required. Install with: pip install redis>=5.0.0"
+            )
+        self.redis_url = redis_url
+        self.redis_client: Optional[redis.Redis] = None
+
+    async def connect(self):
+        """Connect to Redis."""
+        if not REDIS_AVAILABLE:
+            raise ImportError("redis package is required")
+        self.redis_client = await redis.from_url(self.redis_url)
+
+    async def disconnect(self):
+        """Disconnect from Redis."""
+        if self.redis_client:
+            await self.redis_client.close()
+            self.redis_client = None
+
+    async def enqueue_job(self, job: Dict[str, Any]) -> str:
+        """
+        Enqueue a validation job.
+
+        Args:
+            job: Job dictionary with schema_id, data_source, options
+
+        Returns:
+            Job ID
+        """
+        if not self.redis_client:
+            await self.connect()
+
+        job_id = str(uuid.uuid4())
+        job["job_id"] = job_id
+        job["status"] = "queued"
+        job["created_at"] = datetime.utcnow().isoformat()
+
+        # Push to queue
+        await self.redis_client.lpush("validation-jobs", json.dumps(job))
+
+        # Store job metadata
+        await self.redis_client.setex(
+            f"job:{job_id}",
+            3600,  # 1 hour TTL
+            json.dumps({"status": "queued", "created_at": job["created_at"]}),
+        )
+
+        return job_id
+
+    async def get_job_status(self, job_id: str) -> Optional[Dict[str, Any]]:
+        """
+        Get job status.
+
+        Args:
+            job_id: Job identifier
+
+        Returns:
+            Job status dictionary or None if not found
+        """
+        if not self.redis_client:
+            await self.connect()
+
+        status_json = await self.redis_client.get(f"job:{job_id}")
+        if status_json:
+            return json.loads(status_json)
+        return None
+
+    async def update_job_status(
+        self,
+        job_id: str,
+        status: str,
+        result: Optional[Dict[str, Any]] = None,
+        error: Optional[str] = None,
+    ):
+        """
+        Update job status.
+
+        Args:
+            job_id: Job identifier
+            status: New status ("queued", "processing", "completed", "failed")
+            result: Optional result dictionary
+            error: Optional error message
+        """
+        if not self.redis_client:
+            await self.connect()
+
+        status_data = {
+            "status": status,
+            "updated_at": datetime.utcnow().isoformat(),
+        }
+        if result:
+            status_data["result"] = result
+        if error:
+            status_data["error"] = error
+
+        await self.redis_client.setex(
+            f"job:{job_id}",
+            3600,  # 1 hour TTL
+            json.dumps(status_data),
+        )
+
+    async def dequeue_job(self, timeout: int = 1) -> Optional[Dict[str, Any]]:
+        """
+        Dequeue a job from the queue (blocking).
+
+        Args:
+            timeout: Blocking timeout in seconds
+
+        Returns:
+            Job dictionary or None if timeout
+        """
+        if not self.redis_client:
+            await self.connect()
+
+        result = await self.redis_client.brpop("validation-jobs", timeout=timeout)
+        if result:
+            return json.loads(result[1])
+        return None
+

{pycharter-0.0.24.dist-info → pycharter-0.0.26.dist-info}/METADATA

@@ -1,21 +1,19 @@
 Metadata-Version: 2.4
 Name: pycharter
-Version: 0.0.24
+Version: 0.0.26
 Summary: A Python package for data contract management with five core services: contract parsing, metadata storage, Pydantic generation, JSON Schema conversion, and runtime validation
 Author-email: semantic developers <na@example.com>
 License-Expression: MIT
 Project-URL: Homepage, https://github.com/auscheng/pycharter
-Project-URL: Documentation, https://github.com/auscheng/pycharter#readme
+Project-URL: Documentation, https://auscheng.github.io/pycharter/
 Project-URL: Repository, https://github.com/auscheng/pycharter
 Project-URL: Issues, https://github.com/auscheng/pycharter/issues
 Keywords: semantic,python,pydantic,contract,validation,schema
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Requires-Python: >=3.10
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pydantic>=2.0.0
@@ -39,11 +37,18 @@ Requires-Dist: ipykernel>=6.0.0; extra == "dev"
 Requires-Dist: psycopg2-binary>=2.9.0; extra == "dev"
 Requires-Dist: pymongo>=4.0.0; extra == "dev"
 Requires-Dist: boto3>=1.26.0; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: mkdocs-material>=9.5.0; extra == "docs"
+Requires-Dist: mkdocstrings[python]>=0.24.0; extra == "docs"
+Requires-Dist: pymdown-extensions>=10.0.0; extra == "docs"
+Requires-Dist: mike>=2.0.0; extra == "docs"
 Provides-Extra: api
 Requires-Dist: fastapi>=0.104.0; extra == "api"
 Requires-Dist: uvicorn[standard]>=0.24.0; extra == "api"
 Requires-Dist: pydantic-settings>=2.0.0; extra == "api"
 Requires-Dist: python-multipart>=0.0.6; extra == "api"
+Requires-Dist: PyJWT>=2.8.0; extra == "api"
+Requires-Dist: httpx>=0.24.0; extra == "api"
 Provides-Extra: ui
 Requires-Dist: fastapi>=0.104.0; extra == "ui"
 Requires-Dist: uvicorn[standard]>=0.24.0; extra == "ui"
@@ -59,13 +64,14 @@ Requires-Dist: google-cloud-storage>=2.0.0; extra == "etl"
 Requires-Dist: azure-storage-blob>=12.0.0; extra == "etl"
 Requires-Dist: openpyxl>=3.0.0; extra == "etl"
 Requires-Dist: lxml>=4.9.0; extra == "etl"
+Requires-Dist: pymongo>=4.0.0; extra == "etl"
 Dynamic: license-file
 
 # PyCharter
 
-> **Dynamically generate Pydantic models from JSON schemas with coercion and validation support**
+> **Data contract management and validation for Python: define schemas, enforce quality, and run ETL with contracts.**
 
-[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
+[![Python 3.13+](https://img.shields.io/badge/python-3.13+-blue.svg)](https://www.python.org/downloads/)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 
@@ -162,6 +168,27 @@ pycharter ui dev     # Development mode with hot reload
 
 **Note**: When installed from pip, the UI works immediately without Node.js. For development, Node.js is required. See `ui/INSTALLATION.md` for detailed instructions.
 
+### Database setup (for metadata store, API, and seed data)
+
+If you use the **metadata store** (SQLite or PostgreSQL), the **REST API**, or **seed data**, initialize the database once:
+
+```bash
+# Initialize database schema (SQLite default: sqlite:///pycharter.db)
+pycharter db init
+
+# Seed reference data (owners, domains, systems, environments, data feeds, compliance frameworks, tags)
+pycharter db seed
+```
+
+**From the project source** (e.g. after cloning the repo), you can run the full seed script to also load example aviation contracts into the metadata store:
+
+```bash
+python bin/seed                    # seed data/seed + data/aviation_examples
+python bin/seed --skip-aviation    # seed only data/seed
+```
+
+Use `pycharter db init --force` to drop and recreate all tables (SQLite only; **destroys existing data**). See [Configuration Guide](docs/guides/configuration.md) for connection options and migrations.
+
 ## 🚀 Quick Start
 
 ### Quick Start: ETL Pipelines
@@ -398,19 +425,20 @@ artifacts = ContractArtifacts(
 
 contract = build_contract(artifacts)
 # Contract now has:
-# - schema with rules merged
+# - schema (RAW - rules NOT merged into it)
+# - coercion_rules, validation_rules (separate)
 # - metadata, ownership, governance_rules
 # - versions tracking all components
 
 # Or build from metadata store
 contract = build_contract_from_store(store, "user_schema_v1")
 
-# Use for validation
+# Use for validation - Validator merges rules internally
 from pycharter import validate_with_contract
 result = validate_with_contract(contract, {"name": "Alice", "age": "30"})
 ```
 
-**Contribution to Journey**: The contract builder is the **consolidation layer** that combines separate artifacts (stored independently in the database) into a single contract artifact. This consolidated contract tracks all component versions and can be used for runtime validation, distribution, or archival purposes.
+**Contribution to Journey**: The contract builder is the **consolidation layer** that combines separate artifacts (stored independently in the database) into a single contract artifact. The contract contains **raw schema + separate rules** (not merged). The Validator class handles merging internally during validation, keeping the contract structure clear and editable.
 
 ---
 
@@ -469,7 +497,7 @@ validation_rules = store.get_validation_rules(schema_id)
 
 **Contribution to Journey**: The metadata store is the **persistence layer** that ensures contracts and their components are versioned, searchable, and accessible across your organization. It enables governance, audit trails, and schema evolution tracking.
 
-**See [Configuration Guide](CONFIGURATION.md) for database setup and initialization instructions.**
+**See [Configuration Guide](docs/guides/configuration.md) for database setup and initialization instructions.**
 
 ---
 
@@ -808,7 +836,7 @@ def process_user_data_quick(raw_data):
 
 ---
 
-### 7. 🌐 REST API (`api/`) (`api/`)
+### 7. 🌐 REST API (`pycharter.api`)
 
 **Purpose**: Expose all PyCharter services as REST API endpoints.
 
@@ -824,11 +852,11 @@ def process_user_data_quick(raw_data):
 
 **Example**:
 ```bash
-# Start the API server
+# Start the API server (uses PYCHARTER_DATABASE_URL or sqlite:///pycharter.db)
 pycharter api
 
-# Or with uvicorn
-uvicorn api.main:app --reload
+# With host/port
+pycharter api --host 0.0.0.0 --port 8080
 ```
 
 **Endpoints**:
@@ -863,11 +891,11 @@ Each service is designed to be **independent** yet **composable**, allowing you
 
 ## 📖 Documentation
 
-- **[Data Journey Guide](DATA_JOURNEY.md)** - Complete guide to the data production journey, including both combined and separated workflows
-- **[Configuration Guide](CONFIGURATION.md)** - Database setup, connection configuration, initialization, and migration commands
-- **[Database ERD](DATABASE_ERD.md)** - Database schema documentation and entity relationship diagrams
-- **[Examples](examples/README.md)** - Complete working examples for all PyCharter services
-- **[API Documentation](api/README.md)** - REST API endpoints and usage
+- **[Configuration Guide](docs/guides/configuration.md)** - Database connection, `pycharter db init` / `upgrade` / `seed`, migrations, and variable injection
+- **[Data Journey Guide](docs/guides/data-journey.md)** - Data production journey: contract specification → storage → validation → quality
+- **[Database ERD](docs/guides/database-erd.md)** - Database schema and entity relationship diagrams
+- **[Examples & Notebooks](docs/notebooks/README.md)** - Jupyter notebooks (ETL, contracts, validation, quality, metadata store, schema conversion)
+- **[REST API](api/README.md)** - API endpoints and usage (install with `pip install pycharter[api]`)
 
 ## 📚 Usage Examples
 
@@ -1334,7 +1362,7 @@ The workflow automatically:
 make build  # Builds package (UI built automatically via setup.py)
 ```
 
-See `PUBLISHING.md` for complete documentation.
+See [Publishing guide](docs/contributing/publishing.md) for complete documentation.
 
 ## 📋 JSON Schema Compliance
 
@@ -1347,10 +1375,12 @@ PyCharter is fully compliant with **JSON Schema Draft 2020-12** standard:
 
 ## 🔗 Requirements
 
-- Python 3.10+
+- **Python 3.13+**
 - Pydantic >= 2.0.0
 - jsonschema >= 4.0.0 (optional, for enhanced validation)
 
+See [pyproject.toml](pyproject.toml) for full dependencies and optional extras (`api`, `ui`, `dev`, `etl`, etc.).
+
 ## 🤝 Contributing
 
 Contributions are welcome! Please feel free to submit a Pull Request.
@@ -1367,9 +1397,10 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
 
 ## 🔗 Links
 
-- **Repository**: [GitHub](https://github.com/auscheng/schemantic)
-- **Issues**: [GitHub Issues](https://github.com/auscheng/schemantic/issues)
-- **Documentation**: [GitHub README](https://github.com/auscheng/schemantic#readme)
+- **Homepage**: [https://github.com/auscheng/pycharter](https://github.com/auscheng/pycharter)
+- **Repository**: [GitHub](https://github.com/auscheng/pycharter)
+- **Issues**: [GitHub Issues](https://github.com/auscheng/pycharter/issues)
+- **Documentation**: [Configuration & guides](docs/guides/configuration.md) · [API docs](api/README.md)
 
 ---