pycharter 0.0.23__py3-none-any.whl → 0.0.25__py3-none-any.whl

pycharter/__init__.py

@@ -1,22 +1,97 @@
 """
-PyCharter - Data Contract Management and Validation
+PyCharter - Data Contract Management, ETL Pipelines, and Validation
 
 Core Services:
-1. Contract Parser - Reads and decomposes data contract files
-2. Contract Builder - Constructs consolidated contracts from separate artifacts
-3. Metadata Store - Database operations for metadata storage
-4. Pydantic Generator - Generates Pydantic models from JSON Schema
-5. JSON Schema Converter - Converts Pydantic models to JSON Schema
-6. Runtime Validator - Validation utilities
-7. Quality Assurance - Data quality checking and monitoring
-
-API Organization:
-- Tier 1: Primary Interfaces (Classes - Use these for best performance)
-- Tier 2: Convenience Functions (Quick start - one-off use cases)
-- Tier 3: Low-Level Utilities (When you already have models/schemas)
+1. ETL Pipelines - Build and run ETL pipelines with | operator
+2. Contract Parser - Reads and decomposes data contract files
+3. Contract Builder - Constructs consolidated contracts from separate artifacts
+4. Metadata Store - Database operations for metadata storage
+5. Pydantic Generator - Generates Pydantic models from JSON Schema
+6. JSON Schema Converter - Converts Pydantic models to JSON Schema
+7. Runtime Validator - Validation utilities
+8. Quality Assurance - Data quality checking and monitoring
+
+ETL API:
+    >>> from pycharter import Pipeline, HTTPExtractor, PostgresLoader, Rename, AddField
+    >>> 
+    >>> # Programmatic pipeline with | operator
+    >>> pipeline = (
+    ...     Pipeline(HTTPExtractor(url="https://api.example.com/data"))
+    ...     | Rename({"old": "new"})
+    ...     | AddField("processed_at", "now()")
+    ...     | PostgresLoader(connection_string="...", table="users")
+    ... )
+    >>> result = await pipeline.run()  # run() is async; use asyncio.run() from scripts
+    >>> 
+    >>> # Config-driven (explicit files)
+    >>> pipeline = Pipeline.from_config_files(
+    ...     extract="configs/extract.yaml",
+    ...     load="configs/load.yaml",
+    ...     variables={"API_KEY": "secret"}
+    ... )
+    >>> 
+    >>> # Config-driven (directory with extract.yaml, transform.yaml, load.yaml)
+    >>> pipeline = Pipeline.from_config_dir("pipelines/users/")
+    >>> 
+    >>> # Config-driven (single file)
+    >>> pipeline = Pipeline.from_config_file("pipelines/users/pipeline.yaml")
+
+Validator API:
+    >>> from pycharter import Validator
+    >>> 
+    >>> # From explicit files
+    >>> validator = Validator.from_files(schema="schema.yaml")
+    >>> 
+    >>> # From directory
+    >>> validator = Validator.from_dir("contracts/users/")
+    >>> 
+    >>> result = validator.validate({"name": "Alice", "age": 30})
 """
 
-__version__ = "0.0.22"
+__version__ = "0.1.0"
+
+# ============================================================================
+# ETL PIPELINES
+# ============================================================================
+
+from pycharter.etl_generator import (
+    # Core
+    Pipeline,
+    PipelineBuilder,
+    PipelineContext,
+    PipelineResult,
+    BatchResult,
+    LoadResult,
+    # Protocols
+    Extractor,
+    Transformer,
+    Loader,
+    # Extractors
+    BaseExtractor,
+    HTTPExtractor,
+    FileExtractor,
+    DatabaseExtractor,
+    CloudStorageExtractor,
+    # Transformers
+    BaseTransformer,
+    TransformerChain,
+    Rename,
+    AddField,
+    Drop,
+    Select,
+    Filter,
+    Convert,
+    Default,
+    Map,
+    FlatMap,
+    CustomFunction,
+    # Loaders
+    BaseLoader,
+    PostgresLoader,
+    DatabaseLoader,
+    FileLoader,
+    CloudStorageLoader,
+)
 
 # ============================================================================
 # TIER 1: PRIMARY INTERFACES (Classes - Use these for best performance)
@@ -25,7 +100,9 @@ __version__ = "0.0.22"
 # Runtime Validator (PRIMARY INTERFACE)
 from pycharter.runtime_validator import (
     Validator,  # ⭐ PRIMARY: Use this for validation
+    ValidatorBuilder,  # Fluent API for building validators
     create_validator,
+    QualityMetrics as ValidationQualityMetrics,  # Quality metrics from validation
 )
 
 # Quality Assurance (PRIMARY INTERFACE)
@@ -140,22 +217,145 @@ from pycharter.quality import (
     ViolationTracker,
     ViolationRecord,
     DataProfiler,
+    # Tracking submodule
+    MetricsCollector,
+    ValidationMetric,
+    MetricsSummary,
+    InMemoryMetricsStore,
+    SQLiteMetricsStore,
+)
+
+# ============================================================================
+# DOCUMENTATION GENERATION
+# ============================================================================
+
+from pycharter.docs_generator import (
+    DocsGenerator,
+    generate_docs,
+    MarkdownRenderer,
+    HTMLRenderer,
+)
+
+# ============================================================================
+# SCHEMA EVOLUTION
+# ============================================================================
+
+from pycharter.schema_evolution import (
+    check_compatibility,
+    compute_diff,
+    CompatibilityResult,
+    SchemaDiff,
+    SchemaChange,
+    ChangeType,
+    CompatibilityMode,
+)
+
+# ============================================================================
+# PROTOCOLS AND ERROR HANDLING
+# ============================================================================
+
+from pycharter.shared import (
+    # Protocols for extensibility
+    MetadataStore,
+    CoercionRegistry,
+    ValidationRegistry,
+    DataValidator,
+    # Exception hierarchy (catch PyCharterError for any pycharter failure)
+    PyCharterError,
+    ConfigError,
+    ConfigValidationError,
+    ConfigLoadError,
+    ExpressionError,
+    # Error handling
+    ErrorMode,
+    ErrorContext,
+    StrictMode,
+    LenientMode,
+    set_error_mode,
 )
 
 __all__ = [
     # ========================================================================
-    # TIER 1: PRIMARY INTERFACES
+    # ETL PIPELINES
+    # ========================================================================
+    # Core
+    "Pipeline",
+    "PipelineBuilder",
+    "PipelineContext",
+    "PipelineResult",
+    "BatchResult",
+    "LoadResult",
+    # Protocols
+    "Extractor",
+    "Transformer",
+    "Loader",
+    # Extractors
+    "BaseExtractor",
+    "HTTPExtractor",
+    "FileExtractor",
+    "DatabaseExtractor",
+    "CloudStorageExtractor",
+    # Transformers
+    "BaseTransformer",
+    "TransformerChain",
+    "Rename",
+    "AddField",
+    "Drop",
+    "Select",
+    "Filter",
+    "Convert",
+    "Default",
+    "Map",
+    "FlatMap",
+    "CustomFunction",
+    # Loaders
+    "BaseLoader",
+    "PostgresLoader",
+    "DatabaseLoader",
+    "FileLoader",
+    "CloudStorageLoader",
+    # Exceptions and error handling
+    "PyCharterError",
+    "ConfigError",
+    "ConfigValidationError",
+    "ConfigLoadError",
+    "ExpressionError",
+    "ErrorMode",
+    "ErrorContext",
+    "StrictMode",
+    "LenientMode",
+    "set_error_mode",
     # ========================================================================
-    "Validator",  # ⭐ PRIMARY: Use for validation
+    # DATA CONTRACT SERVICES
+    # ========================================================================
+    # Validation
+    "Validator",
+    "ValidatorBuilder",
     "create_validator",
-    "QualityCheck",  # ⭐ PRIMARY: Use for quality checks
+    "ValidationResult",
+    "ValidationQualityMetrics",
+    "validate_with_store",
+    "validate_batch_with_store",
+    "validate_with_contract",
+    "validate_batch_with_contract",
+    "get_model_from_store",
+    "get_model_from_contract",
+    "validate",
+    "validate_batch",
+    "validate_input",
+    "validate_output",
+    "validate_with_contract_decorator",
+    # Quality
+    "QualityCheck",
     "QualityCheckOptions",
     "QualityReport",
     "QualityThresholds",
-    "MetadataStoreClient",
-    # ========================================================================
-    # TIER 2: CONVENIENCE FUNCTIONS
-    # ========================================================================
+    "QualityMetrics",
+    "QualityScore",
+    "FieldQualityMetrics",
+    "ViolationTracker",
+    "ViolationRecord",
+    "DataProfiler",
     # Contract Management
     "parse_contract",
     "parse_contract_file",
@@ -163,49 +363,59 @@ __all__ = [
     "build_contract",
     "build_contract_from_store",
     "ContractArtifacts",
-    # Pydantic Generator (input type helpers)
-    "from_dict",  # Quick: dict → model
-    "from_file",  # Quick: file → model
-    "from_json",  # Quick: JSON string → model
-    "from_url",  # Quick: URL → model
-    "generate_model",  # Advanced: more control
+    # Pydantic Generator
+    "from_dict",
+    "from_file",
+    "from_json",
+    "from_url",
+    "generate_model",
     "generate_model_file",
-    # JSON Schema Converter (output type helpers)
-    "to_dict",  # Quick: model → dict
-    "to_file",  # Quick: model → file
-    "to_json",  # Quick: model → JSON string
-    "model_to_schema",  # Advanced: core conversion
-    # Runtime Validator (data source helpers)
-    "ValidationResult",
-    "validate_with_store",  # Quick: store → validate
-    "validate_batch_with_store",  # Quick: batch validate with store
-    "validate_with_contract",  # Quick: contract → validate
-    "validate_batch_with_contract",  # Quick: batch validate with contract
-    "get_model_from_store",  # Quick: store → model
-    "get_model_from_contract",  # Quick: contract → model
-    "validate_input",  # Decorator
-    "validate_output",  # Decorator
-    "validate_with_contract_decorator",
-    # ========================================================================
-    # TIER 3: LOW-LEVEL UTILITIES
-    # ========================================================================
-    "validate",  # Low-level: model → validate
-    "validate_batch",  # Low-level: model → batch validate
-    # ========================================================================
-    # METADATA STORE IMPLEMENTATIONS
-    # ========================================================================
+    # JSON Schema Converter
+    "to_dict",
+    "to_file",
+    "to_json",
+    "model_to_schema",
+    # Metadata Store
+    "MetadataStoreClient",
     "InMemoryMetadataStore",
     "MongoDBMetadataStore",
     "PostgresMetadataStore",
     "RedisMetadataStore",
     "SQLiteMetadataStore",
+    # Protocols
+    "MetadataStore",
+    "CoercionRegistry",
+    "ValidationRegistry",
+    "DataValidator",
+    # Error handling
+    "ErrorMode",
+    "ErrorContext",
+    "StrictMode",
+    "LenientMode",
+    "set_error_mode",
     # ========================================================================
-    # QUALITY ASSURANCE - Additional utilities
+    # QUALITY TRACKING
     # ========================================================================
-    "QualityMetrics",
-    "QualityScore",
-    "FieldQualityMetrics",
-    "ViolationTracker",
-    "ViolationRecord",
-    "DataProfiler",
+    "MetricsCollector",
+    "ValidationMetric",
+    "MetricsSummary",
+    "InMemoryMetricsStore",
+    "SQLiteMetricsStore",
+    # ========================================================================
+    # DOCUMENTATION GENERATION
+    # ========================================================================
+    "DocsGenerator",
+    "generate_docs",
+    "MarkdownRenderer",
+    "HTMLRenderer",
+    # ========================================================================
+    # SCHEMA EVOLUTION
+    # ========================================================================
+    "check_compatibility",
+    "compute_diff",
+    "CompatibilityResult",
+    "SchemaDiff",
+    "SchemaChange",
+    "ChangeType",
+    "CompatibilityMode",
 ]

pycharter/api/README.md

@@ -0,0 +1,340 @@
+# PyCharter API
+
+REST API wrapper for PyCharter services using FastAPI.
+
+## Overview
+
+The PyCharter API is located at the root level of the repository (`api/`) and provides HTTP endpoints for all core PyCharter services:
+- **Contract Parsing**: Parse data contract files into components
+- **Contract Building**: Reconstruct contracts from metadata store
+- **Metadata Storage**: Store and retrieve schemas, metadata, and rules
+- **Schema Generation**: Generate Pydantic models from JSON Schemas
+- **Schema Conversion**: Convert Pydantic models to JSON Schemas
+- **Runtime Validation**: Validate data against schemas
+- **Quality Assurance**: Run quality checks, query metrics and violations
+
+## Installation
+
+Install PyCharter with API dependencies:
+
+```bash
+pip install pycharter[api]
+```
+
+## Running the API Server
+
+### Using the CLI command:
+
+```bash
+pycharter api
+```
+
+### Using uvicorn directly:
+
+```bash
+uvicorn api.main:app --reload
+```
+
+### Using Python:
+
+```python
+from api.main import main
+main()
+```
+
+## Configuration
+
+The API requires a database connection for most endpoints. Configure it using environment variables:
+
+### Database URL Configuration
+
+The API uses `PYCHARTER_DATABASE_URL` environment variable to connect to the database. You can set it in several ways:
+
+#### Method 1: Export in Shell (Recommended)
+
+```bash
+# Set the environment variable
+export PYCHARTER_DATABASE_URL="postgresql://user:password@localhost:5432/pycharter"
+
+# Then run the API
+pycharter api
+```
+
+#### Method 2: Inline with Command
+
+```bash
+# Set and run in one line
+PYCHARTER_DATABASE_URL="postgresql://user:password@localhost:5432/pycharter" pycharter api
+```
+
+#### Method 3: Add to Shell Profile (Persistent)
+
+Add to your `~/.bashrc`, `~/.zshrc`, or `~/.profile`:
+
+```bash
+export PYCHARTER_DATABASE_URL="postgresql://user:password@localhost:5432/pycharter"
+```
+
+Then reload your shell:
+```bash
+source ~/.bashrc  # or source ~/.zshrc
+```
+
+#### Method 4: Using a .env File
+
+Create a `.env` file in your project root:
+
+```bash
+PYCHARTER_DATABASE_URL=postgresql://user:password@localhost:5432/pycharter
+```
+
+Then load it before running (requires `python-dotenv`):
+```bash
+export $(cat .env | xargs)
+pycharter api
+```
+
+Or use `dotenv-cli`:
+```bash
+pip install dotenv-cli
+dotenv run pycharter api
+```
+
+### Alternative Environment Variable
+
+PyCharter also supports Airflow-style configuration:
+
+```bash
+export PYCHARTER__DATABASE__SQL_ALCHEMY_CONN="postgresql://user:password@localhost:5432/pycharter"
+```
+
+### Metadata Store Configuration
+
+The API can be configured to use different metadata store backends:
+
+```bash
+# Metadata store type (in_memory, postgres, mongodb, redis)
+export PYCHARTER_API_STORE_TYPE=postgres
+
+# Connection string for database-backed stores
+export PYCHARTER_API_CONNECTION_STRING=postgresql://user:pass@localhost:5432/pycharter
+```
+
+**Note**: If `PYCHARTER_DATABASE_URL` is set, it will be used for both the database connection and metadata store (for PostgreSQL).
+
+## API Endpoints
+
+### Contract Endpoints
+
+#### `POST /api/v1/contracts/parse`
+Parse a data contract dictionary into components.
+
+**Request Body:**
+```json
+{
+  "contract": {
+    "schema": {...},
+    "metadata": {...},
+    "ownership": {...},
+    "governance_rules": {...}
+  }
+}
+```
+
+**Response:**
+```json
+{
+  "schema": {...},
+  "metadata": {...},
+  "ownership": {...},
+  "governance_rules": {...},
+  "coercion_rules": {...},
+  "validation_rules": {...},
+  "versions": {...}
+}
+```
+
+#### `POST /api/v1/contracts/build`
+Build a complete contract from metadata store.
+
+**Request Body:**
+```json
+{
+  "schema_id": "user_schema",
+  "version": "1.0.0",
+  "include_metadata": true,
+  "include_ownership": true,
+  "include_governance": true
+}
+```
+
+#### `GET /api/v1/contracts`
+List all data contracts stored in the database.
+
+#### `GET /api/v1/contracts/{contract_id}`
+Get a specific data contract by ID.
+
+### Metadata Endpoints
+
+#### `GET /api/v1/metadata/schemas`
+List all schemas stored in the metadata store.
+
+#### `GET /api/v1/metadata/schemas/{schema_id}`
+Get a schema by ID (optional version parameter).
+
+#### `GET /api/v1/metadata/schemas/{schema_id}/complete`
+Get complete schema with coercion and validation rules merged.
+
+#### `POST /api/v1/metadata/schemas`
+Store a schema in the metadata store.
+
+#### `GET /api/v1/metadata/metadata/{schema_id}`
+Get metadata for a schema (optional version parameter).
+
+#### `POST /api/v1/metadata/metadata`
+Store metadata for a schema.
+
+#### `GET /api/v1/metadata/coercion-rules/{schema_id}`
+Get coercion rules for a schema (optional version parameter).
+
+#### `POST /api/v1/metadata/coercion-rules`
+Store coercion rules for a schema.
+
+#### `GET /api/v1/metadata/validation-rules/{schema_id}`
+Get validation rules for a schema (optional version parameter).
+
+#### `POST /api/v1/metadata/validation-rules`
+Store validation rules for a schema.
+
+### Quality Assurance Endpoints
+
+#### `GET /api/v1/quality/metrics`
+List quality metrics (optional filtering by schema_id, pagination).
+
+#### `GET /api/v1/quality/metrics/{metric_id}`
+Get a specific quality metric by ID.
+
+#### `GET /api/v1/quality/reports/{schema_id}`
+Get quality reports for a schema/data feed (optional data_source filter).
+
+#### `POST /api/v1/quality/check`
+Run a quality check against a data contract.
+
+#### `POST /api/v1/quality/violations`
+Query data quality violations.
+
+### Schema Endpoints
+
+#### `POST /api/v1/schemas/generate`
+Generate Pydantic model from JSON Schema.
+
+#### `POST /api/v1/schemas/convert`
+Convert Pydantic model to JSON Schema.
+
+### Validation Endpoints
+
+#### `POST /api/v1/validation/validate`
+Validate a single record against a schema.
+
+#### `POST /api/v1/validation/validate-batch`
+Validate a batch of records against a schema.
+
+## API Documentation
+
+Once the API server is running, you can access:
+
+- **Swagger UI**: http://localhost:8000/docs
+- **ReDoc**: http://localhost:8000/redoc
+- **OpenAPI JSON**: http://localhost:8000/openapi.json
+
+## Example Usage
+
+### Using curl:
+
+```bash
+# List all contracts
+curl http://localhost:8000/api/v1/contracts
+
+# Get a specific contract
+curl http://localhost:8000/api/v1/contracts/{contract_id}
+
+# List all schemas
+curl http://localhost:8000/api/v1/metadata/schemas
+
+# Get a schema
+curl http://localhost:8000/api/v1/metadata/schemas/{schema_id}
+
+# Get quality metrics
+curl http://localhost:8000/api/v1/quality/metrics
+
+# Get quality report for a schema
+curl http://localhost:8000/api/v1/quality/reports/{schema_id}
+```
+
+### Using Python requests:
+
+```python
+import requests
+
+# List contracts
+response = requests.get("http://localhost:8000/api/v1/contracts")
+contracts = response.json()
+
+# Get quality metrics
+response = requests.get("http://localhost:8000/api/v1/quality/metrics")
+metrics = response.json()
+```
+
+## Troubleshooting
+
+### Database Connection Issues
+
+If you see errors about database URL not being configured:
+
+1. **Check environment variable is set:**
+   ```bash
+   echo $PYCHARTER_DATABASE_URL
+   ```
+
+2. **Verify database is accessible:**
+   ```bash
+   psql $PYCHARTER_DATABASE_URL -c "SELECT 1;"
+   ```
+
+3. **Ensure database is initialized:**
+   ```bash
+   pycharter db init $PYCHARTER_DATABASE_URL
+   ```
+
+### API Not Starting
+
+If the API fails to start:
+
+1. **Check if uvicorn is installed:**
+   ```bash
+   pip install pycharter[api]
+   ```
+
+2. **Check for port conflicts:**
+   ```bash
+   # Use a different port
+   pycharter api --port 8001
+   ```
+
+3. **Check logs for errors:**
+   The API will print error messages to stderr if there are configuration issues.
+
+## Production Deployment
+
+For production, consider:
+
+1. **Use a process manager** (systemd, supervisor, etc.)
+2. **Set environment variables** in your deployment configuration
+3. **Use a reverse proxy** (nginx, traefik) in front of the API
+4. **Enable HTTPS** for secure connections
+5. **Configure CORS** appropriately in `api/main.py`
+6. **Use a production ASGI server** like Gunicorn with Uvicorn workers:
+
+```bash
+gunicorn api.main:app -w 4 -k uvicorn.workers.UvicornWorker
+```