unitysvc-services 0.1.0__py3-none-any.whl

unitysvc_services/validator.py

@@ -0,0 +1,515 @@
+"""Data validation module for unitysvc_services."""
+
+import json
+import os
+import re
+import tomllib as toml
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse
+
+import typer
+from jinja2 import Environment, TemplateSyntaxError
+from jsonschema.validators import Draft7Validator
+from rich.console import Console
+
+
+class DataValidationError(Exception):
+    """Exception raised when data validation fails."""
+
+    pass
+
+
+class DataValidator:
+    """Validates data files against JSON schemas."""
+
+    def __init__(self, data_dir: Path, schema_dir: Path):
+        self.data_dir = data_dir
+        self.schema_dir = schema_dir
+        self.schemas: dict[str, dict[str, Any]] = {}
+        self.load_schemas()
+
+    def load_schemas(self) -> None:
+        """Load all JSON schemas from the schema directory."""
+        for schema_file in self.schema_dir.glob("*.json"):
+            schema_name = schema_file.stem
+            try:
+                with open(schema_file, encoding="utf-8") as f:
+                    schema = json.load(f)
+                    self.schemas[schema_name] = schema
+            except Exception as e:
+                print(f"Error loading schema {schema_file}: {e}")
+
+    def is_url(self, value: str) -> bool:
+        """Check if a string is a valid URL."""
+        try:
+            result = urlparse(value)
+            return all([result.scheme, result.netloc])
+        except Exception:
+            return False
+
+    def find_union_fields(self, schema: dict[str, Any]) -> set[str]:
+        """Find fields that are Union[str, HttpUrl] types in the schema."""
+        union_fields: set[str] = set()
+
+        def traverse_schema(obj: Any, path: str = "") -> None:
+            if isinstance(obj, dict):
+                # Check for Union type with string and URL format
+                if "anyOf" in obj:
+                    any_of = obj["anyOf"]
+                    # Count non-null items for the check
+                    non_null_items = [item for item in any_of if item.get("type") != "null"]
+                    has_plain_string = any(
+                        item.get("type") == "string" and "format" not in item for item in non_null_items
+                    )
+                    has_uri_string = any(
+                        item.get("type") == "string" and item.get("format") == "uri" for item in non_null_items
+                    )
+
+                    # Check for Union[str, HttpUrl] or Union[str, HttpUrl, None]
+                    if len(non_null_items) == 2 and has_plain_string and has_uri_string:
+                        union_fields.add(path)
+
+                # Recursively check properties
+                if "properties" in obj:
+                    for prop_name, prop_schema in obj["properties"].items():
+                        new_path = f"{path}.{prop_name}" if path else prop_name
+                        traverse_schema(prop_schema, new_path)
+
+                # Check other schema structures
+                for key, value in obj.items():
+                    if key not in ["properties", "anyOf"] and isinstance(value, dict | list):
+                        traverse_schema(value, path)
+
+            elif isinstance(obj, list):
+                for item in obj:
+                    traverse_schema(item, path)
+
+        traverse_schema(schema)
+        return union_fields
+
+    def validate_file_references(self, data: dict[str, Any], file_path: Path, union_fields: set[str]) -> list[str]:
+        """
+        Validate that file references in Union[str, HttpUrl] fields exist.
+
+        Also validates that all file_path fields use relative paths.
+        """
+        errors: list[str] = []
+
+        def check_field(obj: Any, field_path: str, current_path: str = "") -> None:
+            if isinstance(obj, dict):
+                for key, value in obj.items():
+                    new_path = f"{current_path}.{key}" if current_path else key
+
+                    # Check if this field is a Union[str, HttpUrl] field
+                    if (
+                        new_path in union_fields
+                        and value is not None
+                        and isinstance(value, str)
+                        and not self.is_url(value)
+                    ):
+                        # Empty string is not a valid file reference
+                        if value == "":
+                            errors.append(f"Empty string in field '{new_path}' is not a valid file reference or URL")
+                        # It's a file reference, must be relative path
+                        elif Path(value).is_absolute():
+                            errors.append(
+                                f"File reference '{value}' in field '{new_path}' "
+                                f"must be a relative path, not an absolute path"
+                            )
+                        else:
+                            referenced_file = file_path.parent / value
+                            if not referenced_file.exists():
+                                errors.append(
+                                    f"File reference '{value}' in field '{new_path}' "
+                                    f"does not exist at {referenced_file}"
+                                )
+
+                    # Check if this is a file_path field (regardless of schema type)
+                    if key == "file_path" and isinstance(value, str):
+                        # file_path fields must not be URLs (use external_url instead)
+                        if self.is_url(value):
+                            errors.append(
+                                f"File path '{value}' in field '{new_path}' "
+                                f"must not be a URL. Use 'external_url' field for URLs instead."
+                            )
+                        # All file_path fields must use relative paths
+                        elif Path(value).is_absolute():
+                            errors.append(
+                                f"File path '{value}' in field '{new_path}' "
+                                f"must be a relative path, not an absolute path"
+                            )
+
+                    # Recurse into nested objects
+                    if isinstance(value, dict | list):
+                        check_field(value, field_path, new_path)
+
+            elif isinstance(obj, list):
+                for i, item in enumerate(obj):
+                    if isinstance(item, dict | list):
+                        check_field(item, field_path, f"{current_path}[{i}]")
+
+        check_field(data, str(file_path))
+        return errors
+
+    def validate_name_consistency(self, data: dict[str, Any], file_path: Path, schema_name: str) -> list[str]:
+        """Validate that the name field matches the directory name."""
+        errors: list[str] = []
+
+        # Only validate files with a 'name' field
+        if "name" not in data:
+            return errors
+
+        name_value = data["name"]
+        if not isinstance(name_value, str):
+            return errors
+
+        # Determine expected directory name based on file type
+        if file_path.name in ["provider.json", "provider.toml"]:
+            # For provider.json, the directory should match the provider name
+            directory_name = file_path.parent.name
+            if self._normalize_name(name_value) != self._normalize_name(directory_name):
+                errors.append(
+                    f"Provider name '{name_value}' does not match directory name '{directory_name}'. "
+                    f"Expected directory name to match normalized provider name: '{self._normalize_name(name_value)}'"
+                )
+
+        elif file_path.name in ["service.json", "service.toml"]:
+            # For service.json, the service directory should match the service name
+            service_directory_name = file_path.parent.name
+            if self._normalize_name(name_value) != self._normalize_name(service_directory_name):
+                normalized_name = self._normalize_name(name_value)
+                errors.append(
+                    f"Service name '{name_value}' does not match "
+                    f"service directory name '{service_directory_name}'. "
+                    f"Expected service directory name to match "
+                    f"normalized service name: '{normalized_name}'"
+                )
+
+        return errors
+
+    def _normalize_name(self, name: str) -> str:
+        """Normalize a name for directory comparison."""
+        # Convert to lowercase and replace spaces/special chars with hyphens
+        normalized = re.sub(r"[^a-zA-Z0-9]+", "-", name.lower())
+        # Remove leading/trailing hyphens
+        normalized = normalized.strip("-")
+        return normalized
+
+    def load_data_file(self, file_path: Path) -> tuple[dict[str, Any] | None, list[str]]:
+        """Load data from JSON or TOML file."""
+        errors: list[str] = []
+
+        try:
+            if file_path.suffix == ".toml":
+                with open(file_path, "rb") as f:
+                    data = toml.load(f)
+            elif file_path.suffix == ".json":
+                with open(file_path, encoding="utf-8") as f:
+                    data = json.load(f)
+            else:
+                return None, [f"Unsupported file format: {file_path.suffix}"]
+            return data, errors
+        except Exception as e:
+            format_name = {".json": "JSON", ".toml": "TOML"}.get(file_path.suffix, "data")
+            return None, [f"Failed to parse {format_name}: {e}"]
+
+    def validate_data_file(self, file_path: Path) -> tuple[bool, list[str]]:
+        """Validate a single data file (JSON or TOML)."""
+        errors: list[str] = []
+
+        data, load_errors = self.load_data_file(file_path)
+        if load_errors:
+            return False, load_errors
+
+        # data could be None if loading failed
+        if data is None:
+            return False, ["Failed to load data file"]
+
+        # Check for schema field
+        if "schema" not in data:
+            return False, ["Missing 'schema' field in data file"]
+
+        schema_name = data["schema"]
+
+        # Check if schema exists
+        if schema_name not in self.schemas:
+            return False, [f"Schema '{schema_name}' not found in schema directory"]
+
+        schema = self.schemas[schema_name]
+
+        # Validate against schema with format checking enabled
+        try:
+            validator = Draft7Validator(schema, format_checker=Draft7Validator.FORMAT_CHECKER)
+            validator.check_schema(schema)  # Validate the schema itself
+            validation_errors = list(validator.iter_errors(data))
+            for error in validation_errors:
+                errors.append(f"Schema validation error: {error.message}")
+                if error.absolute_path:
+                    errors.append(f"  Path: {'.'.join(str(p) for p in error.absolute_path)}")
+        except Exception as e:
+            errors.append(f"Validation error: {e}")
+
+        # Find Union[str, HttpUrl] fields and validate file references
+        union_fields = self.find_union_fields(schema)
+        file_ref_errors = self.validate_file_references(data, file_path, union_fields)
+        errors.extend(file_ref_errors)
+
+        # Validate name consistency with directory name
+        name_errors = self.validate_name_consistency(data, file_path, schema_name)
+        errors.extend(name_errors)
+
+        return len(errors) == 0, errors
+
+    def validate_md_file(self, file_path: Path) -> tuple[bool, list[str]]:
+        """Validate a markdown file (basic existence check and Jinja2 syntax)."""
+        errors: list[str] = []
+
+        try:
+            with open(file_path, encoding="utf-8") as f:
+                content = f.read()
+
+            if not content.strip():
+                return True, []
+
+            # Validate Jinja2 syntax
+            try:
+                env = Environment()
+                env.parse(content)
+            except TemplateSyntaxError as e:
+                errors.append(f"Jinja2 syntax error: {e.message} at line {e.lineno}")
+            except Exception as e:
+                errors.append(f"Jinja2 validation error: {e}")
+
+            return len(errors) == 0, errors
+        except Exception as e:
+            return False, [f"Failed to read markdown file: {e}"]
+
+    def validate_seller_uniqueness(self) -> tuple[bool, list[str]]:
+        """
+        Validate that there is exactly one seller_v1 file in the data directory.
+
+        Each repository should have one and only one seller.json file using the seller_v1 schema.
+        """
+        errors: list[str] = []
+        seller_files: list[Path] = []
+
+        if not self.data_dir.exists():
+            return True, []
+
+        # Find all data files with seller_v1 schema
+        for file_path in self.data_dir.rglob("*"):
+            if file_path.is_file() and file_path.suffix in [".json", ".toml"]:
+                try:
+                    data, load_errors = self.load_data_file(file_path)
+                    if data and "schema" in data and data["schema"] == "seller_v1":
+                        seller_files.append(file_path.relative_to(self.data_dir))
+                except Exception:
+                    # Skip files that can't be loaded (they'll be caught by other validation)
+                    continue
+
+        # Check count
+        if len(seller_files) == 0:
+            errors.append(
+                "No seller file found. Each repository must have exactly one data file using the 'seller_v1' schema."
+            )
+        elif len(seller_files) > 1:
+            errors.append(f"Found {len(seller_files)} seller files, but only one is allowed per repository:")
+            for seller_file in seller_files:
+                errors.append(f"  - {seller_file}")
+
+        return len(errors) == 0, errors
+
+    def validate_all(self) -> dict[str, tuple[bool, list[str]]]:
+        """Validate all files in the data directory."""
+        results: dict[str, tuple[bool, list[str]]] = {}
+
+        if not self.data_dir.exists():
+            return results
+
+        # First, validate seller uniqueness (repository-level validation)
+        seller_valid, seller_errors = self.validate_seller_uniqueness()
+        if not seller_valid:
+            results["_seller_uniqueness"] = (False, seller_errors)
+
+        # Find all data and MD files recursively
+        for file_path in self.data_dir.rglob("*"):
+            if file_path.is_file() and file_path.suffix in [".json", ".toml", ".md"]:
+                relative_path = file_path.relative_to(self.data_dir)
+
+                if file_path.suffix in [".json", ".toml"]:
+                    is_valid, errors = self.validate_data_file(file_path)
+                elif file_path.suffix == ".md":
+                    is_valid, errors = self.validate_md_file(file_path)
+                else:
+                    continue
+
+                results[str(relative_path)] = (is_valid, errors)
+
+        return results
+
+    def validate_directory_data(self, directory: Path) -> None:
+        """Validate data files in a directory for consistency.
+
+        Validation rules:
+        1. All service_v1 files in same directory must have unique names
+        2. All listing_v1 files must reference a service name that exists in the same directory
+        3. If service_name is defined in listing_v1, it must match a service in the directory
+
+        Args:
+            directory: Directory containing data files to validate
+
+        Raises:
+            DataValidationError: If validation fails
+        """
+        # Find all JSON and TOML files in the directory (not recursive)
+        data_files: list[Path] = []
+        for pattern in ["*.json", "*.toml"]:
+            data_files.extend(directory.glob(pattern))
+
+        # Load all files and categorize by schema
+        services: dict[str, Path] = {}  # name -> file_path
+        listings: list[tuple[Path, dict[str, Any]]] = []  # list of (file_path, data)
+
+        for file_path in data_files:
+            try:
+                data, load_errors = self.load_data_file(file_path)
+                if load_errors or data is None:
+                    continue
+
+                schema = data.get("schema")
+
+                if schema == "service_v1":
+                    service_name = data.get("name")
+                    if not service_name:
+                        raise DataValidationError(f"Service file {file_path} missing 'name' field")
+
+                    # Check for duplicate service names in same directory
+                    if service_name in services:
+                        raise DataValidationError(
+                            f"Duplicate service name '{service_name}' found in directory {directory}:\n"
+                            f"  - {services[service_name]}\n"
+                            f"  - {file_path}"
+                        )
+
+                    services[service_name] = file_path
+
+                elif schema == "listing_v1":
+                    listings.append((file_path, data))
+
+            except Exception as e:
+                # Skip files that can't be loaded or don't have schema
+                if isinstance(e, DataValidationError):
+                    raise
+                continue
+
+        # Validate listings reference valid services
+        for listing_file, listing_data in listings:
+            service_name = listing_data.get("service_name")
+
+            if service_name:
+                # If service_name is explicitly defined, it must match a service in the directory
+                if service_name not in services:
+                    available_services = ", ".join(services.keys()) if services else "none"
+                    raise DataValidationError(
+                        f"Listing file {listing_file} references service_name '{service_name}' "
+                        f"which does not exist in the same directory.\n"
+                        f"Available services: {available_services}"
+                    )
+            else:
+                # If service_name not defined, there should be exactly one service in the directory
+                if len(services) == 0:
+                    raise DataValidationError(
+                        f"Listing file {listing_file} does not specify 'service_name' "
+                        f"and no service files found in the same directory."
+                    )
+                elif len(services) > 1:
+                    available_services = ", ".join(services.keys())
+                    raise DataValidationError(
+                        f"Listing file {listing_file} does not specify 'service_name' "
+                        f"but multiple services exist in the same directory: {available_services}. "
+                        f"Please add 'service_name' field to the listing to specify which service it belongs to."
+                    )
+
+    def validate_all_service_directories(self, data_dir: Path) -> list[str]:
+        """
+        Validate all service directories in a directory tree.
+
+        Returns a list of validation error messages (empty if all valid).
+        """
+        errors = []
+
+        # Find all directories containing service or listing files
+        directories_to_validate = set()
+
+        for pattern in ["*.json", "*.toml"]:
+            for file_path in data_dir.rglob(pattern):
+                try:
+                    data, load_errors = self.load_data_file(file_path)
+                    if load_errors or data is None:
+                        continue
+
+                    schema = data.get("schema")
+                    if schema in ["service_v1", "listing_v1"]:
+                        directories_to_validate.add(file_path.parent)
+                except Exception:
+                    continue
+
+        # Validate each directory
+        for directory in sorted(directories_to_validate):
+            try:
+                self.validate_directory_data(directory)
+            except DataValidationError as e:
+                errors.append(str(e))
+
+        return errors
+
+
+# CLI command
+app = typer.Typer(help="Validate data files")
+console = Console()
+
+
+@app.command()
+def validate(
+    data_dir: Path | None = typer.Argument(
+        None,
+        help="Directory containing data files to validate (default: ./data or UNITYSVC_DATA_DIR env var)",
+    ),
+):
+    """
+    Validate data consistency in service and listing files.
+
+    Checks:
+    1. Service names are unique within each directory
+    2. Listing files reference valid service names
+    3. Multiple services in a directory require explicit service_name in listings
+    """
+    # Determine data directory
+    if data_dir is None:
+        data_dir_str = os.environ.get("UNITYSVC_DATA_DIR")
+        if data_dir_str:
+            data_dir = Path(data_dir_str)
+        else:
+            data_dir = Path.cwd() / "data"
+
+    if not data_dir.exists():
+        console.print(f"[red]✗[/red] Data directory not found: {data_dir}")
+        raise typer.Exit(1)
+
+    console.print(f"[cyan]Validating data files in:[/cyan] {data_dir}")
+    console.print()
+
+    # Create validator and run validation
+    validator = DataValidator(data_dir, data_dir.parent / "schema")
+    validation_errors = validator.validate_all_service_directories(data_dir)
+
+    if validation_errors:
+        console.print(f"[red]✗ Validation failed with {len(validation_errors)} error(s):[/red]")
+        console.print()
+        for i, error in enumerate(validation_errors, 1):
+            console.print(f"[red]{i}.[/red] {error}")
+            console.print()
+        raise typer.Exit(1)
+    else:
+        console.print("[green]✓ All data files are valid![/green]")

unitysvc_services-0.1.0.dist-info/METADATA

@@ -0,0 +1,172 @@
+Metadata-Version: 2.4
+Name: unitysvc-services
+Version: 0.1.0
+Summary: SDK for digital service providers on UnitySVC
+Author-email: Bo Peng <bo.peng@unitysvc.com>
+Maintainer-email: Bo Peng <bo.peng@unitysvc.com>
+License: MIT
+Project-URL: bugs, https://github.com/unitysvc/unitysvc-services/issues
+Project-URL: changelog, https://github.com/unitysvc/unitysvc-services/blob/master/changelog.md
+Project-URL: homepage, https://github.com/unitysvc/unitysvc-services
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: typer
+Requires-Dist: pydantic
+Requires-Dist: jsonschema
+Requires-Dist: jinja2
+Requires-Dist: rich
+Requires-Dist: httpx
+Requires-Dist: tomli-w
+Provides-Extra: test
+Requires-Dist: coverage; extra == "test"
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: ruff; extra == "test"
+Requires-Dist: mypy; extra == "test"
+Requires-Dist: ipdb; extra == "test"
+Provides-Extra: dev
+Requires-Dist: coverage; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: ty; extra == "dev"
+Requires-Dist: ipdb; extra == "dev"
+Requires-Dist: mkdocs; extra == "dev"
+Requires-Dist: mkdocs-material; extra == "dev"
+Requires-Dist: mkdocs-autorefs; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: mkdocs; extra == "docs"
+Requires-Dist: mkdocs-material; extra == "docs"
+Requires-Dist: mkdocs-autorefs; extra == "docs"
+Dynamic: license-file
+
+# UnitySVC Provider SDK
+
+![PyPI version](https://img.shields.io/pypi/v/unitysvc-services.svg)
+[![Documentation Status](https://readthedocs.org/projects/unitysvc-services/badge/?version=latest)](https://unitysvc-services.readthedocs.io/en/latest/?version=latest)
+
+Client library and CLI tools for digital service providers to interact with the UnitySVC platform.
+
+**📚 [Full Documentation](https://unitysvc-services.readthedocs.io)** | **🚀 [Getting Started](https://unitysvc-services.readthedocs.io/en/latest/getting-started/)** | **📖 [CLI Reference](https://unitysvc-services.readthedocs.io/en/latest/cli-reference/)**
+
+## Overview
+
+UnitySVC Provider SDK enables digital service providers to manage their service offerings through a **local-first, version-controlled workflow**:
+
+- **Define** service data using schema-validated files (JSON/TOML)
+- **Manage** everything locally in git-controlled directories
+- **Validate** data against schemas before publishing
+- **Publish** to UnitySVC platform when ready
+- **Automate** with populate scripts for dynamic catalogs
+
+## Installation
+
+```bash
+pip install unitysvc-services
+```
+
+Requires Python 3.11+
+
+## Quick Example
+
+```bash
+# Initialize provider and service
+unitysvc_services init provider my-provider
+unitysvc_services init offering my-service
+unitysvc_services init seller my-marketplace
+
+# Validate and format
+unitysvc_services validate
+unitysvc_services format
+
+# Publish to platform
+export UNITYSVC_BACKEND_URL="https://api.unitysvc.com/api/v1"
+export UNITYSVC_API_KEY="your-api-key"
+
+unitysvc_services publish providers
+unitysvc_services publish sellers
+unitysvc_services publish offerings
+unitysvc_services publish listings
+
+# Verify
+unitysvc_services query offerings
+```
+
+## Key Features
+
+- 📋 **Pydantic Models** - Type-safe data models for all entities
+- ✅ **Data Validation** - Comprehensive schema validation
+- 🔄 **Local-First** - Work offline, commit to git, publish when ready
+- 🚀 **CLI Tools** - Complete command-line interface
+- 🤖 **Automation** - Script-based service generation
+- 📝 **Multiple Formats** - Support for JSON and TOML
+
+## Workflows
+
+### Manual Workflow (small catalogs)
+
+```bash
+init → edit files → validate → format → publish → verify
+```
+
+### Automated Workflow (large/dynamic catalogs)
+
+```bash
+init provider → configure populate script → populate → validate → publish
+```
+
+See [Workflows Documentation](https://unitysvc-services.readthedocs.io/en/latest/workflows/) for details.
+
+## Data Structure
+
+```
+data/
+├── seller.json                    # One seller per repo
+├── ${provider_name}/
+│   ├── provider.json              # Provider metadata
+│   ├── docs/                      # Shared documentation
+│   └── services/
+│       └── ${service_name}/
+│           ├── service.json       # Service offering
+│           └── listing-*.json     # Service listing(s)
+```
+
+See [Data Structure Documentation](https://unitysvc-services.readthedocs.io/en/latest/data-structure/) for complete details.
+
+## CLI Commands
+
+| Command | Description |
+|---------|-------------|
+| `init` | Initialize new data files from schemas |
+| `list` | List local data files |
+| `query` | Query backend API for published data |
+| `publish` | Publish data to backend |
+| `update` | Update local file fields |
+| `validate` | Validate data consistency |
+| `format` | Format data files |
+| `populate` | Execute provider populate scripts |
+
+Run `unitysvc_services --help` or see [CLI Reference](https://unitysvc-services.readthedocs.io/en/latest/cli-reference/) for complete documentation.
+
+## Documentation
+
+- **[Getting Started](https://unitysvc-services.readthedocs.io/en/latest/getting-started/)** - Installation and first steps
+- **[Data Structure](https://unitysvc-services.readthedocs.io/en/latest/data-structure/)** - File organization rules
+- **[Workflows](https://unitysvc-services.readthedocs.io/en/latest/workflows/)** - Manual and automated patterns
+- **[CLI Reference](https://unitysvc-services.readthedocs.io/en/latest/cli-reference/)** - All commands and options
+- **[File Schemas](https://unitysvc-services.readthedocs.io/en/latest/file-schemas/)** - Schema specifications
+- **[Python API](https://unitysvc-services.readthedocs.io/en/latest/api-reference/)** - Programmatic usage
+
+## Links
+
+- **PyPI**: https://pypi.org/project/unitysvc-services/
+- **Documentation**: https://unitysvc-services.readthedocs.io
+- **Source Code**: https://github.com/unitysvc/unitysvc-services
+- **Issue Tracker**: https://github.com/unitysvc/unitysvc-services/issues
+
+## License
+
+MIT License - see [LICENSE](LICENSE) file for details.
+
+## Contributing
+
+Contributions welcome! See [Contributing Guide](https://unitysvc-services.readthedocs.io/en/latest/contributing/) for details.