openai-sdk-helpers 0.0.9__py3-none-any.whl → 0.1.1__py3-none-any.whl

openai_sdk_helpers/utils/output_validation.py

@@ -0,0 +1,448 @@
+"""Output validation framework for agent responses.
+
+Provides validators for checking agent outputs against schemas,
+semantic constraints, and content policies.
+
+Classes
+-------
+ValidationRule
+    Base class for validation rules.
+JSONSchemaValidator
+    Validate outputs against JSON schemas.
+SemanticValidator
+    Validate outputs semantically (e.g., must reference sources).
+LengthValidator
+    Validate output length constraints.
+OutputValidator
+    Composite validator for multiple rules.
+
+Functions
+---------
+validate_output
+    Convenience function for validating outputs.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from abc import ABC, abstractmethod
+from typing import Any
+
+from pydantic import BaseModel, ValidationError
+
+
+class ValidationResult(BaseModel):
+    """Result of output validation.
+
+    Attributes
+    ----------
+    valid : bool
+        Whether validation passed.
+    errors : list[str]
+        List of validation error messages.
+    warnings : list[str]
+        List of validation warnings.
+    """
+
+    valid: bool
+    errors: list[str] = []
+    warnings: list[str] = []
+
+
+class ValidationRule(ABC):
+    """Base class for validation rules.
+
+    Methods
+    -------
+    validate
+        Validate output and return ValidationResult.
+    """
+
+    @abstractmethod
+    def validate(self, output: Any) -> ValidationResult:
+        """Validate output.
+
+        Parameters
+        ----------
+        output : Any
+            Output to validate.
+
+        Returns
+        -------
+        ValidationResult
+            Validation result.
+        """
+        pass
+
+
+class JSONSchemaValidator(ValidationRule):
+    """Validate outputs against JSON schemas.
+
+    Parameters
+    ----------
+    schema : dict
+        JSON schema to validate against.
+
+    Examples
+    --------
+    >>> schema = {"type": "object", "required": ["name"]}
+    >>> validator = JSONSchemaValidator(schema)
+    >>> result = validator.validate({"name": "test"})
+    >>> result.valid
+    True
+    """
+
+    def __init__(self, schema: dict[str, Any]) -> None:
+        """Initialize JSON schema validator.
+
+        Parameters
+        ----------
+        schema : dict
+            JSON schema dictionary.
+        """
+        self.schema = schema
+
+    def validate(self, output: Any) -> ValidationResult:
+        """Validate output against JSON schema.
+
+        Parameters
+        ----------
+        output : Any
+            Output to validate.
+
+        Returns
+        -------
+        ValidationResult
+            Validation result.
+        """
+        try:
+            # Try using jsonschema if available
+            import jsonschema
+        except ImportError:
+            # Fallback to basic type checking
+            return self._basic_validate(output)
+
+        try:
+            jsonschema.validate(instance=output, schema=self.schema)
+            return ValidationResult(valid=True)
+        except jsonschema.ValidationError as e:
+            return ValidationResult(valid=False, errors=[str(e)])
+
+    def _basic_validate(self, output: Any) -> ValidationResult:
+        """Perform basic validation without jsonschema library."""
+        errors = []
+
+        # Check type
+        expected_type = self.schema.get("type")
+        if expected_type == "object" and not isinstance(output, dict):
+            errors.append(f"Expected object, got {type(output).__name__}")
+        elif expected_type == "array" and not isinstance(output, list):
+            errors.append(f"Expected array, got {type(output).__name__}")
+
+        # Check required fields
+        if expected_type == "object" and isinstance(output, dict):
+            required = self.schema.get("required", [])
+            for field in required:
+                if field not in output:
+                    errors.append(f"Missing required field: {field}")
+
+        return ValidationResult(valid=len(errors) == 0, errors=errors)
+
+
+class SemanticValidator(ValidationRule):
+    """Validate outputs semantically.
+
+    Parameters
+    ----------
+    must_contain : list[str], optional
+        Phrases that must appear in output.
+    must_not_contain : list[str], optional
+        Phrases that must not appear in output.
+    must_reference_sources : bool
+        Whether output must reference source documents. Default is False.
+
+    Examples
+    --------
+    >>> validator = SemanticValidator(must_contain=["summary", "conclusion"])
+    >>> result = validator.validate("Here is the summary and conclusion.")
+    >>> result.valid
+    True
+    """
+
+    def __init__(
+        self,
+        must_contain: list[str] | None = None,
+        must_not_contain: list[str] | None = None,
+        must_reference_sources: bool = False,
+    ) -> None:
+        """Initialize semantic validator.
+
+        Parameters
+        ----------
+        must_contain : list[str], optional
+            Phrases that must appear.
+        must_not_contain : list[str], optional
+            Phrases that must not appear.
+        must_reference_sources : bool
+            Check for source references.
+        """
+        self.must_contain = must_contain or []
+        self.must_not_contain = must_not_contain or []
+        self.must_reference_sources = must_reference_sources
+
+    def validate(self, output: Any) -> ValidationResult:
+        """Validate output semantically.
+
+        Parameters
+        ----------
+        output : Any
+            Output to validate (converted to string).
+
+        Returns
+        -------
+        ValidationResult
+            Validation result.
+        """
+        text = str(output).lower()
+        errors = []
+        warnings = []
+
+        # Check must contain
+        for phrase in self.must_contain:
+            if phrase.lower() not in text:
+                errors.append(f"Output must contain: '{phrase}'")
+
+        # Check must not contain
+        for phrase in self.must_not_contain:
+            if phrase.lower() in text:
+                errors.append(f"Output must not contain: '{phrase}'")
+
+        # Check for source references
+        if self.must_reference_sources:
+            # Look for common citation patterns
+            citation_patterns = [
+                r"\[\d+\]",  # [1]
+                r"\(\d+\)",  # (1)
+                r"source:",  # source:
+                r"according to",  # according to
+            ]
+            has_citations = any(
+                re.search(pattern, text, re.IGNORECASE) for pattern in citation_patterns
+            )
+            if not has_citations:
+                warnings.append("Output should reference sources")
+
+        return ValidationResult(
+            valid=len(errors) == 0,
+            errors=errors,
+            warnings=warnings,
+        )
+
+
+class LengthValidator(ValidationRule):
+    """Validate output length constraints.
+
+    Parameters
+    ----------
+    min_length : int, optional
+        Minimum output length in characters.
+    max_length : int, optional
+        Maximum output length in characters.
+    min_words : int, optional
+        Minimum word count.
+    max_words : int, optional
+        Maximum word count.
+
+    Examples
+    --------
+    >>> validator = LengthValidator(min_words=10, max_words=100)
+    >>> result = validator.validate("Short text")
+    >>> result.valid
+    False
+    """
+
+    def __init__(
+        self,
+        min_length: int | None = None,
+        max_length: int | None = None,
+        min_words: int | None = None,
+        max_words: int | None = None,
+    ) -> None:
+        """Initialize length validator.
+
+        Parameters
+        ----------
+        min_length : int, optional
+            Minimum character count.
+        max_length : int, optional
+            Maximum character count.
+        min_words : int, optional
+            Minimum word count.
+        max_words : int, optional
+            Maximum word count.
+        """
+        self.min_length = min_length
+        self.max_length = max_length
+        self.min_words = min_words
+        self.max_words = max_words
+
+    def validate(self, output: Any) -> ValidationResult:
+        """Validate output length.
+
+        Parameters
+        ----------
+        output : Any
+            Output to validate (converted to string).
+
+        Returns
+        -------
+        ValidationResult
+            Validation result.
+        """
+        text = str(output)
+        errors = []
+
+        # Check character length
+        if self.min_length and len(text) < self.min_length:
+            errors.append(
+                f"Output too short: {len(text)} chars (min: {self.min_length})"
+            )
+        if self.max_length and len(text) > self.max_length:
+            errors.append(
+                f"Output too long: {len(text)} chars (max: {self.max_length})"
+            )
+
+        # Check word count
+        words = text.split()
+        if self.min_words and len(words) < self.min_words:
+            errors.append(f"Too few words: {len(words)} (min: {self.min_words})")
+        if self.max_words and len(words) > self.max_words:
+            errors.append(f"Too many words: {len(words)} (max: {self.max_words})")
+
+        return ValidationResult(valid=len(errors) == 0, errors=errors)
+
+
+class OutputValidator:
+    """Composite validator for multiple rules.
+
+    Parameters
+    ----------
+    rules : list[ValidationRule]
+        List of validation rules to apply.
+
+    Methods
+    -------
+    validate
+        Validate output against all rules.
+    add_rule
+        Add a validation rule.
+
+    Examples
+    --------
+    >>> validator = OutputValidator([
+    ...     LengthValidator(min_words=10),
+    ...     SemanticValidator(must_contain=["summary"])
+    ... ])
+    >>> result = validator.validate("This is a brief summary with enough words.")
+    """
+
+    def __init__(self, rules: list[ValidationRule] | None = None) -> None:
+        """Initialize output validator.
+
+        Parameters
+        ----------
+        rules : list[ValidationRule], optional
+            Initial validation rules.
+        """
+        self.rules = rules or []
+
+    def add_rule(self, rule: ValidationRule) -> None:
+        """Add a validation rule.
+
+        Parameters
+        ----------
+        rule : ValidationRule
+            Rule to add.
+        """
+        self.rules.append(rule)
+
+    def validate(self, output: Any) -> ValidationResult:
+        """Validate output against all rules.
+
+        Parameters
+        ----------
+        output : Any
+            Output to validate.
+
+        Returns
+        -------
+        ValidationResult
+            Combined validation result.
+        """
+        all_errors: list[str] = []
+        all_warnings: list[str] = []
+
+        for rule in self.rules:
+            result = rule.validate(output)
+            all_errors.extend(result.errors)
+            all_warnings.extend(result.warnings)
+
+        return ValidationResult(
+            valid=len(all_errors) == 0,
+            errors=all_errors,
+            warnings=all_warnings,
+        )
+
+
+def validate_output(
+    output: Any,
+    schema: dict[str, Any] | None = None,
+    min_length: int | None = None,
+    max_length: int | None = None,
+    must_contain: list[str] | None = None,
+) -> ValidationResult:
+    """Validate outputs with common validation rules.
+
+    Parameters
+    ----------
+    output : Any
+        Output to validate.
+    schema : dict, optional
+        JSON schema to validate against.
+    min_length : int, optional
+        Minimum character length.
+    max_length : int, optional
+        Maximum character length.
+    must_contain : list[str], optional
+        Phrases that must appear.
+
+    Returns
+    -------
+    ValidationResult
+        Validation result.
+
+    Examples
+    --------
+    >>> result = validate_output(
+    ...     "Short",
+    ...     min_length=10,
+    ...     must_contain=["summary"]
+    ... )
+    >>> result.valid
+    False
+    """
+    validator = OutputValidator()
+
+    if schema:
+        validator.add_rule(JSONSchemaValidator(schema))
+
+    if min_length or max_length:
+        validator.add_rule(
+            LengthValidator(min_length=min_length, max_length=max_length)
+        )
+
+    if must_contain:
+        validator.add_rule(SemanticValidator(must_contain=must_contain))
+
+    return validator.validate(output)

openai_sdk_helpers/utils/path_utils.py

@@ -0,0 +1,46 @@
+"""File and path utilities."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def check_filepath(
+    filepath: Path | None = None, *, fullfilepath: str | None = None
+) -> Path:
+    """Ensure the parent directory for a file path exists.
+
+    Creates parent directories as needed. Exactly one of ``filepath`` or
+    ``fullfilepath`` must be provided.
+
+    Parameters
+    ----------
+    filepath : Path or None, optional
+        Path object to validate. Mutually exclusive with ``fullfilepath``.
+    fullfilepath : str or None, optional
+        String path to validate. Mutually exclusive with ``filepath``.
+
+    Returns
+    -------
+    Path
+        Path object representing the validated file path.
+    """
+    if filepath is None and fullfilepath is None:
+        raise ValueError("filepath or fullfilepath is required.")
+    if fullfilepath is not None:
+        target = Path(fullfilepath)
+    elif filepath is not None:
+        target = Path(filepath)
+    else:
+        raise ValueError("filepath or fullfilepath is required.")
+    ensure_directory(target.parent)
+    return target
+
+
+def ensure_directory(path: Path) -> Path:
+    """Ensure a directory exists and return it."""
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+__all__ = ["check_filepath", "ensure_directory"]

openai_sdk_helpers/{validation.py → utils/validation.py}

@@ -16,7 +16,7 @@ V = TypeVar("V")
 U = TypeVar("U")
 
 
-def validate_non_empty_string(value: str, field_name: str) -> str:
+def validate_non_empty_string(value: str, *, field_name: str) -> str:
     """Validate that a string is non-empty.
 
     Parameters
@@ -46,7 +46,7 @@ def validate_non_empty_string(value: str, field_name: str) -> str:
     return stripped
 
 
-def validate_max_length(value: str, max_len: int, field_name: str) -> str:
+def validate_max_length(value: str, max_len: int, *, field_name: str) -> str:
     """Validate that a string doesn't exceed maximum length.
 
     Parameters
@@ -76,7 +76,7 @@ def validate_max_length(value: str, max_len: int, field_name: str) -> str:
     return value
 
 
-def validate_url_format(url: str, field_name: str = "URL") -> str:
+def validate_url_format(url: str, *, field_name: str = "URL") -> str:
     """Validate that a string is a valid URL.
 
     Parameters
@@ -106,6 +106,7 @@ def validate_url_format(url: str, field_name: str = "URL") -> str:
 def validate_dict_mapping(
     mapping: Mapping[K, V],
     expected_keys: set[K],
+    *,
     field_name: str,
     allow_extra: bool = False,
 ) -> dict[K, V]:
@@ -156,6 +157,7 @@ def validate_dict_mapping(
 def validate_list_items(
     items: list[U],
     item_validator: Callable[[U], T],
+    *,
     field_name: str,
     allow_empty: bool = False,
 ) -> list[T]:
@@ -203,6 +205,7 @@ def validate_list_items(
 def validate_choice(
     value: U,
     allowed_values: set[U],
+    *,
     field_name: str,
 ) -> U:
     """Validate that a value is one of allowed choices.
@@ -235,6 +238,7 @@ def validate_choice(
 
 def validate_safe_path(
     path: Path | str,
+    *,
     base_dir: Path | None = None,
     field_name: str = "path",
 ) -> Path:

openai_sdk_helpers/vector_storage/storage.py

@@ -96,6 +96,7 @@ class VectorStorage:
 
     def __init__(
         self,
+        *,
         store_name: str,
         client: OpenAIClient | None = None,
         model: str | None = None,
@@ -228,6 +229,7 @@ class VectorStorage:
     def upload_file(
         self,
         file_path: str,
+        *,
         purpose: str = "assistants",
         attributes: dict[str, str | float | bool] | None = None,
         overwrite: bool = False,
@@ -316,6 +318,7 @@ class VectorStorage:
     def upload_files(
         self,
         file_patterns: str | list[str],
+        *,
         purpose: str = "assistants",
         attributes: dict[str, str | float | bool] | None = None,
         overwrite: bool = False,
@@ -369,10 +372,10 @@ class VectorStorage:
                 executor.submit(
                     self.upload_file,
                     path,
-                    purpose,
-                    attributes,
-                    overwrite,
-                    False,
+                    purpose=purpose,
+                    attributes=attributes,
+                    overwrite=overwrite,
+                    refresh_cache=False,
                 ): path
                 for path in all_paths
             }
@@ -552,7 +555,7 @@ class VectorStorage:
         return stats
 
     def search(
-        self, query: str, top_k: int = 5
+        self, query: str, *, top_k: int = 5
     ) -> SyncPage[VectorStoreSearchResponse] | None:
         """Perform a semantic search within the vector store.
 
@@ -583,7 +586,7 @@ class VectorStorage:
             log(f"Error searching vector store: {str(exc)}", level=logging.ERROR)
             return None
 
-    def summarize(self, query: str, top_k: int = 15) -> str | None:
+    def summarize(self, query: str, *, top_k: int = 15) -> str | None:
         """Perform a semantic search and summarize results by topic.
 
         Retrieves top search results and generates a summary. This method

{openai_sdk_helpers-0.0.9.dist-info → openai_sdk_helpers-0.1.1.dist-info}/METADATA

@@ -1,14 +1,24 @@
 Metadata-Version: 2.4
 Name: openai-sdk-helpers
-Version: 0.0.9
+Version: 0.1.1
 Summary: Composable helpers for OpenAI SDK agents, prompts, and storage
 Author: openai-sdk-helpers maintainers
 License: MIT
 License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Typing :: Typed
 Requires-Python: >=3.10
 Requires-Dist: jinja2
-Requires-Dist: openai
-Requires-Dist: openai-agents
+Requires-Dist: openai-agents<1.0.0,>=0.6.4
+Requires-Dist: openai<3.0.0,>=2.14.0
 Requires-Dist: pydantic<3,>=2.7
 Requires-Dist: python-dotenv
 Requires-Dist: streamlit
@@ -90,6 +100,26 @@ The `agent` module provides a higher-level abstraction for building agents, whil
 - **Tool execution framework** with custom handlers and structured outputs
 - **Session persistence** for saving and restoring conversation history
 
+#### Infrastructure & Utilities
+- **Centralized logger factory** for consistent application logging
+- **Retry patterns** with exponential backoff and jitter
+- **Output validation** framework with JSON schema, semantic, and length validators
+- **CLI tool** for testing agents, validating templates, and inspecting registries
+- **Deprecation utilities** for managing API changes
+
+#### Shared Components
+- **Typed structures** using Pydantic for prompts, responses, and search workflows 
+  to ensure predictable inputs and outputs
+- **OpenAI configuration management** with environment variable and `.env` file support
+- **Vector storage abstraction** for seamless integration with OpenAI vector stores
+- **Type-safe interfaces** with full type hints and `py.typed` marker for external projects
+  - **ValidatorAgent**: Check inputs and outputs against safety guardrails
+
+#### Response Module (Built on `openai` SDK)
+- **Response handling utilities** for direct API control with fine-grained message management
+- **Tool execution framework** with custom handlers and structured outputs
+- **Session persistence** for saving and restoring conversation history
+
 #### Shared Components
 - **Typed structures** using Pydantic for prompts, responses, and search workflows 
   to ensure predictable inputs and outputs
@@ -502,6 +532,31 @@ See `AGENTS.md` for detailed contributing guidelines and conventions.
 This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
 for details.
 
+## CLI Tool
+
+The package includes a command-line tool for development and testing:
+
+```bash
+# List all registered response configurations
+openai-helpers registry list
+
+# Inspect a specific configuration
+openai-helpers registry inspect my_config
+
+# Validate Jinja2 templates
+openai-helpers template validate ./templates
+
+# Test an agent (coming soon)
+openai-helpers agent test MyAgent --input "test input"
+```
+
+### CLI Commands
+
+- **registry list** - Show all registered response configurations
+- **registry inspect** - Display details of a configuration
+- **template validate** - Check template syntax and structure
+- **agent test** - Test agents locally with sample inputs
+
 ## Troubleshooting
 
 ### Common Issues
@@ -523,6 +578,7 @@ OPENAI_API_KEY=your-api-key-here
 Vector search workflows require custom prompt templates. Either:
 1. Create the required `.jinja` files in your `prompt_dir`
 2. Omit the `prompt_dir` parameter to use built-in defaults (for text agents only)
+3. Use the CLI to validate templates: `openai-helpers template validate ./templates`
 
 **Import errors after installation**