prompture 0.0.1__tar.gz

prompture-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Juan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

prompture-0.0.1/PKG-INFO

@@ -0,0 +1,122 @@
+Metadata-Version: 2.4
+Name: prompture
+Version: 0.0.1
+Summary: Ask LLMs to return structured JSON and run cross-model tests. API-first.
+Home-page: https://github.com/jhd3197/prompture
+Author: Juan Denis
+Author-email: juan@vene.co
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.28
+Requires-Dist: jsonschema>=4.0
+Requires-Dist: pydantic>=1.10
+Requires-Dist: pydantic-settings>=2.0
+Requires-Dist: click>=8.0
+Provides-Extra: test
+Requires-Dist: pytest>=7.0; extra == "test"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Prompture
+
+`Prompture` is an API-first library for requesting structured **JSON** output from LLMs (or any structure), validating it against a schema, and running comparative tests between models.
+
+## ✨ Features
+
+- ✅ **Structured Output**: Request models to return JSON only
+- ✅ **Validation**: Automatic validation with `jsonschema`
+- ✅ **Multi-driver**: Run the same specification against multiple drivers (OpenAI, Ollama, HTTP, mock)
+- ✅ **Reports**: Generate JSON reports with results
+- ✅ **Usage Tracking**: **NEW** - Automatic token and cost monitoring for all calls
+
+## 🆕 Token and Cost Tracking (New)
+
+Starting with this version, `extract_and_jsonify` and `ask_for_json` automatically include token usage and cost information:
+
+```python
+from prompture import extract_and_jsonify
+from prompture.drivers import OllamaDriver
+
+driver = OllamaDriver(endpoint="http://localhost:11434/api/generate", model="gemma3")
+result = extract_and_jsonify(driver, "Text to process", json_schema)
+
+# Now returns both the response and usage information
+json_output = result["json_string"]
+usage = result["usage"]
+
+print(f"Tokens used: {usage['total_tokens']}")
+print(f"Cost: ${usage['cost']:.6f}")
+```
+
+### Return Structure
+
+The main functions now return:
+```python
+{
+    "json_string": str,    # The original JSON string
+    "json_object": dict,   # The parsed JSON object
+    "usage": {
+        "prompt_tokens": int,
+        "completion_tokens": int,
+        "total_tokens": int,
+        "cost": float      # Cost in USD (0.0 for free models)
+    }
+}
+```
+
+### Supported Drivers
+
+- **OllamaDriver**: Cost = $0.00 (free local models)
+- **OpenAIDriver**: Cost automatically calculated based on the model
+
+## Batch Running and Testing Prompts
+
+`run_suite_from_spec` enables you to define and run test suites against multiple models using a specification file. This powerful feature allows you to systematically test and compare different models using a consistent set of prompts and validation criteria. Here's how it works:
+
+```python
+from prompture import run_suite_from_spec
+from prompture.drivers import MockDriver
+
+spec = {
+    "meta": {"project": "test"},
+    "models": [{"id": "mock1", "driver": "mock", "options": {}}],
+    "tests": [
+        {
+            "id": "t1",
+            "prompt_template": "Extract user info: '{text}'",
+            "inputs": [{"text": "Juan is 28 and lives in Miami. He likes basketball and coding."}],
+            "schema": {"type": "object", "required": ["name", "interests"]}
+        }
+    ]
+}
+drivers = {"mock": MockDriver()}
+report = run_suite_from_spec(spec, drivers)
+print(report)
+```
+
+The generated report includes comprehensive results for each test, model, and input combination:
+- Validation status for each response
+- Usage statistics (tokens, costs) per model
+- Execution times
+- Generated JSON responses
+
+## Quick Usage (example):
+
+```py
+from prompture import run_suite_from_spec, drivers
+spec = { ... }
+report = run_suite_from_spec(spec, drivers={"mock": drivers.MockDriver()})
+print(report)

prompture-0.0.1/Prompture.egg-info/PKG-INFO

@@ -0,0 +1,122 @@
+Metadata-Version: 2.4
+Name: prompture
+Version: 0.0.1
+Summary: Ask LLMs to return structured JSON and run cross-model tests. API-first.
+Home-page: https://github.com/jhd3197/prompture
+Author: Juan Denis
+Author-email: juan@vene.co
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.28
+Requires-Dist: jsonschema>=4.0
+Requires-Dist: pydantic>=1.10
+Requires-Dist: pydantic-settings>=2.0
+Requires-Dist: click>=8.0
+Provides-Extra: test
+Requires-Dist: pytest>=7.0; extra == "test"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Prompture
+
+`Prompture` is an API-first library for requesting structured **JSON** output from LLMs (or any structure), validating it against a schema, and running comparative tests between models.
+
+## ✨ Features
+
+- ✅ **Structured Output**: Request models to return JSON only
+- ✅ **Validation**: Automatic validation with `jsonschema`
+- ✅ **Multi-driver**: Run the same specification against multiple drivers (OpenAI, Ollama, HTTP, mock)
+- ✅ **Reports**: Generate JSON reports with results
+- ✅ **Usage Tracking**: **NEW** - Automatic token and cost monitoring for all calls
+
+## 🆕 Token and Cost Tracking (New)
+
+Starting with this version, `extract_and_jsonify` and `ask_for_json` automatically include token usage and cost information:
+
+```python
+from prompture import extract_and_jsonify
+from prompture.drivers import OllamaDriver
+
+driver = OllamaDriver(endpoint="http://localhost:11434/api/generate", model="gemma3")
+result = extract_and_jsonify(driver, "Text to process", json_schema)
+
+# Now returns both the response and usage information
+json_output = result["json_string"]
+usage = result["usage"]
+
+print(f"Tokens used: {usage['total_tokens']}")
+print(f"Cost: ${usage['cost']:.6f}")
+```
+
+### Return Structure
+
+The main functions now return:
+```python
+{
+    "json_string": str,    # The original JSON string
+    "json_object": dict,   # The parsed JSON object
+    "usage": {
+        "prompt_tokens": int,
+        "completion_tokens": int,
+        "total_tokens": int,
+        "cost": float      # Cost in USD (0.0 for free models)
+    }
+}
+```
+
+### Supported Drivers
+
+- **OllamaDriver**: Cost = $0.00 (free local models)
+- **OpenAIDriver**: Cost automatically calculated based on the model
+
+## Batch Running and Testing Prompts
+
+`run_suite_from_spec` enables you to define and run test suites against multiple models using a specification file. This powerful feature allows you to systematically test and compare different models using a consistent set of prompts and validation criteria. Here's how it works:
+
+```python
+from prompture import run_suite_from_spec
+from prompture.drivers import MockDriver
+
+spec = {
+    "meta": {"project": "test"},
+    "models": [{"id": "mock1", "driver": "mock", "options": {}}],
+    "tests": [
+        {
+            "id": "t1",
+            "prompt_template": "Extract user info: '{text}'",
+            "inputs": [{"text": "Juan is 28 and lives in Miami. He likes basketball and coding."}],
+            "schema": {"type": "object", "required": ["name", "interests"]}
+        }
+    ]
+}
+drivers = {"mock": MockDriver()}
+report = run_suite_from_spec(spec, drivers)
+print(report)
+```
+
+The generated report includes comprehensive results for each test, model, and input combination:
+- Validation status for each response
+- Usage statistics (tokens, costs) per model
+- Execution times
+- Generated JSON responses
+
+## Quick Usage (example):
+
+```py
+from prompture import run_suite_from_spec, drivers
+spec = { ... }
+report = run_suite_from_spec(spec, drivers={"mock": drivers.MockDriver()})
+print(report)

prompture-0.0.1/Prompture.egg-info/SOURCES.txt

@@ -0,0 +1,31 @@
+LICENSE
+README.md
+pyproject.toml
+setup.py
+Prompture.egg-info/PKG-INFO
+Prompture.egg-info/SOURCES.txt
+Prompture.egg-info/dependency_links.txt
+Prompture.egg-info/entry_points.txt
+Prompture.egg-info/requires.txt
+Prompture.egg-info/top_level.txt
+prompture/__init__.py
+prompture/cli.py
+prompture/core.py
+prompture/runner.py
+prompture/settings.py
+prompture/validator.py
+prompture.egg-info/PKG-INFO
+prompture.egg-info/SOURCES.txt
+prompture.egg-info/dependency_links.txt
+prompture.egg-info/entry_points.txt
+prompture.egg-info/requires.txt
+prompture.egg-info/top_level.txt
+prompture/drivers/__init__.py
+prompture/drivers/local_http_driver.py
+prompture/drivers/mock_driver.py
+prompture/drivers/ollama_driver.py
+prompture/drivers/openai_driver.py
+tests/test_cli.py
+tests/test_core.py
+tests/test_drivers.py
+tests/test_runner.py

prompture-0.0.1/Prompture.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

prompture-0.0.1/Prompture.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+prompture = prompture.cli:cli

prompture-0.0.1/Prompture.egg-info/requires.txt

@@ -0,0 +1,8 @@
+requests>=2.28
+jsonschema>=4.0
+pydantic>=1.10
+pydantic-settings>=2.0
+click>=8.0
+
+[test]
+pytest>=7.0

prompture-0.0.1/Prompture.egg-info/top_level.txt

@@ -0,0 +1 @@
+prompture

prompture-0.0.1/README.md

@@ -0,0 +1,90 @@
+# Prompture
+
+`Prompture` is an API-first library for requesting structured **JSON** output from LLMs (or any structure), validating it against a schema, and running comparative tests between models.
+
+## ✨ Features
+
+- ✅ **Structured Output**: Request models to return JSON only
+- ✅ **Validation**: Automatic validation with `jsonschema`
+- ✅ **Multi-driver**: Run the same specification against multiple drivers (OpenAI, Ollama, HTTP, mock)
+- ✅ **Reports**: Generate JSON reports with results
+- ✅ **Usage Tracking**: **NEW** - Automatic token and cost monitoring for all calls
+
+## 🆕 Token and Cost Tracking (New)
+
+Starting with this version, `extract_and_jsonify` and `ask_for_json` automatically include token usage and cost information:
+
+```python
+from prompture import extract_and_jsonify
+from prompture.drivers import OllamaDriver
+
+driver = OllamaDriver(endpoint="http://localhost:11434/api/generate", model="gemma3")
+result = extract_and_jsonify(driver, "Text to process", json_schema)
+
+# Now returns both the response and usage information
+json_output = result["json_string"]
+usage = result["usage"]
+
+print(f"Tokens used: {usage['total_tokens']}")
+print(f"Cost: ${usage['cost']:.6f}")
+```
+
+### Return Structure
+
+The main functions now return:
+```python
+{
+    "json_string": str,    # The original JSON string
+    "json_object": dict,   # The parsed JSON object
+    "usage": {
+        "prompt_tokens": int,
+        "completion_tokens": int,
+        "total_tokens": int,
+        "cost": float      # Cost in USD (0.0 for free models)
+    }
+}
+```
+
+### Supported Drivers
+
+- **OllamaDriver**: Cost = $0.00 (free local models)
+- **OpenAIDriver**: Cost automatically calculated based on the model
+
+## Batch Running and Testing Prompts
+
+`run_suite_from_spec` enables you to define and run test suites against multiple models using a specification file. This powerful feature allows you to systematically test and compare different models using a consistent set of prompts and validation criteria. Here's how it works:
+
+```python
+from prompture import run_suite_from_spec
+from prompture.drivers import MockDriver
+
+spec = {
+    "meta": {"project": "test"},
+    "models": [{"id": "mock1", "driver": "mock", "options": {}}],
+    "tests": [
+        {
+            "id": "t1",
+            "prompt_template": "Extract user info: '{text}'",
+            "inputs": [{"text": "Juan is 28 and lives in Miami. He likes basketball and coding."}],
+            "schema": {"type": "object", "required": ["name", "interests"]}
+        }
+    ]
+}
+drivers = {"mock": MockDriver()}
+report = run_suite_from_spec(spec, drivers)
+print(report)
+```
+
+The generated report includes comprehensive results for each test, model, and input combination:
+- Validation status for each response
+- Usage statistics (tokens, costs) per model
+- Execution times
+- Generated JSON responses
+
+## Quick Usage (example):
+
+```py
+from prompture import run_suite_from_spec, drivers
+spec = { ... }
+report = run_suite_from_spec(spec, drivers={"mock": drivers.MockDriver()})
+print(report)

prompture-0.0.1/prompture/__init__.py

@@ -0,0 +1,8 @@
+"""prompture - API package to convert LLM outputs into JSON + test harness."""
+
+from .core import ask_for_json, extract_and_jsonify, Driver, clean_json_text, clean_json_text_with_ai
+from .runner import run_suite_from_spec
+from .validator import validate_against_schema
+
+__all__ = ["ask_for_json", "extract_and_jsonify", "run_suite_from_spec", "validate_against_schema", "Driver", "clean_json_text", "clean_json_text_with_ai"]
+__version__ = "0.0.1"

prompture-0.0.1/prompture/cli.py

@@ -0,0 +1,23 @@
+import json
+import click
+from .runner import run_suite_from_spec
+
+@click.group()
+def cli():
+    """CLI simple para correr specs JSON"""
+    pass
+
+@cli.command()
+@click.argument("specfile", type=click.Path(exists=True))
+@click.argument("outfile", type=click.Path())
+def run(specfile, outfile):
+    """Run a spec JSON and save report."""
+    with open(specfile, "r", encoding="utf-8") as fh:
+        spec = json.load(fh)
+    # drivers mínimos: mock disponible por defecto
+    from .drivers import MockDriver
+    drivers = {"mock": MockDriver()}
+    report = run_suite_from_spec(spec, drivers)
+    with open(outfile, "w", encoding="utf-8") as fh:
+        json.dump(report, fh, indent=2, ensure_ascii=False)
+    click.echo(f"Report saved to {outfile}")

prompture-0.0.1/prompture/core.py

@@ -0,0 +1,172 @@
+"""Core utilities: Driver base class y helper para pedir JSON al LLM.
+Comentarios en español.
+"""
+from __future__ import annotations
+import json
+from typing import Any, Dict, Optional
+
+class Driver:
+    """Adapter base. Implementar generate(prompt, options) -> {"text": ... , "meta": {...}}
+
+    The 'meta' object in the response should have a standardized structure:
+
+    {
+        "prompt_tokens": int,     # Number of tokens in the prompt
+        "completion_tokens": int, # Number of tokens in the completion
+        "total_tokens": int,      # Total tokens used (prompt + completion)
+        "cost": float,            # Cost in USD (0.0 for free models)
+        "raw_response": dict      # Raw response from LLM provider
+    }
+
+    All drivers must populate these fields. The 'raw_response' field can contain
+    additional provider-specific metadata while the core fields provide
+    standardized access to token usage and cost information.
+    """
+    def generate(self, prompt: str, options: Dict[str,Any]) -> Dict[str,Any]:
+        raise NotImplementedError
+
+
+def clean_json_text(text: str) -> str:
+    """Intentos básicos para extraer JSON si viene con ````` o explicaciones.
+    No es perfecto; se recomienda usar prompts con ejemplos para forzar JSON válido.
+    """
+    # eliminar fences ```json ``` o ```
+    text = text.strip()
+    # detect code fence and extract first code block
+    if text.startswith("```"):
+        # Find the first opening ```
+        start_fence = text.find("```")
+        if start_fence != -1:
+            # Skip the opening fence and language tag
+            start_content = text.find("\n", start_fence)
+            if start_content != -1:
+                # Find the first closing ```
+                end_fence = text.find("```", start_content)
+                if end_fence != -1:
+                    # Extract content between fences
+                    rest = text[start_content + 1:end_fence]
+                    return rest.strip()
+                else:
+                    # No closing fence, take from start content to end
+                    rest = text[start_content + 1:]
+                    return rest.strip()
+    # intentar extraer la primera ocurrencia de un objeto JSON
+    start = text.find("{")
+    end = text.rfind("}")
+    if start != -1 and end != -1 and end > start:
+        return text[start:end+1]
+    return text
+
+def clean_json_text_with_ai(driver: Driver, text: str, options: Dict[str, Any] = {}) -> str:
+    """Use LLM to fix malformed JSON strings.
+
+    Creates a specialized prompt instructing the LLM to correct the provided text
+    into a valid JSON object, then cleans the response to ensure no markdown fences remain.
+    """
+    prompt = (
+        "The following text is supposed to be a single JSON object, but it is malformed. "
+        "Please correct it and return only the valid JSON object. Do not add any explanations or markdown. "
+        f"The text to correct is:\n\n{text}"
+    )
+    resp = driver.generate(prompt, options)
+    raw = resp.get("text", "")
+    cleaned = clean_json_text(raw)
+    return cleaned
+
+def ask_for_json(driver: Driver, content_prompt: str, json_schema: Dict[str, Any], ai_cleanup: bool = False, options: Dict[str, Any] = {}) -> Dict[str, Any]:
+    """Sends a prompt to the driver and returns both JSON output and usage metadata.
+
+    This function enforces a schema-first approach by requiring a json_schema parameter
+    and automatically generating instructions for the LLM to return valid JSON matching the schema.
+
+    Args:
+        driver: adapter that implements generate(prompt, options)
+        content_prompt: main prompt content (may include examples)
+        json_schema: required JSON schema dictionary defining the expected structure
+        ai_cleanup: whether to attempt AI-based cleanup if JSON parsing fails
+        options: additional options to pass to the driver
+
+    Returns:
+        A dictionary containing:
+        - json_string: the JSON string output
+        - json_object: the parsed JSON object
+        - usage: token usage and cost information from the driver's meta object
+    """
+    schema_string = json.dumps(json_schema, indent=2)
+    instruct = (
+        "Return only a single JSON object (no markdown, no extra text) that validates against this JSON schema:\n"
+        f"{schema_string}\n\n"
+        "If a value is unknown use null. Use double quotes for keys and strings."
+    )
+    full_prompt = f"{content_prompt}\n\n{instruct}"
+    resp = driver.generate(full_prompt, options)
+    raw = resp.get("text", "")
+    cleaned = clean_json_text(raw)
+    try:
+        json_obj = json.loads(cleaned)
+        return {
+            "json_string": cleaned,
+            "json_object": json_obj,
+            "usage": resp.get("meta", {})
+        }
+    except json.JSONDecodeError:
+        if ai_cleanup:
+            # clean_json_text_with_ai returns just the cleaned string, so we need to get fresh metadata
+            cleaned_fixed = clean_json_text_with_ai(driver, cleaned, options)
+            try:
+                json_obj = json.loads(cleaned_fixed)
+                return {
+                    "json_string": cleaned_fixed,
+                    "json_object": json_obj,
+                    "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0, "cost": 0.0}  # Placeholder for cleanup call
+                }
+            except json.JSONDecodeError:
+                raise
+        else:
+            raise
+        
+def extract_and_jsonify(
+    driver: Driver,
+    text: str,
+    json_schema: Dict[str, Any],
+    instruction_template: str = "Extract information from the following text:",
+    ai_cleanup: bool = False,
+    options: Dict[str, Any] = {}
+) -> Dict[str, Any]:
+    """Extracts structured information from text and returns it as a JSON object with usage metadata.
+
+    This is a higher-level function that simplifies the process of extracting information
+    into JSON by automatically constructing the content prompt and calling ask_for_json.
+
+    Args:
+        driver: The LLM driver instance to use for generation
+        text: The raw text to extract information from
+        json_schema: JSON schema dictionary defining the expected structure
+        instruction_template: Template string for the extraction instruction
+                          (default: "Extract information from the following text:")
+        ai_cleanup: Whether to attempt AI-based cleanup if JSON parsing fails
+        options: Additional options to pass to the driver
+
+    Returns:
+        A dictionary containing:
+        - json_string: the JSON string output
+        - json_object: the parsed JSON object
+        - usage: token usage and cost information from the driver's meta object
+
+    Raises:
+        ValueError: If text is empty or None
+        json.JSONDecodeError: If the response cannot be parsed as JSON and ai_cleanup is False
+
+    Example:
+        >>> schema = {"type": "object", "properties": {"name": {"type": "string"}}}
+        >>> result = extract_and_jsonify(driver, "John is a developer", schema)
+        >>> result["json_string"]
+        '{"name": "John"}'
+        >>> result["usage"]["total_tokens"]
+        150
+    """
+    if not text or not text.strip():
+        raise ValueError("Text input cannot be empty")
+
+    content_prompt = f"{instruction_template} {text}"
+    return ask_for_json(driver, content_prompt, json_schema, ai_cleanup, options)

prompture-0.0.1/prompture/drivers/__init__.py

@@ -0,0 +1,20 @@
+from .mock_driver import MockDriver
+from .openai_driver import OpenAIDriver
+from .local_http_driver import LocalHTTPDriver
+from .ollama_driver import OllamaDriver
+from ..settings import settings
+
+# Factory to get a driver instance
+def get_driver(provider_name: str = None):
+    provider = provider_name or settings.default_provider
+    if provider == "mock":
+        return MockDriver()
+    if provider == "openai":
+        return OpenAIDriver(api_key=settings.openai_api_key, model=settings.openai_model)
+    if provider == "local_http":
+        return LocalHTTPDriver(endpoint=settings.hf_endpoint)
+    if provider == "ollama":
+        return OllamaDriver(endpoint=settings.ollama_endpoint, model=settings.ollama_model)
+    raise ValueError(f"Unknown provider: {provider}")
+
+__all__ = ["MockDriver", "OpenAIDriver", "LocalHTTPDriver", "OllamaDriver", "get_driver"]

prompture-0.0.1/prompture/drivers/local_http_driver.py

@@ -0,0 +1,14 @@
+import requests
+from ..core import Driver
+from typing import Any, Dict
+
+class LocalHTTPDriver(Driver):
+    def __init__(self, endpoint: str):
+        self.endpoint = endpoint
+
+    def generate(self, prompt: str, options: Dict[str,Any]) -> Dict[str,Any]:
+        payload = {"prompt": prompt, "options": options}
+        r = requests.post(self.endpoint, json=payload, timeout=30)
+        r.raise_for_status()
+        # se espera {"text": "...", "meta": {...}}
+        return r.json()