ostruct-cli 0.4.0__py3-none-any.whl → 0.6.0__py3-none-any.whl

ostruct/cli/token_utils.py

@@ -0,0 +1,43 @@
+"""Token estimation utilities."""
+
+from typing import Any, Dict, List, Union
+
+import tiktoken
+
+
+def estimate_tokens_with_encoding(
+    messages: Union[str, Dict[str, str], List[Dict[str, str]]],
+    model: str,
+    encoder: Any = None,
+) -> int:
+    """Estimate the number of tokens in a chat completion.
+
+    Args:
+        messages: Message content - can be string, single message dict, or list of messages
+        model: Model name
+        encoder: Optional tiktoken encoder for testing
+
+    Returns:
+        int: Estimated token count
+    """
+    if encoder is None:
+        # Use o200k_base for gpt-4o and o1 models
+        if model.startswith(("gpt-4o", "o1", "o3")):
+            encoder = tiktoken.get_encoding("o200k_base")
+        else:
+            encoder = tiktoken.get_encoding("cl100k_base")
+
+    if isinstance(messages, str):
+        return len(encoder.encode(messages))
+    elif isinstance(messages, dict):
+        return len(encoder.encode(str(messages.get("content", ""))))
+    else:
+        num_tokens = 0
+        for message in messages:
+            num_tokens += 4  # message overhead
+            for key, value in message.items():
+                num_tokens += len(encoder.encode(str(value)))
+                if key == "name":
+                    num_tokens -= 1  # role is omitted
+        num_tokens += 2  # reply priming
+        return num_tokens

ostruct/cli/validators.py

@@ -0,0 +1,109 @@
+"""Validators for CLI options and arguments."""
+
+import json
+from pathlib import Path
+from typing import Any, List, Optional, Tuple, Union
+
+import click
+
+from .errors import InvalidJSONError, VariableNameError
+
+
+def validate_name_path_pair(
+    ctx: click.Context,
+    param: click.Parameter,
+    value: List[Tuple[str, Union[str, Path]]],
+) -> List[Tuple[str, Union[str, Path]]]:
+    """Validate name/path pairs for files and directories.
+
+    Args:
+        ctx: Click context
+        param: Click parameter
+        value: List of (name, path) tuples
+
+    Returns:
+        List of validated (name, Path) tuples
+
+    Raises:
+        click.BadParameter: If validation fails
+    """
+    if not value:
+        return value
+
+    result: List[Tuple[str, Union[str, Path]]] = []
+    for name, path in value:
+        if not name.isidentifier():
+            raise click.BadParameter(f"Invalid variable name: {name}")
+        result.append((name, Path(path)))
+    return result
+
+
+def validate_variable(
+    ctx: click.Context, param: click.Parameter, value: Optional[List[str]]
+) -> Optional[List[Tuple[str, str]]]:
+    """Validate name=value format for simple variables.
+
+    Args:
+        ctx: Click context
+        param: Click parameter
+        value: List of "name=value" strings
+
+    Returns:
+        List of validated (name, value) tuples
+
+    Raises:
+        click.BadParameter: If validation fails
+    """
+    if not value:
+        return None
+
+    result = []
+    for var in value:
+        if "=" not in var:
+            raise click.BadParameter(
+                f"Variable must be in format name=value: {var}"
+            )
+        name, val = var.split("=", 1)
+        if not name.isidentifier():
+            raise click.BadParameter(f"Invalid variable name: {name}")
+        result.append((name, val))
+    return result
+
+
+def validate_json_variable(
+    ctx: click.Context, param: click.Parameter, value: Optional[List[str]]
+) -> Optional[List[Tuple[str, Any]]]:
+    """Validate JSON variable format.
+
+    Args:
+        ctx: Click context
+        param: Click parameter
+        value: List of "name=json_string" values
+
+    Returns:
+        List of validated (name, parsed_json) tuples
+
+    Raises:
+        click.BadParameter: If validation fails
+    """
+    if not value:
+        return None
+
+    result = []
+    for var in value:
+        if "=" not in var:
+            raise InvalidJSONError(
+                f'JSON variable must be in format name=\'{"json":"value"}\': {var}'
+            )
+        name, json_str = var.split("=", 1)
+        if not name.isidentifier():
+            raise VariableNameError(f"Invalid variable name: {name}")
+        try:
+            json_value = json.loads(json_str)
+            result.append((name, json_value))
+        except json.JSONDecodeError as e:
+            raise InvalidJSONError(
+                f"Invalid JSON value for variable {name!r}: {json_str!r}",
+                context={"variable_name": name},
+            ) from e
+    return result

ostruct_cli-0.6.0.dist-info/METADATA

@@ -0,0 +1,404 @@
+Metadata-Version: 2.3
+Name: ostruct-cli
+Version: 0.6.0
+Summary: CLI for OpenAI Structured Output
+Author: Yaniv Golan
+Author-email: yaniv@golan.name
+Requires-Python: >=3.10,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: cachetools (>=5.3.2,<6.0.0)
+Requires-Dist: chardet (>=5.0.0,<6.0.0)
+Requires-Dist: click (>=8.1.7,<9.0.0)
+Requires-Dist: ijson (>=3.2.3,<4.0.0)
+Requires-Dist: jsonschema (>=4.23.0,<5.0.0)
+Requires-Dist: openai (>=1.0.0,<2.0.0)
+Requires-Dist: openai-structured (>=2.0.0,<3.0.0)
+Requires-Dist: pydantic (>=2.6.3,<3.0.0)
+Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
+Requires-Dist: tiktoken (==0.9.0)
+Requires-Dist: tomli (>=2.0.1,<3.0.0) ; python_version < "3.11"
+Requires-Dist: typing-extensions (>=4.9.0,<5.0.0)
+Requires-Dist: werkzeug (>=3.1.3,<4.0.0)
+Description-Content-Type: text/markdown
+
+# ostruct-cli
+
+[![PyPI version](https://badge.fury.io/py/ostruct-cli.svg)](https://badge.fury.io/py/ostruct-cli)
+[![Python Versions](https://img.shields.io/pypi/pyversions/ostruct-cli.svg)](https://pypi.org/project/ostruct-cli)
+[![Documentation Status](https://readthedocs.org/projects/ostruct/badge/?version=latest)](https://ostruct.readthedocs.io/en/latest/?badge=latest)
+[![CI](https://github.com/yaniv-golan/ostruct/actions/workflows/ci.yml/badge.svg)](https://github.com/yaniv-golan/ostruct/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+Command-line interface for working with OpenAI models and structured output, powered by the [openai-structured](https://github.com/yaniv-golan/openai-structured) library.
+
+## Features
+
+- Generate structured JSON output from natural language using OpenAI models and a JSON schema
+- Rich template system for defining prompts (Jinja2-based)
+- Automatic token counting and context window management
+- Streaming support for real-time output
+- Secure handling of sensitive data
+
+## Installation
+
+### For Users
+
+To install the latest stable version from PyPI:
+
+```bash
+pip install ostruct-cli
+```
+
+### For Developers
+
+If you plan to contribute to the project, see the [Development Setup](#development-setup) section below for instructions on setting up the development environment with Poetry.
+
+## Shell Completion
+
+ostruct-cli supports shell completion for Bash, Zsh, and Fish shells. To enable it:
+
+### Bash
+
+Add this to your `~/.bashrc`:
+
+```bash
+eval "$(_OSTRUCT_COMPLETE=bash_source ostruct)"
+```
+
+### Zsh
+
+Add this to your `~/.zshrc`:
+
+```bash
+eval "$(_OSTRUCT_COMPLETE=zsh_source ostruct)"
+```
+
+### Fish
+
+Add this to your `~/.config/fish/completions/ostruct.fish`:
+
+```fish
+eval (env _OSTRUCT_COMPLETE=fish_source ostruct)
+```
+
+After adding the appropriate line, restart your shell or source the configuration file.
+Shell completion will help you with:
+
+- Command options and their arguments
+- File paths for template and schema files
+- Directory paths for `-d` and `--base-dir` options
+- And more!
+
+## Quick Start
+
+1. Set your OpenAI API key:
+
+```bash
+export OPENAI_API_KEY=your-api-key
+```
+
+### Example 1: Using stdin (Simplest)
+
+1. Create a template file `extract_person.j2`:
+
+```jinja
+Extract information about the person from this text: {{ stdin }}
+```
+
+2. Create a schema file `schema.json`:
+
+```json
+{
+  "type": "object",
+  "properties": {
+    "person": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "type": "string",
+          "description": "The person's full name"
+        },
+        "age": {
+          "type": "integer",
+          "description": "The person's age"
+        },
+        "occupation": {
+          "type": "string",
+          "description": "The person's job or profession"
+        }
+      },
+      "required": ["name", "age", "occupation"],
+      "additionalProperties": false
+    }
+  },
+  "required": ["person"],
+  "additionalProperties": false
+}
+```
+
+3. Run the CLI:
+
+```bash
+# Basic usage
+echo "John Smith is a 35-year-old software engineer" | ostruct run extract_person.j2 schema.json
+
+# For longer text using heredoc
+cat << EOF | ostruct run extract_person.j2 schema.json
+John Smith is a 35-year-old software engineer
+working at Tech Corp. He has been programming
+for over 10 years.
+EOF
+
+# With advanced options
+echo "John Smith is a 35-year-old software engineer" | \
+  ostruct run extract_person.j2 schema.json \
+  --model gpt-4o \
+  --sys-prompt "Extract precise information about the person" \
+  --temperature 0.7
+```
+
+The command will output:
+
+```json
+{
+  "person": {
+    "name": "John Smith",
+    "age": 35,
+    "occupation": "software engineer"
+  }
+}
+```
+
+### Example 2: Processing a Single File
+
+1. Create a template file `extract_from_file.j2`:
+
+```jinja
+Extract information about the person from this text: {{ text.content }}
+```
+
+2. Use the same schema file `schema.json` as above.
+
+3. Run the CLI:
+
+```bash
+# Basic usage
+ostruct run extract_from_file.j2 schema.json -f text input.txt
+
+# With advanced options
+ostruct run extract_from_file.j2 schema.json \
+  -f text input.txt \
+  --model gpt-4o \
+  --max-output-tokens 1000 \
+  --temperature 0.7
+```
+
+The command will output:
+
+```json
+{
+  "person": {
+    "name": "John Smith",
+    "age": 35,
+    "occupation": "software engineer"
+  }
+}
+```
+
+### Example 3: Processing Multiple Files
+
+1. Create a template file `extract_from_profiles.j2`:
+
+```jinja
+Extract information about the people from this data:
+
+{% for profile in profiles %}
+== {{ profile.name }}
+
+{{ profile.content }}
+
+{% endfor %}
+```
+
+2. Use the same schema file `schema.json` as above, but updated for multiple people:
+
+```json
+{
+  "type": "object",
+  "properties": {
+    "people": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "name": {
+            "type": "string",
+            "description": "The person's full name"
+          },
+          "age": {
+            "type": "integer",
+            "description": "The person's age"
+          },
+          "occupation": {
+            "type": "string",
+            "description": "The person's job or profession"
+          }
+        },
+        "required": ["name", "age", "occupation"],
+        "additionalProperties": false
+      }
+    }
+  },
+  "required": ["people"],
+  "additionalProperties": false
+}
+```
+
+3. Run the CLI:
+
+```bash
+# Basic usage
+ostruct run extract_from_profiles.j2 schema.json -p profiles "profiles/*.txt"
+
+# With advanced options
+ostruct run extract_from_profiles.j2 schema.json \
+  -p profiles "profiles/*.txt" \
+  --model gpt-4o \
+  --sys-prompt "Extract precise information about the person" \
+  --temperature 0.5
+```
+
+The command will output:
+
+```json
+{
+  "people": [
+    {
+      "name": "John Smith",
+      "age": 35,
+      "occupation": "software engineer"
+    },
+    {
+      "name": "Jane Doe",
+      "age": 28,
+      "occupation": "data scientist"
+    }
+  ]
+}
+```
+
+### About Template Files
+
+Template files use the `.j2` extension to indicate they contain Jinja2 template syntax. This convention:
+
+- Enables proper syntax highlighting in most editors
+- Makes it clear the file contains template logic
+- Follows industry standards for Jinja2 templates
+
+## CLI Options
+
+The CLI revolves around a single subcommand called `run`. Basic usage:
+
+```bash
+ostruct run <TASK_TEMPLATE> <SCHEMA_FILE> [OPTIONS]
+```
+
+Common options include:
+
+- File & Directory Inputs:
+  - `-f <NAME> <PATH>`: Map a single file to a variable name
+  - `-d <NAME> <DIR>`: Map a directory to a variable name
+  - `-p <NAME> <PATTERN>`: Map files matching a glob pattern to a variable name
+  - `-R, --recursive`: Enable recursive directory/pattern scanning
+
+- Variables:
+  - `-V name=value`: Define a simple string variable
+  - `-J name='{"key":"value"}'`: Define a JSON variable
+
+- Model Parameters:
+  - `-m, --model MODEL`: Select the OpenAI model (supported: gpt-4o, o1, o3-mini)
+  - `--temperature FLOAT`: Set sampling temperature (0.0-2.0)
+  - `--max-output-tokens INT`: Set maximum output tokens
+  - `--top-p FLOAT`: Set top-p sampling parameter (0.0-1.0)
+  - `--frequency-penalty FLOAT`: Adjust frequency penalty (-2.0-2.0)
+  - `--presence-penalty FLOAT`: Adjust presence penalty (-2.0-2.0)
+  - `--reasoning-effort [low|medium|high]`: Control model reasoning effort
+
+- System Prompt:
+  - `--sys-prompt TEXT`: Provide system prompt directly
+  - `--sys-file FILE`: Load system prompt from file
+  - `--ignore-task-sysprompt`: Ignore system prompt in template frontmatter
+
+- API Configuration:
+  - `--api-key KEY`: OpenAI API key (defaults to OPENAI_API_KEY env var)
+  - `--timeout FLOAT`: API timeout in seconds (default: 60.0)
+
+## Debug Options
+
+- `--debug-validation`: Show detailed schema validation debugging
+- `--debug-openai-stream`: Enable low-level debug output for OpenAI streaming
+- `--progress-level {none,basic,detailed}`: Set progress reporting level
+  - `none`: No progress indicators
+  - `basic`: Show key operation steps (default)
+  - `detailed`: Show all steps with additional info
+- `--show-model-schema`: Display the generated Pydantic model schema
+- `--verbose`: Enable verbose logging
+- `--dry-run`: Validate and render template without making API calls
+- `--no-progress`: Disable all progress indicators
+
+All debug and error logs are written to:
+
+- `~/.ostruct/logs/ostruct.log`: General application logs
+- `~/.ostruct/logs/openai_stream.log`: OpenAI streaming operations logs
+
+For more detailed documentation and examples, visit our [documentation](https://ostruct.readthedocs.io/).
+
+## Development
+
+To contribute or report issues, please visit our [GitHub repository](https://github.com/yaniv-golan/ostruct).
+
+## Development Setup
+
+1. Clone the repository:
+
+```bash
+git clone https://github.com/yanivgolan/ostruct.git
+cd ostruct
+```
+
+2. Install Poetry if you haven't already:
+
+```bash
+curl -sSL https://install.python-poetry.org | python3 -
+```
+
+3. Install dependencies:
+
+```bash
+poetry install
+```
+
+4. Install openai-structured in editable mode:
+
+```bash
+poetry add --editable ../openai-structured  # Adjust path as needed
+```
+
+5. Run tests:
+
+```bash
+poetry run pytest
+```
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## License
+
+This project is licensed under the MIT License - see the LICENSE file for details.
+

ostruct_cli-0.6.0.dist-info/RECORD

@@ -0,0 +1,43 @@
+ostruct/__init__.py,sha256=X6zo6V7ZNMv731Wi388aTVQngD1410ExGwGx4J6lpyo,187
+ostruct/cli/__init__.py,sha256=sYHKT6o1kFy1acbXejzAvVm8Cy8U91Yf1l4DlzquHKg,409
+ostruct/cli/base_errors.py,sha256=S1cQxoiALbXKPxzgLo6XdSWpzPRb7RKz0QARmu9Zt4g,5987
+ostruct/cli/cache_manager.py,sha256=ej3KrRfkKKZ_lEp2JswjbJ5bW2ncsvna9NeJu81cqqs,5192
+ostruct/cli/cli.py,sha256=wfO5Z8PPoP8eUn5CfhxrjrdMzfbvr4ryo_tsRST0LlU,74588
+ostruct/cli/click_options.py,sha256=WbRJdB9sO63ChN3fnCP7XWs73DHKl0C1ervfwL11am0,11371
+ostruct/cli/errors.py,sha256=zJdJ-AyzjCE8glVKbJGAcB-Mz1J1SlzTDJDmhqAVFYc,14930
+ostruct/cli/exit_codes.py,sha256=uNjvQeUGwU1mlUJYIDrExAn7YlwOXZo603yLAwpqIwk,338
+ostruct/cli/file_info.py,sha256=ilpT8IuckfhadLF1QQAPLXJp7p8kVpffDEEJ2erHPZU,14485
+ostruct/cli/file_list.py,sha256=jLuCd1ardoAXX8FNwPgIqEM-ixzr1xP5ZSqXo2lmrj0,11270
+ostruct/cli/file_utils.py,sha256=J3-6fbEGQ7KD_bU81pAxueHLv9XV0X7f8FSMt_0AJGQ,22537
+ostruct/cli/path_utils.py,sha256=j44q1OoLkqMErgK-qEuhuIZ1VyzqRIvNgxR1et9PoXA,4813
+ostruct/cli/progress.py,sha256=rj9nVEco5UeZORMbzd7mFJpFGJjbH9KbBFh5oTE5Anw,3415
+ostruct/cli/schema_validation.py,sha256=ohEuxJ0KF93qphj0JSZDnrxDn0C2ZU37g-U2JY03onM,8154
+ostruct/cli/security/__init__.py,sha256=CQpkCgTFYlA1p6atpQeNgIKtE4LZGUKt4EbytbGKpCs,846
+ostruct/cli/security/allowed_checker.py,sha256=N5UXlpjdj5zAbKk-lRDlHiHV3KtQHtJNhtZI_qGB4zw,1638
+ostruct/cli/security/base.py,sha256=q9YUdHEj2eg5w8GEw5403E9OQKIjZbEiaWsvYFnCGLw,1359
+ostruct/cli/security/case_manager.py,sha256=I_ZJSyntLuGx5qVzze559CI-OxsaNPSibkAN8zZ7PvE,2345
+ostruct/cli/security/errors.py,sha256=VZDOGGD-jYLf6E5gCkKxrE34RJXJP_CPWGOF5jV_r4I,5230
+ostruct/cli/security/normalization.py,sha256=qevvxW3hHDtD1cVvDym8LJEQD1AKenVB-0ZvjCYjn5E,5242
+ostruct/cli/security/safe_joiner.py,sha256=PHowCeBAkfHfPqRwuO5Com0OemGuq3cHkdu2p9IYNT0,7107
+ostruct/cli/security/security_manager.py,sha256=R54CgE7eG_0VybvjXj4fNn1jB-RHMUlnJ6Yw8BOtKKc,13512
+ostruct/cli/security/symlink_resolver.py,sha256=wtZdJ_T_0FOy6B1P5ty1odEXQk9vr8BzlWeAFD4huJE,16744
+ostruct/cli/security/types.py,sha256=15yuG_T4CXyAFFFdSWLjVS7ACmDGIPXhQpZ8awcDwCQ,2991
+ostruct/cli/security/windows_paths.py,sha256=qxC2H2kLwtmQ7YePYde3UrmOJcGnsLEebDLh242sUaI,13453
+ostruct/cli/serialization.py,sha256=ec0UswDE2onwtZVUoZaMCsGv6zW_tSKdBng2qVo6Ucs,704
+ostruct/cli/template_env.py,sha256=S2ZvxuMQMicodSVqUhrw0kOzbNmlpQjSHtWlOwjXCms,1538
+ostruct/cli/template_extensions.py,sha256=tJN3HGAS2yzGI8Up6STPday8NVL0VV6UCClBrtDKYr0,1623
+ostruct/cli/template_filters.py,sha256=SjuQxlM5S283TS2El_AbrzETGnYoQeTpmA9sv5et3QI,19222
+ostruct/cli/template_io.py,sha256=yUWO-8rZnSdX97DTMSEX8fG9CP1ISsOhm2NZN3Fab9A,8821
+ostruct/cli/template_rendering.py,sha256=vp_4gvrYLd_kbQi3TYrYNniXLTeLmTaitGVBQManXvo,13342
+ostruct/cli/template_schema.py,sha256=ckH4rUZnEgfm_BHS9LnMGr8LtDxRmZ0C6UBVrSp8KTc,19604
+ostruct/cli/template_utils.py,sha256=Lf1TvonlRA835nxyevEBSPTEbKiz_momvQYM0ZoZDZU,8034
+ostruct/cli/template_validation.py,sha256=AXa2zmsws1j-0CTFlp7fMiZR43iNLnj4h467up2JdgU,12693
+ostruct/cli/token_utils.py,sha256=r4KPEO3Sec18Q6mU0aClK6XGShvusgUggXEQgEPPlaA,1369
+ostruct/cli/utils.py,sha256=1UCl4rHjBWKR5EKugvlVGHiHjO3XXmqvkgeAUSyIPDU,831
+ostruct/cli/validators.py,sha256=BYFZeebCPZObTUjO1TaAMpsD6h7ROkYAFn9C7uf1Q68,2992
+ostruct/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ostruct_cli-0.6.0.dist-info/LICENSE,sha256=QUOY6QCYVxAiH8vdrUTDqe3i9hQ5bcNczppDSVpLTjk,1068
+ostruct_cli-0.6.0.dist-info/METADATA,sha256=Zrq8a-EvLhnZdOQBBlYvONWKo61XmdAR8934_OtHUa4,10426
+ostruct_cli-0.6.0.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+ostruct_cli-0.6.0.dist-info/entry_points.txt,sha256=NFq9IuqHVTem0j9zKjV8C1si_zGcP1RL6Wbvt9fUDXw,48
+ostruct_cli-0.6.0.dist-info/RECORD,,

{ostruct_cli-0.4.0.dist-info → ostruct_cli-0.6.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.0.1
+Generator: poetry-core 2.1.1
 Root-Is-Purelib: true
 Tag: py3-none-any