machine-dialect 0.1.0a1__py3-none-any.whl

machine_dialect/cfg/generate_with_ai.py

@@ -0,0 +1,187 @@
+#!/usr/bin/env python3
+"""Generate Machine Dialect™ code using AI models.
+
+This module provides functionality to generate Machine Dialect™ code using
+AI language models (such as OpenAI's GPT models). It includes code generation,
+validation, and configuration management for AI API integration.
+
+The module supports:
+- Generating Machine Dialect™ code from natural language descriptions
+- Validating generated code against the CFG parser
+- Flexible configuration through environment variables or config files
+- Command-line interface for code generation tasks
+- Saving generated code to files
+
+Note:
+    OpenAI integration is currently commented out. Uncomment the relevant
+    sections when the OpenAI library is installed.
+
+Example:
+    Generate code from the command line::
+
+        $ python -m machine_dialect.cfg.generate_with_ai "calculate rectangle area"
+        $ python -m machine_dialect.cfg.generate_with_ai "sort a list" --model gpt-4
+"""
+
+import argparse
+
+from machine_dialect.cfg import CFGParser
+
+# Uncomment when you have OpenAI installed
+# from openai import OpenAI
+from machine_dialect.cfg.config import ConfigLoader
+
+
+def generate_code(
+    task: str,
+    api_key: str | None = None,
+    model: str | None = None,
+    temperature: float = 0.7,
+    max_tokens: int = 500,
+    validate: bool = True,
+) -> str:
+    """Generate Machine Dialect™ code for a given task.
+
+    Args:
+        task: Description of what the code should do.
+        api_key: AI API key (overrides config/env).
+        model: AI model to use (overrides config/env).
+        temperature: Sampling temperature (0-2, lower = more deterministic).
+        max_tokens: Maximum tokens to generate.
+        validate: Whether to validate generated code.
+
+    Returns:
+        Generated Machine Dialect™ code.
+
+    Raises:
+        ValueError: If API key or model is not configured.
+
+    Example:
+        >>> code = generate_code(
+        ...     "calculate area",
+        ...     model="gpt-3.5-turbo",
+        ...     temperature=0.5
+        ... )
+        >>> print(code)
+        Set `width` to 10.
+        ...
+    """
+    # Load configuration
+    loader = ConfigLoader()
+    config = loader.load()
+
+    # Override with function arguments if provided
+    if api_key:
+        config.key = api_key
+    if model:
+        config.model = model
+
+    # Check configuration
+    if not config.key:
+        raise ValueError(loader.get_error_message())
+    if not config.model:
+        raise ValueError("No AI model configured. " + loader.get_error_message())
+
+    # Create OpenAI client
+    # Uncomment when you have OpenAI installed:
+    # from openai import OpenAI
+    # client = OpenAI(api_key=config.key)
+
+    print(f"\nModel: {config.model}")
+    print(f"Task: {task}")
+    print(f"Temperature: {temperature}")
+    print(f"Max tokens: {max_tokens}")
+
+    # Actual API call (uncomment when you have OpenAI):
+    # print(f"\nGenerating code with {config.model}...")
+    # generated_code = generate_with_openai(client, config.model, task, max_tokens, temperature)
+
+    # For demonstration, show configuration
+    print("\n" + "=" * 60)
+    print("Configuration:")
+    print("=" * 60)
+    print(f"Model: {config.model}")
+    print(f"API Key: {'*' * 10 if config.key else 'Not configured'}")
+
+    # Example of what would be returned
+    example_code = """Set `width` to 10.
+Set `height` to 5.
+Set `area` to width * height.
+Say "The area is: ".
+Say area."""
+
+    print("\n" + "=" * 60)
+    print(f"Example Generated Code (what {config.model} would return):")
+    print("=" * 60)
+    print(example_code)
+
+    # Validate if requested
+    if validate:
+        print("\n" + "=" * 60)
+        print("Validating generated code...")
+        print("=" * 60)
+
+        parser = CFGParser()
+        is_valid = parser.validate(example_code)
+
+        if is_valid:
+            print("✓ Generated code is syntactically valid!")
+        else:
+            print("✗ Generated code has syntax errors")
+
+    return example_code
+
+
+def main() -> int:
+    """Main entry point for the AI code generation script.
+
+    Parses command-line arguments and generates Machine Dialect™ code
+    based on the provided task description. Supports configuration
+    overrides, validation, and saving output to files.
+
+    Returns:
+        Exit code: 0 for success, 1 for errors.
+
+    Example:
+        >>> # Generate code for calculating area
+        >>> sys.argv = ['prog', 'calculate area of rectangle']
+        >>> main()
+        0
+    """
+    parser = argparse.ArgumentParser(description="Generate Machine Dialect™ code using AI models")
+    parser.add_argument("task", help="Description of what the code should do")
+    parser.add_argument("--api-key", help="AI API key (overrides config/env)")
+    parser.add_argument("--model", help="AI model to use (overrides config/env)")
+    parser.add_argument("--temperature", type=float, default=0.7, help="Sampling temperature (0-2, default: 0.7)")
+    parser.add_argument("--max-tokens", type=int, default=500, help="Maximum tokens to generate (default: 500)")
+    parser.add_argument("--no-validate", action="store_true", help="Skip validation of generated code")
+    parser.add_argument("--save", help="Save generated code to file")
+
+    args = parser.parse_args()
+
+    try:
+        code = generate_code(
+            task=args.task,
+            api_key=args.api_key,
+            model=args.model,
+            temperature=args.temperature,
+            max_tokens=args.max_tokens,
+            validate=not args.no_validate,
+        )
+
+        if args.save:
+            with open(args.save, "w") as f:
+                f.write(code)
+            print(f"\nCode saved to: {args.save}")
+
+    except Exception as e:
+        print(f"Error: {e}")
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(main())

machine_dialect/cfg/openai_generation.py

@@ -0,0 +1,200 @@
+"""Grammar-based generation module for Machine Dialect™ using GPT-5's CFG support."""
+
+from pathlib import Path
+from typing import Any
+
+
+def generate_with_openai(
+    client: Any,  # OpenAI client
+    model: str,
+    task_description: str,
+    max_tokens: int = 500,
+    temperature: float = 0.7,
+) -> tuple[str, dict[str, Any]]:
+    """Generate Machine Dialect™ code using GPT-5's context-free grammar constraints.
+
+    This function uses GPT-5's custom tools with CFG to ensure syntactically correct
+    Machine Dialect™ code generation. The model is constrained to only produce
+    strings that match the Machine Dialect™ grammar.
+
+    Args:
+        client: OpenAI client instance.
+        model: Model name (must support CFG, e.g., 'gpt-5').
+        task_description: What the code should do.
+        max_tokens: Maximum tokens to generate.
+        temperature: Sampling temperature (0-2).
+
+    Returns:
+        Tuple of (generated_code, token_info) where:
+        - generated_code: Machine Dialect™ code that is syntactically valid.
+        - token_info: Dictionary with prompt_tokens, completion_tokens, total_tokens.
+
+    Raises:
+        ValueError: If the model doesn't support CFG or response is invalid.
+    """
+    # Check if model supports CFG (currently only GPT-5 family)
+    if "gpt-5" not in model.lower():
+        raise ValueError(
+            f"Model '{model}' does not support context-free grammar constraints. "
+            "Please use a GPT-5 model (gpt-5, gpt-5-mini, or gpt-5-nano)."
+        )
+
+    # Create the CFG definition for Machine Dialect™
+    machine_dialect_cfg = _get_machine_dialect_cfg()
+
+    # Create the API request using GPT-5's custom tools with CFG
+    # Note: GPT-5 doesn't support temperature parameter (always uses 1.0)
+    import time
+
+    api_start = time.time()
+
+    try:
+        response = client.responses.create(
+            model=model,
+            input=[
+                {
+                    "role": "developer",
+                    "content": (
+                        "You are a Machine Dialect™ code generator. Generate code that performs the "
+                        "requested task using the Machine Dialect™ language. The output must conform "
+                        "to the provided context-free grammar.\n"
+                        "IMPORTANT:\n"
+                        "- Write in English even if the instruction is in another language.\n"
+                        "- Always define variables before trying to use them.\n"
+                        "- When creating utilities, define proper Inputs (parameters the utility accepts) "
+                        "and Outputs (values it returns) sections.\n"
+                        "- Don't hardcode values that should be parameters - use the Inputs section instead."
+                    ),
+                },
+                {"role": "user", "content": f"Generate Machine Dialect™ code for: {task_description}"},
+            ],
+            tools=[
+                {
+                    "type": "custom",
+                    "name": "machine_dialect_generator",
+                    "description": "Generates syntactically valid Machine Dialect™ code",
+                    "format": machine_dialect_cfg,
+                }
+            ],
+            parallel_tool_calls=False,
+            timeout=30.0,  # 30 second timeout
+            # temperature parameter removed - GPT-5 doesn't support it
+        )
+
+        api_time = time.time() - api_start
+        if api_time > 5.0:  # Log if it takes more than 5 seconds
+            print(f"   ⚠️ API call took {api_time:.2f}s")
+
+    except Exception as e:
+        api_time = time.time() - api_start
+        raise ValueError(f"API call failed after {api_time:.2f}s: {e!s}") from e
+
+    # Extract the generated code from the response
+    # The response should have an output_text attribute directly
+    if hasattr(response, "output_text"):
+        generated_code = response.output_text
+    elif hasattr(response, "output"):
+        # Fallback to output attribute if output_text doesn't exist
+        if isinstance(response.output, list) and len(response.output) > 1:
+            # Try to get the second output (tool output)
+            tool_output = response.output[1]
+
+            # Check various attributes on the tool output
+            if hasattr(tool_output, "text"):
+                generated_code = tool_output.text
+            elif hasattr(tool_output, "input"):
+                generated_code = tool_output.input
+            elif hasattr(tool_output, "tool_input"):
+                generated_code = tool_output.tool_input
+            elif hasattr(tool_output, "content"):
+                generated_code = tool_output.content
+            else:
+                generated_code = str(tool_output)
+        elif isinstance(response.output, str):
+            generated_code = response.output
+        else:
+            generated_code = str(response.output)
+    else:
+        # Last resort: try to extract from string representation
+        response_str = str(response)
+        if "output_text=" in response_str:
+            import re
+
+            match = re.search(r"output_text='([^']*)'", response_str)
+            if not match:
+                match = re.search(r'output_text="([^"]*)"', response_str)
+            if match:
+                generated_code = match.group(1)
+            else:
+                raise ValueError(f"Could not extract code from response: {response_str[:200]}...")
+        else:
+            raise ValueError(f"Response has no output_text or output attribute: {dir(response)}")
+
+    if not generated_code or generated_code == "None":
+        # Provide more helpful error message
+        error_msg = "Failed to extract valid code from GPT-5 response.\n"
+        error_msg += f"Response type: {type(response).__name__}\n"
+        error_msg += f"Response attributes: {[attr for attr in dir(response) if not attr.startswith('_')][:10]}\n"
+        if hasattr(response, "output"):
+            error_msg += f"Output type: {type(response.output).__name__}\n"
+        error_msg += f"Extracted value: {repr(generated_code[:100]) if generated_code else 'None'}"
+        raise ValueError(error_msg)
+
+    # Extract token usage if available
+    token_info = {}
+    if hasattr(response, "usage"):
+        usage = response.usage
+        if usage:
+            token_info["prompt_tokens"] = getattr(usage, "prompt_tokens", None)
+            token_info["completion_tokens"] = getattr(usage, "completion_tokens", None)
+            token_info["total_tokens"] = getattr(usage, "total_tokens", None)
+
+            # If individual counts are not available but total is, try to estimate
+            if token_info["total_tokens"] and not token_info["prompt_tokens"]:
+                # Can't accurately split, just show total
+                token_info["prompt_tokens"] = None
+                token_info["completion_tokens"] = None
+
+    # Return both code and token info as a tuple
+    return (str(generated_code), token_info)
+
+
+# Cache the grammar to avoid re-reading the file
+_cached_grammar: dict[str, Any] | None = None
+
+
+def _get_machine_dialect_cfg() -> dict[str, Any]:
+    """Get the Machine Dialect™ context-free grammar in GPT-5 format.
+
+    Returns:
+        Dictionary containing the CFG definition for GPT-5's custom tools.
+    """
+    global _cached_grammar
+
+    if _cached_grammar is None:
+        # Read the Machine Dialect™ Lark grammar file for GPT-5
+        grammar_path = Path(__file__).parent / "machine_dialect.lark"
+
+        with open(grammar_path) as f:
+            lark_grammar = f.read()
+
+        _cached_grammar = {
+            "type": "grammar",
+            "syntax": "lark",  # Using Lark syntax as required by GPT-5
+            "definition": lark_grammar,
+        }
+
+    return _cached_grammar
+
+
+def validate_model_support(model: str) -> bool:
+    """Check if a model supports context-free grammar constraints.
+
+    Args:
+        model: The model name to check.
+
+    Returns:
+        True if the model supports CFG, False otherwise.
+    """
+    supported_models = ["gpt-5", "gpt-5-mini", "gpt-5-nano"]
+    return any(supported in model.lower() for supported in supported_models)

machine_dialect/cfg/parser.py

@@ -0,0 +1,94 @@
+"""CFG Parser for simplified Machine Dialect™ using Lark."""
+
+from pathlib import Path
+from typing import Any
+
+from lark import Lark, Token, Tree
+from lark.exceptions import LarkError
+
+
+class CFGParser:
+    """Parser for simplified Machine Dialect™ using Lark CFG."""
+
+    def __init__(self) -> None:
+        """Initialize the parser with the grammar file."""
+        grammar_path = Path(__file__).parent / "machine_dialect.lark"
+        with open(grammar_path) as f:
+            grammar_content = f.read()
+
+        self.parser = Lark(grammar_content, parser="lalr", start="start", debug=False)
+
+    def parse(self, code: str) -> Tree[Any]:
+        """Parse Machine Dialect™ code into an AST.
+
+        Args:
+            code: The Machine Dialect™ code to parse.
+
+        Returns:
+            A Lark Tree representing the parsed AST.
+
+        Raises:
+            LarkError: If the code cannot be parsed.
+        """
+        # Handle empty or whitespace-only input
+        if not code or not code.strip():
+            # Return an empty tree for empty programs
+            from lark import Tree
+
+            return Tree("program", [Tree("statement_list", [])])
+
+        try:
+            return self.parser.parse(code)
+        except LarkError as e:
+            # Convert Lark errors to match main parser behavior
+            raise ValueError(f"Syntax error: {e}") from e
+
+    def validate(self, code: str) -> bool:
+        """Validate if the code conforms to the grammar.
+
+        Args:
+            code: The Machine Dialect™ code to validate.
+
+        Returns:
+            True if valid, False otherwise.
+        """
+        try:
+            self.parse(code)
+            return True
+        except (LarkError, ValueError):
+            return False
+
+    def get_grammar_rules(self) -> str:
+        """Get the grammar rules in a format suitable for GPT-5 CFG.
+
+        Returns:
+            String representation of grammar rules.
+        """
+        grammar_path = Path(__file__).parent / "machine_dialect.lark"
+        with open(grammar_path) as f:
+            return f.read()
+
+    def tree_to_dict(self, tree: Tree[Any] | Token) -> dict[str, Any]:
+        """Convert a Lark tree to a dictionary representation.
+
+        Args:
+            tree: The Lark tree or token to convert.
+
+        Returns:
+            Dictionary representation of the tree.
+        """
+        if isinstance(tree, Token):
+            return {"type": "token", "name": tree.type, "value": tree.value}
+
+        return {"type": "tree", "name": tree.data, "children": [self.tree_to_dict(child) for child in tree.children]}
+
+    def pretty_print(self, tree: Tree[Any]) -> str:
+        """Pretty print a parsed tree.
+
+        Args:
+            tree: The Lark tree to print.
+
+        Returns:
+            A formatted string representation of the tree.
+        """
+        return str(tree.pretty())

machine_dialect/cfg/tests/__init__.py

@@ -0,0 +1 @@
+"""Tests for the CFG module."""