pdd-cli 0.0.2__py3-none-any.whl

pdd/postprocess.py

@@ -0,0 +1,121 @@
+from typing import Tuple
+from rich import print
+from pydantic import BaseModel, Field
+from .load_prompt_template import load_prompt_template
+from .llm_invoke import llm_invoke
+
+class ExtractedCode(BaseModel):
+    """Pydantic model for the extracted code."""
+    extracted_code: str = Field(description="The extracted code from the LLM output")
+
+def postprocess_0(text: str) -> str:
+    """
+    Simple code extraction for strength = 0.
+    Extracts code between triple backticks.
+    """
+    lines = text.split('\n')
+    code_lines = []
+    in_code_block = False
+    
+    for line in lines:
+        if line.startswith('```'):
+            if not in_code_block:
+                # Skip the language identifier line
+                in_code_block = True
+                continue
+            else:
+                in_code_block = False
+                continue
+        if in_code_block:
+            code_lines.append(line)
+    
+    return '\n'.join(code_lines)
+
+def postprocess(
+    llm_output: str,
+    language: str,
+    strength: float = 0.9,
+    temperature: float = 0,
+    verbose: bool = False
+) -> Tuple[str, float, str]:
+    """
+    Extract code from LLM output string.
+    
+    Args:
+        llm_output (str): The string output from the LLM containing code sections
+        language (str): The programming language of the code to extract
+        strength (float): The strength of the LLM model to use (0-1)
+        temperature (float): The temperature parameter for the LLM (0-1)
+        verbose (bool): Whether to print detailed processing information
+    
+    Returns:
+        Tuple[str, float, str]: (extracted_code, total_cost, model_name)
+    """
+    try:
+        # Input validation
+        if not llm_output or not isinstance(llm_output, str):
+            raise ValueError("llm_output must be a non-empty string")
+        if not language or not isinstance(language, str):
+            raise ValueError("language must be a non-empty string")
+        if not 0 <= strength <= 1:
+            raise ValueError("strength must be between 0 and 1")
+        if not 0 <= temperature <= 1:
+            raise ValueError("temperature must be between 0 and 1")
+
+        # Step 1: If strength is 0, use simple extraction
+        if strength == 0:
+            if verbose:
+                print("[blue]Using simple code extraction (strength = 0)[/blue]")
+            return (postprocess_0(llm_output), 0.0, "simple_extraction")
+
+        # Step 2: Load the prompt template
+        prompt_template = load_prompt_template("extract_code_LLM")
+        if not prompt_template:
+            raise ValueError("Failed to load prompt template")
+
+        if verbose:
+            print("[blue]Loaded prompt template for code extraction[/blue]")
+
+        # Step 3: Process using llm_invoke
+        input_json = {
+            "llm_output": llm_output,
+            "language": language
+        }
+
+        response = llm_invoke(
+            prompt=prompt_template,
+            input_json=input_json,
+            strength=strength,
+            temperature=temperature,
+            verbose=verbose,
+            output_pydantic=ExtractedCode
+        )
+
+        if not response or 'result' not in response:
+            raise ValueError("Failed to get valid response from LLM")
+
+        extracted_code: ExtractedCode = response['result']
+        code_text = extracted_code.extracted_code
+
+        # Step 3c: Remove triple backticks and language identifier if present
+        lines = code_text.split('\n')
+        if lines and lines[0].startswith('```'):
+            lines = lines[1:]
+        if lines and lines[-1].startswith('```'):
+            lines = lines[:-1]
+        
+        final_code = '\n'.join(lines)
+
+        if verbose:
+            print("[green]Successfully extracted code[/green]")
+
+        # Step 4: Return the results
+        return (
+            final_code,
+            response['cost'],
+            response['model_name']
+        )
+
+    except Exception as e:
+        print(f"[red]Error in postprocess: {str(e)}[/red]")
+        raise

pdd/postprocess_0.py

@@ -0,0 +1,52 @@
+#Here's the implementation of the `postprocess_0` function based on your requirements:
+#
+#```python
+from .get_comment import get_comment
+from .comment_line import comment_line
+from .find_section import find_section
+
+def postprocess_0(llm_output: str, language: str) -> str:
+    # Step 1: Get the comment character for the specified language
+    comment_char = get_comment(language)
+
+    # Step 2: Find code sections in the llm_output
+    lines = llm_output.splitlines()
+    sections = find_section(lines)
+
+    # Step 3: Find the largest section of the specified language
+    largest_section = None
+    max_size = 0
+    for code_lang, start, end in sections:
+        if code_lang.lower() == language.lower():
+            size = end - start
+            if size > max_size:
+                max_size = size
+                largest_section = (start, end)
+
+    # Step 4 & 5: Comment out lines outside the largest section
+    processed_lines = []
+    in_code_section = False
+    for i, line in enumerate(lines):
+        if largest_section and i == largest_section[0] + 1:
+            in_code_section = True
+        elif largest_section and i == largest_section[1]:
+            in_code_section = False
+
+        if not in_code_section:
+            processed_lines.append(comment_line(line, comment_char))
+        else:
+            processed_lines.append(line)
+
+    # Return the processed string
+    return '\n'.join(processed_lines)
+#```
+#
+#This implementation follows the steps you outlined:
+#
+#1. We use `get_comment` to get the appropriate comment character for the specified language.
+#2. We use `find_section` to identify all code sections in the input.
+#3. We find the largest section of code in the specified language.
+#4. We iterate through the lines, commenting out everything outside the largest section of the specified language using `comment_line`.
+#5. Finally, we join the processed lines and return the result as a string.
+#
+#This function will produce a string where only the largest section of code in the specified language is left uncommented, while all other text and code sections are commented out using the appropriate comment character for the language.

pdd/preprocess.py

@@ -0,0 +1,199 @@
+import os
+import re
+import subprocess
+from typing import List
+from rich import print
+from rich.console import Console
+from rich.panel import Panel
+
+console = Console()
+
+def preprocess(prompt: str, recursive: bool = False, double_curly_brackets: bool = True, exclude_keys: List[str] = None) -> str:
+    """
+    Preprocess the given prompt by handling includes, specific tags, and doubling curly brackets.
+
+    :param prompt: The input text to preprocess.
+    :param recursive: Whether to recursively preprocess included content.
+    :param double_curly_brackets: Whether to double curly brackets in the text.
+    :param exclude_keys: List of keys to exclude from curly bracket doubling.
+    :return: The preprocessed text.
+    """
+    console.print(Panel("Starting preprocessing", style="bold green"))
+
+    # Process includes in triple backticks
+    prompt = process_backtick_includes(prompt, recursive)
+
+    # Process specific tags without adding closing tags
+    prompt = process_specific_tags(prompt, recursive)
+
+    # Double curly brackets if needed
+    if double_curly_brackets:
+        prompt = double_curly(prompt, exclude_keys)
+
+    console.print(Panel("Preprocessing complete", style="bold green"))
+    return prompt
+
+
+def process_backtick_includes(text: str, recursive: bool) -> str:
+    """
+    Process includes within triple backticks in the text.
+
+    :param text: The input text containing backtick includes.
+    :param recursive: Whether to recursively preprocess included content.
+    :return: The text with includes processed.
+    """
+    pattern = r"```<(.*?)>```"
+    matches = re.findall(pattern, text)
+
+    for match in matches:
+        console.print(f"Processing include: [cyan]{match}[/cyan]")
+        file_path = get_file_path(match)
+        try:
+            with open(file_path, 'r') as file:
+                content = file.read()
+                if recursive:
+                    content = preprocess(content, recursive, False)
+                text = text.replace(f"```<{match}>```", f"```{content}```")
+        except FileNotFoundError:
+            console.print(f"[bold red]Warning:[/bold red] File not found: {file_path}")
+
+    return text
+
+
+def process_specific_tags(text: str, recursive: bool) -> str:
+    """
+    Process specific tags in the text without adding closing tags.
+
+    :param text: The input text containing specific tags.
+    :param recursive: Whether to recursively preprocess included content.
+    :return: The text with specific tags processed.
+    """
+    def process_tag(match: re.Match) -> str:
+        pre_whitespace = match.group(1)
+        tag = match.group(2)
+        content = match.group(3) if match.group(3) else ""
+        post_whitespace = match.group(4)
+
+        if tag == 'include':
+            file_path = get_file_path(content.strip())
+            console.print(f"Processing XML include: [cyan]{file_path}[/cyan]")
+            try:
+                with open(file_path, 'r') as file:
+                    included_content = file.read()
+                    if recursive:
+                        included_content = preprocess(included_content, recursive, False)
+                    return pre_whitespace + included_content + post_whitespace
+            except FileNotFoundError:
+                console.print(f"[bold red]Warning:[/bold red] File not found: {file_path}")
+                return pre_whitespace + post_whitespace
+        elif tag == 'pdd':
+            return pre_whitespace + post_whitespace
+        elif tag == 'shell':
+            command = content.strip()
+            console.print(f"Executing shell command: [cyan]{command}[/cyan]")
+            try:
+                result = subprocess.run(command, shell=True, check=True, capture_output=True, text=True)
+                return pre_whitespace + result.stdout + post_whitespace
+            except subprocess.CalledProcessError as e:
+                console.print(f"[bold red]Error:[/bold red] Shell command failed: {e}")
+                return pre_whitespace + f"Error: {e}" + post_whitespace
+        else:
+            return match.group(0)  # Return the original match for any other tags
+
+    # Process only specific tags, capturing whitespace around them
+    pattern = r'(\s*)<(include|pdd|shell)(?:\s+[^>]*)?(?:>(.*?)</\2>|/|>)(\s*)'
+    return re.sub(pattern, process_tag, text, flags=re.DOTALL)
+
+
+def get_file_path(file_name: str) -> str:
+    """
+    Get the full file path based on the current directory ('./').
+
+    :param file_name: The name of the file to locate.
+    :return: The full path to the file.
+    """
+    pdd_path = './'  # Using './' as the base path
+    return os.path.join(pdd_path, file_name)
+
+
+def double_curly(text: str, exclude_keys: List[str] = None) -> str:
+    """
+    Double the curly brackets in the text, excluding specified keys.
+    Supports nested curly brackets and handles all code blocks uniformly.
+
+    :param text: The input text with single curly brackets.
+    :param exclude_keys: List of keys to exclude from doubling.
+    :return: The text with doubled curly brackets.
+    """
+    console.print("Doubling curly brackets")
+    if exclude_keys is None:
+        exclude_keys = []
+
+    # console.print(f"Before doubling:\n{text}")
+
+    # Define the pattern for all code blocks (e.g., ```javascript, ```json)
+    code_pattern = r"```[\w]*\n[\s\S]*?```"
+
+    # Split the text into code and non-code segments
+    parts = re.split(f"({code_pattern})", text)
+
+    processed_parts = []
+    placeholder_mapping = {}
+    placeholder_prefix_excl = "__EXCLUDE_KEY_PLACEHOLDER_"
+    placeholder_suffix = "__"
+    placeholder_prefix_empty = "__EMPTY_BRACE_PLACEHOLDER_"
+
+    placeholder_counter = 0
+
+    for part in parts:
+        if re.match(code_pattern, part):
+            # It's a code block; process separately
+            console.print("Processing code block for curly brackets")
+            first_line_end = part.find('\n') + 1
+            code_content = part[first_line_end:-3]  # Exclude the last ```
+            # Double curly brackets inside the code block
+            code_content = re.sub(r'(?<!{){(?!{)', '{{', code_content)
+            code_content = re.sub(r'(?<!})}(?!})', '}}', code_content)
+            # Reconstruct the code block
+            processed_part = part[:first_line_end] + code_content + part[-3:]
+            processed_parts.append(processed_part)
+        else:
+            # It's a non-code segment
+            temp_part = part
+
+            # Step 1: Protect excluded keys by replacing {exclude_key} with placeholders
+            for key in exclude_keys:
+                pattern_excl = r'\{' + re.escape(key) + r'\}'
+                placeholder_excl = f"{placeholder_prefix_excl}{placeholder_counter}{placeholder_suffix}"
+                temp_part = re.sub(pattern_excl, placeholder_excl, temp_part)
+                placeholder_mapping[placeholder_excl] = f"{{{key}}}"
+                placeholder_counter += 1
+
+            # Step 2: Protect empty braces '{}' by replacing with placeholders
+            pattern_empty = r'\{\}'
+            placeholder_empty = f"{placeholder_prefix_empty}{placeholder_counter}{placeholder_suffix}"
+            temp_part = re.sub(pattern_empty, placeholder_empty, temp_part)
+            placeholder_mapping[placeholder_empty] = '{{}}'
+            placeholder_counter += 1
+
+            # Step 3: Replace single '{' with '{{' and '}' with '}}'
+            temp_part = re.sub(r'(?<!{){(?!{)', '{{', temp_part)
+            temp_part = re.sub(r'(?<!})}(?!})', '}}', temp_part)
+
+            # Step 4: Restore excluded keys from placeholders
+            for placeholder, original in placeholder_mapping.items():
+                if original != '{{}}':
+                    temp_part = temp_part.replace(placeholder, original)
+
+            # Step 5: Restore empty braces from placeholders
+            for placeholder, original in placeholder_mapping.items():
+                if original == '{{}}':
+                    temp_part = temp_part.replace(placeholder, original)
+
+            processed_parts.append(temp_part)
+
+    # Reconstruct the full text after processing
+    text = ''.join(processed_parts)
+
+    # console.print(f"After doubling:\n{text}")
+    return text

pdd/preprocess_main.py

@@ -0,0 +1,72 @@
+import csv
+import sys
+from typing import Tuple, Optional
+import click
+from rich import print as rprint
+
+from .construct_paths import construct_paths
+from .preprocess import preprocess
+from .xml_tagger import xml_tagger
+
+def preprocess_main(
+    ctx: click.Context, prompt_file: str, output: Optional[str], xml: bool, recursive: bool, double: bool, exclude: list
+) -> Tuple[str, float, str]:
+    """
+    CLI wrapper for preprocessing prompts.
+
+    :param ctx: Click context object containing CLI options and parameters.
+    :param prompt_file: Path to the prompt file to preprocess.
+    :param output: Optional path where to save the preprocessed prompt.
+    :param xml: If True, insert XML delimiters for better structure.
+    :param recursive: If True, recursively preprocess all prompt files in the prompt file.
+    :param double: If True, curly brackets will be doubled.
+    :param exclude: List of keys to exclude from curly bracket doubling.
+    :return: Tuple containing the preprocessed prompt, total cost, and model name used.
+    """
+    try:
+        # Construct file paths
+        input_file_paths = {"prompt_file": prompt_file}
+        command_options = {"output": output}
+        input_strings, output_file_paths, _ = construct_paths(
+            input_file_paths=input_file_paths,
+            force=ctx.obj.get("force", False),
+            quiet=ctx.obj.get("quiet", False),
+            command="preprocess",
+            command_options=command_options,
+        )
+
+        # Load prompt file
+        prompt = input_strings["prompt_file"]
+
+        if xml:
+            # Use xml_tagger to add XML delimiters
+            strength = ctx.obj.get("strength", 0.5)
+            temperature = ctx.obj.get("temperature", 0.0)
+            verbose = ctx.obj.get("verbose", False)
+            xml_tagged, total_cost, model_name = xml_tagger(prompt, strength, temperature, verbose)
+            processed_prompt = xml_tagged
+        else:
+            # Preprocess the prompt
+            processed_prompt = preprocess(prompt, recursive, double, exclude_keys=exclude)
+            total_cost, model_name = 0.0, "N/A"
+
+        # Save the preprocessed prompt
+        with open(output_file_paths["output"], "w") as f:
+            f.write(processed_prompt)
+
+        # Provide user feedback
+        if not ctx.obj.get("quiet", False):
+            rprint("[bold green]Prompt preprocessing completed successfully.[/bold green]")
+            if xml:
+                rprint(f"[bold]XML Tagging used: {model_name}[/bold]")
+            else:
+                rprint(f"[bold]Model used: {model_name}[/bold]")
+            rprint(f"[bold]Total cost: ${total_cost:.6f}[/bold]")
+            rprint(f"[bold]Preprocessed prompt saved to:[/bold] {output_file_paths['output']}")
+
+        return processed_prompt, total_cost, model_name
+
+    except Exception as e:
+        if not ctx.obj.get("quiet", False):
+            rprint(f"[bold red]Error during preprocessing:[/bold red] {e}")
+        sys.exit(1)

pdd/process_csv_change.py

@@ -0,0 +1,182 @@
+# process_csv_change.py
+
+from typing import List, Dict, Tuple
+import os
+import csv
+from pathlib import Path
+import logging
+
+from rich.console import Console
+from rich.pretty import Pretty
+from rich.panel import Panel
+
+from .change import change  # Relative import for the internal change function
+
+console = Console()
+
+# Set up logging
+logging.basicConfig(level=logging.WARNING)
+logger = logging.getLogger(__name__)
+
+def process_csv_change(
+    csv_file: str,
+    strength: float,
+    temperature: float,
+    code_directory: str,
+    language: str,
+    extension: str,
+    budget: float
+) -> Tuple[bool, List[Dict[str, str]], float, str]:
+    """
+    Processes a CSV file to apply changes to code prompts using an LLM model.
+
+    Args:
+        csv_file (str): Path to the CSV file containing 'prompt_name' and 'change_instructions' columns.
+        strength (float): Strength parameter for the LLM model (0.0 to 1.0).
+        temperature (float): Temperature parameter for the LLM model.
+        code_directory (str): Path to the directory where code files are stored.
+        language (str): Programming language of the code files.
+        extension (str): File extension of the code files.
+        budget (float): Maximum allowed total cost for the change process.
+
+    Returns:
+        Tuple[bool, List[Dict[str, str]], float, str]:
+            - success (bool): Indicates if changes were successfully made within the budget and without errors.
+            - list_of_jsons (List[Dict[str, str]]): List of dictionaries with 'file_name' and 'modified_prompt'.
+            - total_cost (float): Total accumulated cost of all change attempts.
+            - model_name (str): Name of the LLM model used.
+    """
+    list_of_jsons: List[Dict[str, str]] = []
+    total_cost: float = 0.0
+    model_name: str = ""
+    success: bool = False
+    any_failures: bool = False  # Track if any failures occur
+
+    # Validate inputs
+    if not os.path.isfile(csv_file):
+        console.print(f"[bold red]Error:[/bold red] CSV file '{csv_file}' does not exist.")
+        return success, list_of_jsons, total_cost, model_name
+
+    if not (0.0 <= strength <= 1.0):
+        console.print(f"[bold red]Error:[/bold red] 'strength' must be between 0 and 1. Given: {strength}")
+        return success, list_of_jsons, total_cost, model_name
+
+    if not (0.0 <= temperature <= 1.0):
+        console.print(f"[bold red]Error:[/bold red] 'temperature' must be between 0 and 1. Given: {temperature}")
+        return success, list_of_jsons, total_cost, model_name
+
+    code_dir_path = Path(code_directory)
+    if not code_dir_path.is_dir():
+        console.print(f"[bold red]Error:[/bold red] Code directory '{code_directory}' does not exist or is not a directory.")
+        return success, list_of_jsons, total_cost, model_name
+
+    try:
+        with open(csv_file, mode='r', newline='', encoding='utf-8') as csvfile:
+            reader = csv.DictReader(csvfile)
+            if 'prompt_name' not in reader.fieldnames or 'change_instructions' not in reader.fieldnames:
+                console.print("[bold red]Error:[/bold red] CSV file must contain 'prompt_name' and 'change_instructions' columns.")
+                return success, list_of_jsons, total_cost, model_name
+
+            for row_number, row in enumerate(reader, start=1):
+                prompt_name = row.get('prompt_name', '').strip()
+                change_instructions = row.get('change_instructions', '').strip()
+
+                if not prompt_name:
+                    console.print(f"[yellow]Warning:[/yellow] Missing 'prompt_name' in row {row_number}. Skipping.")
+                    any_failures = True
+                    continue
+
+                if not change_instructions:
+                    console.print(f"[yellow]Warning:[/yellow] Missing 'change_instructions' in row {row_number}. Skipping.")
+                    any_failures = True
+                    continue
+
+                # Parse the prompt_name to get the input_code filename
+                try:
+                    prompt_path = Path(prompt_name)
+                    base_name = prompt_path.stem  # Removes suffix
+                    # Remove the '_<language>' part if present
+                    if '_' in base_name:
+                        base_name = base_name.rsplit('_', 1)[0]
+                    input_code_name = f"{base_name}{extension}"
+                    input_code_path = code_dir_path / input_code_name
+
+                    if not input_code_path.is_file():
+                        console.print(f"[yellow]Warning:[/yellow] Input code file '{input_code_path}' does not exist. Skipping row {row_number}.")
+                        logger.warning(f"Input code file '{input_code_path}' does not exist for row {row_number}")
+                        any_failures = True
+                        continue
+
+                    # Check if prompt file exists
+                    if not prompt_path.is_file():
+                        console.print(f"[yellow]Warning:[/yellow] Prompt file '{prompt_name}' does not exist. Skipping row {row_number}.")
+                        logger.warning(f"Prompt file '{prompt_name}' does not exist for row {row_number}")
+                        any_failures = True
+                        continue
+
+                    # Read the input_code from the file
+                    with open(input_code_path, 'r', encoding='utf-8') as code_file:
+                        input_code = code_file.read()
+
+                    # Read the input_prompt from the prompt file
+                    with open(prompt_path, 'r', encoding='utf-8') as prompt_file:
+                        input_prompt = prompt_file.read()
+
+                    # Call the change function
+                    modified_prompt, cost, current_model_name = change(
+                        input_prompt=input_prompt,
+                        input_code=input_code,
+                        change_prompt=change_instructions,
+                        strength=strength,
+                        temperature=temperature
+                    )
+
+                    # Accumulate the total cost
+                    total_cost += cost
+
+                    # Check if budget is exceeded
+                    if total_cost > budget:
+                        console.print(f"[bold red]Budget exceeded after row {row_number}. Stopping further processing.[/bold red]")
+                        any_failures = True
+                        break
+
+                    # Set the model_name (assumes all calls use the same model)
+                    if not model_name:
+                        model_name = current_model_name
+                    elif model_name != current_model_name:
+                        console.print(f"[yellow]Warning:[/yellow] Model name changed from '{model_name}' to '{current_model_name}' in row {row_number}.")
+
+                    # Add to the list_of_jsons
+                    list_of_jsons.append({
+                        "file_name": prompt_name,
+                        "modified_prompt": modified_prompt
+                    })
+
+                    console.print(Panel(f"[green]Row {row_number} processed successfully.[/green]"))
+
+                except Exception as e:
+                    console.print(f"[red]Error:[/red] Failed to process 'prompt_name' in row {row_number}: {str(e)}")
+                    logger.exception(f"Failed to process row {row_number}")
+                    any_failures = True
+                    continue
+
+        # Determine success based on whether total_cost is within budget and no failures occurred
+        success = (total_cost <= budget) and not any_failures
+
+        # Pretty print the results
+        console.print(Panel(f"[bold]Processing Complete[/bold]\n"
+                           f"Success: {'Yes' if success else 'No'}\n"
+                           f"Total Cost: ${total_cost:.6f}\n"
+                           f"Model Used: {model_name if model_name else 'N/A'}"))
+
+        # Optionally, pretty print the list of modified prompts
+        if list_of_jsons:
+            console.print(Panel("[bold]List of Modified Prompts[/bold]"))
+            console.print(Pretty(list_of_jsons))
+
+        return success, list_of_jsons, total_cost, model_name
+
+    except Exception as e:
+        console.print(f"[bold red]Unexpected Error:[/bold red] {str(e)}")
+        logger.exception("Unexpected error occurred")
+        return success, list_of_jsons, total_cost, model_name