pdd-cli 0.0.45__py3-none-any.whl → 0.0.118__py3-none-any.whl

pdd/prompts/auto_include_LLM.prompt

@@ -12,10 +12,46 @@ You are a prompt expert that helps select the necessary subset of "includes" (li
         'Step 1.' - A string of possible includes based on the input_prompt.
         'Step 2.' - A string explaining why an include might or might not be necessary for the prompt.
         'Step 3.' - A string of the minimum set of includes required to achieve the goal of the input_prompt.
-        'Step 4.' - A string of the string_of_includes based on Step 3.
+        'Step 4.' - A string of the string_of_includes based on Step 3 (see strict tag naming rules below).
     </output>
 </definitions>
 
+<tag_naming_rules_for_step_4>
+    IMPORTANT: Step 4 must emit XML snippets that wrap each <include> in a canonical dotted Python module tag.
+
+    Format:
+      <CANONICAL_MODULE_PATH><include>INCLUDE_PATH</include></CANONICAL_MODULE_PATH>
+
+    Canonical tag name rules:
+      - The wrapper tag MUST be a dotted Python import path (examples: utils.auth_helpers, utils.db_helpers, models.user).
+      - NEVER use *_example as the wrapper tag (e.g. do NOT output <auth_helpers_example>...</auth_helpers_example>).
+      - If INCLUDE_PATH is a context example (e.g. context/auth_helpers_example.py), the wrapper tag must be the real module
+        being exemplified (e.g. utils.auth_helpers). Prefer extracting the module path from the input_prompt text. If it is not
+        explicitly present, infer from filename + summary:
+          - default to utils.<base_name> for helpers/config/clients/etc
+          - default to models.<base_name> for data models
+      - If INCLUDE_PATH is not a context example, derive CANONICAL_MODULE_PATH from the file path by stripping the extension and
+        replacing '/' with '.' (e.g. core/change_handler.py -> core.change_handler).
+</tag_naming_rules_for_step_4>
+
+<common_mistakes>
+    CRITICAL: The available_includes list uses "File: path" format for INPUT only.
+    You must TRANSFORM this into proper <include> syntax for OUTPUT.
+
+    These output formats are WRONG (never output these):
+    - [File: path/to/file.py]
+    - File: path/to/file.py
+    - {{path/to/file.py}}
+    - path/to/file.py (without include tags)
+
+    This output format is CORRECT:
+    - <include>path/to/file.py</include>
+
+    Example transformation:
+    INPUT (available_includes):  File: context/auth_helpers_example.py
+    OUTPUT (Step 4):             <utils.auth_helpers><include>context/auth_helpers_example.py</include></utils.auth_helpers>
+</common_mistakes>
+
 <context>
     Here is the input_prompt to find the includes for: <input_prompt>{input_prompt}</input_prompt>
     Here is the available_includes: <available_includes>{available_includes}</available_includes>
@@ -27,8 +63,7 @@ Here are some examples of how to do this:
         <example_input_prompt>
             % You are an expert Python Software Engineer. Your goal is to write a Python function, "process_csv_change", that will read in a take in a csv file name and call change_example for each of the lines.
 
-            % The function should be part of a Python package, using relative imports (single dot) for internal modules (e.g. 'from .module_name import module_name'). All output to the console will be pretty printed using the Python Rich library. Ensure the function handles edge cases, such as missing inputs or model errors, and provide clear error messages.
-% The ./pdd/__init__.py file will have the EXTRACTION_STRENGTH, DEFAULT_STRENGTH, DEFAULT_TIME and other global constants. Example: ```from . import DEFAULT_STRENGTH```
+            <include>context/python_preamble.prompt</include>
 
             % Here are the inputs and outputs of the function:
                 Inputs: 
@@ -64,100 +99,32 @@ Here are some examples of how to do this:
                 Step 3. Return the success status, list of modified prompts, total cost, and model name.
         </example_input_prompt>
         <example_available_includes>
-            context/DSPy_example.py
-            context/anthropic_counter_example.py
-            context/autotokenizer_example.py
-            context/bug_to_unit_test_example.py
-            context/bug_to_unit_test_failure_example.py
-            context/change_example.py
-            context/cli_example.py
-            context/cli_python_preprocessed.prompt
-            context/click_example.py
-            context/cloud_function_call.py
-            context/code_generator_example.py
-            context/comment_line_example.py
-            context/conflicts_in_prompts_example.py
-            context/conflicts_in_prompts_python.prompt
-            context/construct_paths_example.py
-            context/context_generator_example.py
-            context/continue_generation_example.py
-            context/detect_change_example.py
-            context/execute_bug_to_unit_test_failure.py
-            context/final_llm_output.py
-            context/find_section_example.py
-            context/fix_code_module_errors_example.py
-            context/fix_error_loop_example.py
-            context/fix_errors_from_unit_tests_example.py
-            context/generate_output_paths_example.py
-            context/generate_test_example.py
-            context/get_comment_example.py
-            context/get_extension_example.py
-            context/get_language_example.py
-            context/git_update_example.py
-            context/langchain_lcel_example.py
-            context/llm_selector_example.py
-            context/llm_token_counter_example.py
-            context/postprocess_0_example.py
-            context/postprocess_example.py
-            context/postprocessed_runnable_llm_output.py
-            context/preprocess_example.py
-            context/process_csv_change_example.py
-            context/prompt_caching.ipynb
-            context/split_example.py
-            context/tiktoken_example.py
-            context/trace_example.py
-            context/unfinished_prompt_example.py
-            context/unrunnable_raw_llm_output.py
-            context/update_prompt_example.py
-            context/xml_tagger_example.py
+            File: utils/csv_parser.py
+            Summary: Utility functions for parsing and processing CSV files with pandas
+            File: utils/file_processor.py
+            Summary: Generic file processing utilities for reading and writing various formats
+            File: core/change_handler.py
+            Summary: Main function for applying changes to code files with validation
+            File: models/data_validator.py
+            Summary: Pydantic models and validation functions for input data
+            File: cli/command_interface.py
+            Summary: Click-based command line interface utilities
+            File: database/connection.py
+            Summary: Database connection and query utilities
+            File: api/http_client.py
+            Summary: HTTP client for making API requests with retry logic
+            File: processing/text_analyzer.py
+            Summary: Text analysis and natural language processing functions
+            File: config/settings.py
+            Summary: Configuration management and environment variable handling
+            File: logging/logger.py
+            Summary: Centralized logging configuration and utilities
         </example_available_includes>
         <example_string_of_includes>
             % Here are examples of how to use internal modules:
             <internal_example_modules>
-                % Here is an example of the change function that will be used: <change_example>import os
-from pdd.change import change
-from rich.console import Console
-
-console = Console()
-
-def main() -> None:
-    """
-    Main function to demonstrate the use of the `change` function from the `pdd.change` module.
-    Sets up environment variables, defines input parameters, and calls the `change` function.
-    """
-    # Set up the environment variable for PDD_PATH
-    # os.environ['PDD_PATH'] = '/path/to/pdd'  # Replace with actual path
-
-    # Example inputs
-    input_prompt = "Write a function to calculate the factorial of a number."
-    input_code = """
-def factorial(n):
-    if n == 0 or n == 1:
-        return 1
-    else:
-        return n * factorial(n-1)
-    """
-    change_prompt = "Modify the function to take the square root of the factorial output."
-    strength = 0.5  # Strength parameter for the LLM (0.0 to 1.0)
-    temperature = 0.0  # Temperature parameter for the LLM (0.0 to 1.0)
-
-    try:
-        # Call the change function
-        modified_prompt, total_cost, model_name = change(
-            input_prompt, input_code, change_prompt, strength, temperature
-        )
-
-        # Print the results
-        console.print(f"[bold]Modified Prompt:[/bold]\n{modified_prompt}")
-        console.print(f"[bold]Total Cost:[/bold] ${total_cost:.6f}")
-        console.print(f"[bold]Model Used:[/bold] {model_name}")
-
-    except Exception as e:
-        console.print(f"[bold red]An error occurred:[/bold red] {str(e)}")
-
-if __name__ == "__main__":
-    main()
-</change_example>
+                % Here is an example of the change function that will be used:
+                <core.change_handler><include>core/change_handler.py</include></core.change_handler>
             </internal_example_modules>
         </example_string_of_includes>
     </example_1>
@@ -166,8 +133,7 @@ if __name__ == "__main__":
         <example_input_prompt>
             % You are an expert Python Software Engineer. Your goal is to write a Python function, "generate_test", that will create a unit test from a code file.
 
-            % The function should be part of a Python package, using relative imports (single dot) for internal modules (e.g. 'from .module_name import module_name'). All output to the console will be pretty printed using the Python Rich library. Ensure the function handles edge cases, such as missing inputs or model errors, and provide clear error messages.
-% The ./pdd/__init__.py file will have the EXTRACTION_STRENGTH, DEFAULT_STRENGTH, DEFAULT_TIME and other global constants. Example: ```from . import DEFAULT_STRENGTH```
+            <include>./context/python_preamble.prompt</include>
 
             % Here are the inputs and outputs of the function:
                 Inputs: 
@@ -200,827 +166,74 @@ if __name__ == "__main__":
                 Step 7. Return the unit_test, total_cost and model_name
         </example_input_prompt>
         <example_available_includes>
-            context/DSPy_example.py
-            context/anthropic_counter_example.py
-            context/autotokenizer_example.py
-            context/bug_to_unit_test_example.py
-            context/bug_to_unit_test_failure_example.py
-            context/change_example.py
-            context/cli_example.py
-            context/cli_python_preprocessed.prompt
-            context/click_example.py
-            context/cloud_function_call.py
-            context/code_generator_example.py
-            context/comment_line_example.py
-            context/conflicts_in_prompts_example.py
-            context/conflicts_in_prompts_python.prompt
-            context/construct_paths_example.py
-            context/context_generator_example.py
-            context/continue_generation_example.py
-            context/detect_change_example.py
-            context/execute_bug_to_unit_test_failure.py
-            context/final_llm_output.py
-            context/find_section_example.py
-            context/fix_code_module_errors_example.py
-            context/fix_error_loop_example.py
-            context/fix_errors_from_unit_tests_example.py
-            context/generate_output_paths_example.py
-            context/generate_test_example.py
-            context/get_comment_example.py
-            context/get_extension_example.py
-            context/get_language_example.py
-            context/git_update_example.py
-            context/langchain_lcel_example.py
-            context/llm_selector_example.py
-            context/llm_token_counter_example.py
-            context/postprocess_0_example.py
-            context/postprocess_example.py
-            context/postprocessed_runnable_llm_output.py
-            context/preprocess_example.py
-            context/process_csv_change_example.py
-            context/prompt_caching.ipynb
-            context/split_example.py
-            context/tiktoken_example.py
-            context/trace_example.py
-            context/unfinished_prompt_example.py
-            context/unrunnable_raw_llm_output.py
-            context/update_prompt_example.py
-            context/xml_tagger_example.py
+            File: frameworks/langchain_utils.py
+            Summary: LangChain LCEL utilities and chain composition helpers
+            File: llm/model_selector.py
+            Summary: Dynamic LLM model selection based on task requirements
+            File: llm/token_counter.py
+            Summary: Token counting utilities for various LLM providers
+            File: processing/prompt_processor.py
+            Summary: Prompt preprocessing and template management functions
+            File: testing/test_generator.py
+            Summary: Automated unit test generation from code and prompts
+            File: processing/completion_detector.py
+            Summary: Functions to detect incomplete LLM outputs and continue generation
+            File: processing/output_postprocessor.py
+            Summary: Post-processing utilities for cleaning and formatting LLM outputs
+            File: utils/file_reader.py
+            Summary: Safe file reading utilities with encoding detection
+            File: config/environment.py
+            Summary: Environment variable management and project path resolution
+            File: markdown/renderer.py
+            Summary: Rich markdown rendering and formatting utilities
         </example_available_includes>
         <example_string_of_includes>
-            % Here is an example of a LangChain Expression Language (LCEL) program: <lcel_example>import os
-from langchain_core.prompts import PromptTemplate
-from langchain_community.cache import SQLiteCache
-from langchain_community.llms.mlx_pipeline import MLXPipeline
-from langchain.globals import set_llm_cache
-from langchain_core.output_parsers import JsonOutputParser, PydanticOutputParser # Parsers are only avaiable in langchain_core.output_parsers not langchain.output_parsers
-from langchain_core.output_parsers import StrOutputParser
-from langchain_core.prompts import ChatPromptTemplate
-from langchain_core.runnables import RunnablePassthrough, ConfigurableField
-
-from langchain_openai import AzureChatOpenAI
-from langchain_fireworks import Fireworks 
-from langchain_anthropic import ChatAnthropic
-from langchain_openai import ChatOpenAI # Chatbot and conversational tasks
-from langchain_openai import OpenAI # General language tasks
-from langchain_google_genai import ChatGoogleGenerativeAI
-from langchain_google_vertexai import ChatVertexAI
-from langchain_groq import ChatGroq
-from langchain_together import Together
-
-from langchain.callbacks.base import BaseCallbackHandler
-from langchain.schema import LLMResult
-
-import json
-
-from langchain_community.chat_models.mlx import ChatMLX
-from langchain_core.messages import HumanMessage
-
-from langchain_ollama.llms import OllamaLLM
-from langchain_aws import ChatBedrockConverse
-
-# Define a base output parser (e.g., PydanticOutputParser)
-from pydantic import BaseModel, Field
-
-
-
-class CompletionStatusHandler(BaseCallbackHandler):
-    def __init__(self):
-        self.is_complete = False
-        self.finish_reason = None
-        self.input_tokens = None
-        self.output_tokens = None
-
-    def on_llm_end(self, response: LLMResult, **kwargs) -> None:
-        self.is_complete = True
-        if response.generations and response.generations[0]:
-            generation = response.generations[0][0]
-            self.finish_reason = generation.generation_info.get('finish_reason').lower()
-            
-            # Extract token usage
-            if hasattr(generation.message, 'usage_metadata'):
-                usage_metadata = generation.message.usage_metadata
-                self.input_tokens = usage_metadata.get('input_tokens')
-                self.output_tokens = usage_metadata.get('output_tokens')
-        # print("response:",response)
-        print("Extracted information:")
-        print(f"Finish reason: {self.finish_reason}")
-        print(f"Input tokens: {self.input_tokens}")
-        print(f"Output tokens: {self.output_tokens}")
-
-# Set up the LLM with the custom handler
-handler = CompletionStatusHandler()
-# Always setup cache to save money and increase speeds
-set_llm_cache(SQLiteCache(database_path=".langchain.db"))
-
-
-# Create the LCEL template. Make note of the variable {topic} which will be filled in later.
-prompt_template = PromptTemplate.from_template("Tell me a joke about {topic}")
-
-llm = ChatGoogleGenerativeAI(model="gemini-2.5-pro-exp-03-25", temperature=0, callbacks=[handler])
-# Combine with a model and parser to output a string
-chain = prompt_template |llm| StrOutputParser()
-
-# Run the template. Notice that the input is a dictionary with a single key "topic" which feeds it into the above prompt template. This is needed because the prompt template has a variable {topic} which needs to be filled in when invoked.
-result = chain.invoke({"topic": "cats"})
-print("********Google:", result)
-
-
-llm = ChatVertexAI(model="gemini-2.5-pro-exp-03-25", temperature=0, callbacks=[handler])
-# Combine with a model and parser to output a string
-chain = prompt_template |llm| StrOutputParser()
-
-# Run the template. Notice that the input is a dictionary with a single key "topic" which feeds it into the above prompt template. This is needed because the prompt template has a variable {topic} which needs to be filled in when invoked.
-result = chain.invoke({"topic": "cats"})
-print("********GoogleVertex:", result)
-
-
-# Define your desired data structure.
-class Joke(BaseModel):
-    setup: str = Field(description="question to set up a joke")
-    punchline: str = Field(description="answer to resolve the joke")
-
-
-# Set up a parser
-parser = JsonOutputParser(pydantic_object=Joke)
-
-# Create a prompt template
-prompt = PromptTemplate(
-    template="Answer the user query.\n{format_instructions}\n{query}\n",
-    input_variables=["query"],
-    partial_variables={"format_instructions": parser.get_format_instructions()},
-)
-
-llm_no_struct = ChatOpenAI(model="gpt-4o-mini", temperature=0, 
-                           callbacks=[handler]) 
-llm = llm_no_struct.with_structured_output(Joke) # with structured output forces the output to be a specific object, in this case Joke. Only OpenAI models have structured output
-# Chain the components. 
-#  The class `LLMChain` was deprecated in LangChain 0.1.17 and will be removed in 1.0. Use RunnableSequence, e.g., `prompt | llm` instead.
-chain = prompt | llm 
-
-# Invoke the chain with a query. 
-# IMPORTANT: chain.run is now obsolete. Use chain.invoke instead.
-result = chain.invoke({"query": "Tell me a joke about openai."})
-print("4o mini JSON: ",result)
-print(result.setup) # How to access the structured output
-
-llm = ChatOpenAI(model="o1", temperature=1, 
-                           callbacks=[handler],model_kwargs = {"max_completion_tokens" : 1000})
-# Chain the components. 
-#  The class `LLMChain` was deprecated in LangChain 0.1.17 and will be removed in 1.0. Use RunnableSequence, e.g., `prompt | llm` instead.
-chain = prompt | llm | parser
-
-# Invoke the chain with a query. 
-# IMPORTANT: chain.run is now obsolete. Use chain.invoke instead.
-result = chain.invoke({"query": "Tell me a joke about openai."})
-print("o1 JSON: ",result)
-
-# Get DEEPSEEK_API_KEY environmental variable
-
-deepseek_api_key = os.getenv('DEEPSEEK_API_KEY')
-
-# Ensure the API key is retrieved successfully
-if deepseek_api_key is None:
-    raise ValueError("DEEPSEEK_API_KEY environment variable is not set")
-
-llm = ChatOpenAI(
-    model='deepseek-chat', 
-    openai_api_key=deepseek_api_key, 
-    openai_api_base='https://api.deepseek.com',
-    temperature=0, callbacks=[handler]
-)
-
-# Chain the components
-chain = prompt | llm | parser
-
-# Invoke the chain with a query
-result = chain.invoke({"query": "Write joke about deepseek."})
-print("deepseek",result)
-
-
-# Set up a parser
-parser = PydanticOutputParser(pydantic_object=Joke)
-# Chain the components
-chain = prompt | llm | parser
-
-# Invoke the chain with a query
-result = chain.invoke({"query": "Write joke about deepseek and pydantic."})
-print("deepseek pydantic",result)
-
-# Set up the Azure ChatOpenAI LLM instance
-llm_no_struct = AzureChatOpenAI(
-    model="o4-mini",
-    temperature=1,
-    callbacks=[handler]
-)
-llm = llm_no_struct.with_structured_output(Joke) # with structured output forces the output to be a specific JSON format
-# Chain the components: prompt | llm | parser
-chain = prompt | llm # returns a Joke object
-
-# Invoke the chain with a query
-result = chain.invoke({"query": "What is Azure?"})  # Pass a dictionary if `invoke` expects it
-print("Azure Result:", result)
-
-# Set up a parser
-parser = JsonOutputParser(pydantic_object=Joke)
-
-llm = Fireworks(
-    model="accounts/fireworks/models/llama4-maverick-instruct-basic",
-    temperature=0, callbacks=[handler])
-# Chain the components
-chain = prompt | llm | parser
-
-# Invoke the chain with a query
-# no money in account
-# result = chain.invoke({"query": "Tell me a joke about the president"})
-# print("fireworks",result)
-
-
-
-
-
-prompt = ChatPromptTemplate.from_template(
-    "Tell me a short joke about {topic}"
-)
-chat_openai = ChatOpenAI(model="gpt-3.5-turbo", callbacks=[handler])
-openai = OpenAI(model="gpt-3.5-turbo-instruct", callbacks=[handler])
-anthropic = ChatAnthropic(model="claude-2", callbacks=[handler])
-model = (
-    chat_openai
-    .with_fallbacks([anthropic])
-    .configurable_alternatives(
-        ConfigurableField(id="model"),
-        default_key="chat_openai",
-        openai=openai,
-        anthropic=anthropic,
-    )
-)
-
-chain = (
-    {"topic": RunnablePassthrough()} 
-    | prompt 
-    | model 
-    | StrOutputParser()
-)
-result = chain.invoke({"topic": "Tell me a joke about the president"})
-print("config alt:",result)
-
-
-
-llm = ChatAnthropic(
-    model="claude-3-7-sonnet-latest",
-    max_tokens=5000,  # Total tokens for the response
-    thinking={"type": "enabled", "budget_tokens": 2000},  # Tokens for internal reasoning
-)
-
-response = llm.invoke("What is the cube root of 50.653?")
-print(json.dumps(response.content, indent=2))
-
-
-llm = ChatGroq(temperature=0, model_name="qwen-qwq-32b", callbacks=[handler])
-system = "You are a helpful assistant."
-human = "{text}"
-prompt = ChatPromptTemplate.from_messages([("system", system), ("human", human)])
-
-chain = prompt | llm | StrOutputParser()
-print(chain.invoke({"text": "Explain the importance of low latency LLMs."}))
-
-
-llm = Together(
-    model="meta-llama/Llama-3-70b-chat-hf",
-    max_tokens=500, callbacks=[handler]
-)
-chain = prompt | llm | StrOutputParser()
-print(chain.invoke({"text": "Explain the importance of together.ai."}))
-
-
-# Define a prompt template with placeholders for variables
-prompt_template = PromptTemplate.from_template("Tell me a {adjective} joke about {content}.")
-
-# Format the prompt with the variables
-formatted_prompt = prompt_template.format(adjective="funny", content="data scientists")
-
-# Print the formatted prompt
-print(formatted_prompt)
-
-
-# Set up the LLM with the custom handler
-handler = CompletionStatusHandler()
-
-
-llm = ChatOpenAI(model="gpt-3.5-turbo", temperature=0.9, callbacks=[handler])
-
-prompt = PromptTemplate.from_template("What is a good name for a company that makes {product}?")
-
-chain = prompt | llm
-
-# Invoke the chain
-response = chain.invoke({"product":"colorful socks"})
-
-# Check completion status
-print(f"Is complete: {handler.is_complete}")
-print(f"Finish reason: {handler.finish_reason}")
-print(f"Response: {response}")
-print(f"Input tokens: {handler.input_tokens}")
-print(f"Output tokens: {handler.output_tokens}")
-
-
-
-template = """Question: {question}"""
-
-prompt = ChatPromptTemplate.from_template(template)
-
-model = OllamaLLM(model="qwen2.5-coder:32b")
-
-chain = prompt | model
-
-output = chain.invoke({"question": "Write a python function that calculates Pi"})
-print(output)
-
-
-
-llm = MLXPipeline.from_model_id(
-    "mlx-community/quantized-gemma-2b-it",
-    pipeline_kwargs={"max_tokens": 10, "temp": 0.1},
-)
-
-
-chat_model = ChatMLX(llm=llm)
-messages = [HumanMessage(content="What happens when an unstoppable force meets an immovable object?")]
-response = chat_model.invoke(messages)
-print(response.content)
-
-
-
-llm = ChatBedrockConverse(
-    model_id="anthropic.claude-3-5-sonnet-20240620-v1:0",
-    # Additional parameters like temperature, max_tokens can be set here
-)
-
-messages = [HumanMessage(content="What happens when an unstoppable force meets an immovable sonnet?")]
-response = llm.invoke(messages)
-print(response.content)</lcel_example>
+            % Here is an example of a LangChain Expression Language (LCEL) program:
+            <frameworks.langchain_utils><include>frameworks/langchain_utils.py</include></frameworks.langchain_utils>
 
             % Here are examples of how to use internal modules:
             <internal_example_modules>
-                % Here is an example how to preprocess the prompt from a file: <preprocess_example>from pdd.preprocess import preprocess
-from rich.console import Console   
-console = Console()     
-
-prompt = """
-<prompt>
-    Hello World
-
-    <pdd>This is a comment</pdd>
-    [About](https://about.google/?fg=1&utm_source=google-US&utm_medium=referral&utm_campaign=hp-header) [Store](https://store.google.com/US?utm_source=hp_header&utm_medium=google_ooo&utm_campaign=GS100042&hl=en-US)
-
-AI Mode
-
-Choose what you’re giving feedback on
-
-* * *
-
-See more
-
-Delete
-
-Delete
-
-Report inappropriate predictions
-
-I'm Feeling Curious
-
-I'm Feeling Hungry
-
-I'm Feeling Adventurous
-
-I'm Feeling Playful
-
-I'm Feeling Stellar
-
-I'm Feeling Doodley
-
-I'm Feeling Trendy
-
-I'm Feeling Artistic
-
-I'm Feeling Funny
-
-[Advertising](https://www.google.com/intl/en_us/ads/?subid=ww-ww-et-g-awa-a-g_hpafoot1_1!o2&utm_source=google.com&utm_medium=referral&utm_campaign=google_hpafooter&fg=1) [Business](https://www.google.com/services/?subid=ww-ww-et-g-awa-a-g_hpbfoot1_1!o2&utm_source=google.com&utm_medium=referral&utm_campaign=google_hpbfooter&fg=1) [How Search works](https://google.com/search/howsearchworks/?fg=1)
-
-[![](<Base64-Image-Removed>)Applying AI towards science and the environment](https://ai.google/societal-impact/?utm_source=googlehpfooter&utm_medium=housepromos&utm_campaign=bottom-footer)
-
-[Privacy](https://policies.google.com/privacy?hl=en&fg=1) [Terms](https://policies.google.com/terms?hl=en&fg=1)
-
-Settings
-
-[Search settings](https://www.google.com/preferences?hl=en&fg=1)
-
-[Advanced search](https://www.google.com/advanced_search?hl=en&fg=1)
-
-[Your data in Search](https://www.google.com/history/privacyadvisor/search/unauth?utm_source=googlemenu&fg=1&cctld=com)
-
-[Search history](https://www.google.com/history/optout?hl=en&fg=1)
-
-[Search help](https://support.google.com/websearch/?p=ws_results_help&hl=en&fg=1)
-
-Send feedback
-
-Dark theme: Off
-
-Google apps
-
-![](<Base64-Image-Removed>)
-
-Sign in to GoogleGet the most from your Google account
-
-Stay signed out
-
-Sign in
-    {test}
-    {test2}
-    ```<TODO.md>```
-
-    <pdd>
-        multi-line
-        comment should not show up
-    </pdd>
-</prompt>
-"""
-
-recursive = False
-double_curly_brackets = True
-exclude_keys = ["test2"] # exclude test2 from being doubled
-
-# Debug info
-console.print(f"[bold yellow]Debug: exclude_keys = {exclude_keys}[/bold yellow]")
-
-processed = preprocess(prompt, recursive, double_curly_brackets, exclude_keys=exclude_keys)
-console.print("[bold white]Processed Prompt:[/bold white]")
-console.print(processed)
-</preprocess_example>
-
-                % Example of selecting a Langchain LLM and counting tokens using llm_selector: <llm_selector_example>from pdd.llm_selector import llm_selector
-
-def main() -> None:
-    """
-    Main function to demonstrate the usage of the llm_selector function.
-    """
-    # Define the strength and temperature parameters
-    strength: float = 0.5  # Example strength value for the LLM model
-    temperature: float = 1.0  # Example temperature value for the LLM model
-
-    try:       
-        while strength <= 1.1: 
-            # Call the llm_selector function with the specified strength and temperature
-            llm, token_counter, input_cost, output_cost, model_name = llm_selector(strength, temperature)
-            print(f"Strength: {strength}")
-            
-            # Print the details of the selected LLM model
-            print(f"Selected LLM Model: {model_name}")
-            print(f"Input Cost per Million Tokens: {input_cost}")
-            print(f"Output Cost per Million Tokens: {output_cost}")
-
-            # Example usage of the token counter function
-            sample_text: str = "This is a sample text to count tokens."
-            token_count: int = token_counter(sample_text)
-            print(f"Token Count for Sample Text: {token_count}")
-            print(f"model_name: {model_name}")
-            strength += 0.05
-    except FileNotFoundError as e:
-        print(f"Error: {e}")
-    except ValueError as e:
-        print(f"Error: {e}")
+                % Here is an example how to preprocess the prompt from a file:
+                <processing.prompt_processor><include>processing/prompt_processor.py</include></processing.prompt_processor>
 
-if __name__ == "__main__":
-    main()</llm_selector_example>
+                % Example of selecting a Langchain LLM and counting tokens using llm_selector:
+                <llm.model_selector><include>llm/model_selector.py</include></llm.model_selector>
 
-                % Example usage of the unfinished_prompt function: <unfinished_prompt_example>from pdd.unfinished_prompt import unfinished_prompt
-from rich import print as rprint
+                % Example usage of the unfinished_prompt function:
+                <processing.completion_detector><include>processing/completion_detector.py</include></processing.completion_detector>
 
-# This script provides a concise example of how to use the `unfinished_prompt` function
-# from the `pdd.unfinished_prompt` module.
+                % Here is an example how to continue the generation of a model output:
+                <processing.completion_detector><include>processing/completion_detector.py</include></processing.completion_detector>
 
-# --- Pre-requisites for running this example: ---
-# 1. The `pdd` Python package must be accessible. This means:
-#    - It's installed in your Python environment (e.g., via pip if it's a package), OR
-#    - The directory containing the `pdd` package is added to your PYTHONPATH.
-#      For instance, if your project structure is:
-#      my_project/
-#      ├── pdd/  # The module's package
-#      │   ├── __init__.py
-#      │   ├── unfinished_prompt.py
-#      │   ├── load_prompt_template.py
-#      │   └── llm_invoke.py
-#      └── examples/
-#          └── run_this_example.py (this file)
-#      You would typically run this script from the `my_project` directory
-#      (e.g., `python examples/run_this_example.py`) after ensuring `my_project`
-#      is in PYTHONPATH (e.g., `export PYTHONPATH=$PYTHONPATH:/path/to/my_project`).
-#
-# 2. The `pdd` package requires internal setup for its dependencies:
-#    - A prompt template file named "unfinished_prompt_LLM" (e.g., "unfinished_prompt_LLM.txt")
-#      must be present where `pdd.load_prompt_template` (used internally by `unfinished_prompt`)
-#      can find it. This location is usually relative to the `pdd` package structure.
-#    - The `pdd.llm_invoke` function (used internally) must be configured for access to an LLM.
-#      This typically involves setting environment variables for API keys (e.g., `OPENAI_API_KEY`).
-#
-# This script should be saved outside the `pdd` package, for instance, in an
-# `examples/` directory as shown above.
-# To run: `python name_of_this_script.py` (adjust path as needed).
-
-# --- Example Usage ---
-
-# 1. Define the prompt text you want to analyze.
-#    This example uses a prompt that is intentionally incomplete to demonstrate
-#    the function's ability to detect incompleteness.
-my_prompt_text = "Write a comprehensive guide on how to bake a sourdough bread, starting from creating a starter, then the kneading process, and finally"
-
-rprint(f"[bold cyan]Analyzing prompt:[/bold cyan] \"{my_prompt_text}\"")
-
-# 2. Call the `unfinished_prompt` function.
-#    Review the function's docstring for detailed parameter information.
-#    - `prompt_text` (str): The text of the prompt to analyze.
-#    - `strength` (float, optional, 0.0-1.0, default=0.5): Influences the LLM's behavior or model choice.
-#    - `temperature` (float, optional, 0.0-1.0, default=0.0): Controls the randomness of the LLM's output.
-#    - `verbose` (bool, optional, default=False): If True, the function will print detailed internal logs.
-#
-#    The function returns a tuple: (reasoning, is_finished, total_cost, model_name)
-#    - `reasoning` (str): The LLM's structured explanation for its completeness assessment.
-#    - `is_finished` (bool): True if the prompt is considered complete, False otherwise.
-#    - `total_cost` (float): The estimated cost of the LLM call. The unit (e.g., USD) depends on the LLM provider.
-#    - `model_name` (str): The name of the LLM model that was used for the analysis.
-
-# Example call with verbose output and custom strength/temperature settings.
-reasoning_str, is_complete_flag, call_cost, llm_model = unfinished_prompt(
-    prompt_text=my_prompt_text,
-    strength=0.6,       # Example: using a specific strength value
-    temperature=0.1,    # Example: using a low temperature for more deterministic reasoning
-    verbose=True        # Set to True to see detailed logs from within the unfinished_prompt function
-)
-
-# 3. Print the results returned by the function.
-rprint("\n[bold green]--- Analysis Results ---[/bold green]")
-rprint(f"  [bold]Prompt Analyzed:[/bold] \"{my_prompt_text}\"")
-rprint(f"  [bold]Is prompt complete?:[/bold] {'Yes, the LLM considers the prompt complete.' if is_complete_flag else 'No, the LLM suggests the prompt needs continuation.'}")
-rprint(f"  [bold]LLM's Reasoning:[/bold]\n    {reasoning_str}") # Rich print will handle newlines in the reasoning string
-rprint(f"  [bold]Cost of Analysis:[/bold] ${call_cost:.6f}") # Display cost, assuming USD. Adjust currency/format as needed.
-rprint(f"  [bold]LLM Model Used:[/bold] {llm_model}")
-
-# --- Example of calling with default parameters ---
-# If you want to use the default strength (0.5), temperature (0.0), and verbose (False):
-#
-# default_prompt_text = "What is the capital of Canada?"
-# rprint(f"\n[bold cyan]Analyzing prompt with default settings:[/bold cyan] \"{default_prompt_text}\"")
-#
-# reasoning_def, is_finished_def, cost_def, model_def = unfinished_prompt(
-#     prompt_text=default_prompt_text
-# )
-#
-# rprint("\n[bold green]--- Default Call Analysis Results ---[/bold green]")
-# rprint(f"  [bold]Prompt Analyzed:[/bold] \"{default_prompt_text}\"")
-# rprint(f"  [bold]Is prompt complete?:[/bold] {'Yes' if is_finished_def else 'No'}")
-# rprint(f"  [bold]LLM's Reasoning:[/bold]\n    {reasoning_def}")
-# rprint(f"  [bold]Cost of Analysis:[/bold] ${cost_def:.6f}")
-# rprint(f"  [bold]LLM Model Used:[/bold] {model_def}")
-</unfinished_prompt_example>
-
-                % Here is an example how to continue the generation of a model output: <continue_generation_example>from pdd.continue_generation import continue_generation
-
-def main() -> None:
-    """
-    Main function to demonstrate the usage of the continue_generation function.
-    It continues the generation of text using a language model and calculates the cost.
-    """
-    # Define the input parameters for the continue_generation function
-    # formatted_input_prompt: str = "Once upon a time in a land far away, there was a"
-    # load context/cli_python_preprocessed.prompt into formatted_input_prompt
-    with open("context/cli_python_preprocessed.prompt", "r") as file:
-        formatted_input_prompt = file.read()
-    
-    # llm_output: str = ""  # Initial LLM output is empty
-    # load context/unfinished_prompt.txt into llm_output
-    with open("context/llm_output_fragment.txt", "r") as file:
-        llm_output = file.read()
-    strength: float = .915  # Strength parameter for the LLM model
-    temperature: float = 0  # Temperature parameter for the LLM model
-
-    try:
-        # Call the continue_generation function
-        final_llm_output, total_cost, model_name = continue_generation(
-            formatted_input_prompt=formatted_input_prompt,
-            llm_output=llm_output,
-            strength=strength,
-            temperature=temperature,
-            verbose=True
-        )
-
-        # Output the results
-        # print(f"Final LLM Output: {final_llm_output}")
-        print(f"Total Cost: ${total_cost:.6f}")
-        print(f"Model Name: {model_name}")
-        # write final_llm_output to context/final_llm_output.txt
-        with open("context/final_llm_output.py", "w") as file:
-            file.write(final_llm_output)
-
-    except FileNotFoundError as e:
-        print(f"Error: {e}")
-    except Exception as e:
-        print(f"An error occurred: {e}")
-
-if __name__ == "__main__":
-    main()</continue_generation_example>
-
-                % Here is an example how to postprocess the model output result: <postprocess_example>"""
-Example demonstrating the usage of the `postprocess` function 
-from the `pdd.postprocess` module.
-
-This example showcases two scenarios for extracting code from an LLM's text output:
-1. Simple code extraction (strength = 0): Uses basic string manipulation to find code
-   blocks enclosed in triple backticks. This method is fast and has no cost.
-2. Advanced code extraction (strength > 0): Leverages an LLM for more robust extraction.
-   This method is more powerful but incurs a cost and takes more time.
-
-To run this example:
-1. Ensure the `pdd` package (containing the `postprocess` module) is in your PYTHONPATH
-   or installed in your environment.
-2. Ensure the `rich` library is installed (`pip install rich`).
-3. This script uses `unittest.mock` (part of Python's standard library) to simulate
-   the behavior of internal dependencies (`load_prompt_template` and `llm_invoke`)
-   for the LLM-based extraction scenario. This allows the example to run without
-   requiring actual LLM API calls or specific prompt files.
-"""
-from rich import print
-from unittest.mock import patch, MagicMock
-
-# Assuming 'pdd' package is in PYTHONPATH or installed.
-# The 'postprocess' module is expected to be at pdd/postprocess.py
-from pdd.postprocess import postprocess, ExtractedCode # ExtractedCode is needed for the mock
-
-def main():
-    """
-    Runs the demonstration for the postprocess function.
-    """
-    print("[bold underline blue]Demonstrating `postprocess` function from `pdd.postprocess`[/bold underline blue]\n")
-
-    # --- Common Inputs ---
-    # This is a sample string that might be output by an LLM, containing text and code.
-    llm_output_text_with_code = """
-This is some text from an LLM.
-It includes a Python code block:
-```python
-def greet(name):
-    # A simple greeting function
-    print(f"Hello, {name}!")
-
-greet("Developer")
-```
-And some more text after the code block.
-There might be other language blocks too:
-```javascript
-console.log("This is JavaScript");
-```
-But we are only interested in Python.
-"""
-    # The target programming language for extraction.
-    target_language = "python"
-
-    # --- Scenario 1: Simple Extraction (strength = 0) ---
-    # This mode uses the `postprocess_0` internal function, which performs a basic
-    # extraction of content between triple backticks. It does not use an LLM.
-    print("[bold cyan]Scenario 1: Simple Extraction (strength = 0)[/bold cyan]")
-    print("Demonstrates extracting code using basic string processing.")
-    print(f"  Input LLM Output: (see below)")
-    # print(f"[dim]{llm_output_text_with_code}[/dim]") # Printing for brevity in console
-    print(f"  Target Language: '{target_language}' (Note: simple extraction is language-agnostic but extracts first block)")
-    print(f"  Strength: 0 (activates simple, non-LLM extraction)")
-    print(f"  Verbose: True (enables detailed console output from `postprocess`)\n")
-
-    # Call postprocess with strength = 0
-    # Input parameters:
-    #   llm_output (str): The LLM's raw output string.
-    #   language (str): The programming language to extract (less critical for strength=0).
-    #   strength (float): 0-1, model strength. 0 means simple extraction.
-    #   temperature (float): 0-1, LLM temperature (not used for strength=0).
-    #   time (float): 0-1, LLM thinking effort (not used for strength=0).
-    #   verbose (bool): If True, prints internal processing steps.
-    extracted_code_s0, cost_s0, model_s0 = postprocess(
-        llm_output=llm_output_text_with_code,
-        language=target_language,
-        strength=0,
-        verbose=True
-    )
-
-    print("[bold green]Output for Scenario 1:[/bold green]")
-    # Output tuple:
-    #   extracted_code (str): The extracted code.
-    #   total_cost (float): Cost of the operation (in dollars). Expected to be 0.0 for simple extraction.
-    #   model_name (str): Identifier for the method/model used. Expected to be 'simple_extraction'.
-    print(f"  Extracted Code:\n[yellow]{extracted_code_s0}[/yellow]")
-    print(f"  Total Cost: ${cost_s0:.6f}")
-    print(f"  Model Name: '{model_s0}'")
-    print("-" * 60)
-
-    # --- Scenario 2: LLM-based Extraction (strength > 0) ---
-    # This mode uses an LLM via `llm_invoke` to perform a more sophisticated extraction.
-    # It requires a prompt template (`extract_code_LLM.prompt`).
-    # For this example, `load_prompt_template` and `llm_invoke` are mocked.
-    print("\n[bold cyan]Scenario 2: LLM-based Extraction (strength = 0.9)[/bold cyan]")
-    print("Demonstrates extracting code using an LLM (mocked).")
-    print(f"  Input LLM Output: (same as above)")
-    print(f"  Target Language: '{target_language}'")
-    print(f"  Strength: 0.9 (activates LLM-based extraction)")
-    print(f"  Temperature: 0.0 (LLM creativity, 0-1 scale)")
-    print(f"  Time: 0.5 (LLM thinking effort, 0-1 scale, influences model choice/cost)")
-    print(f"  Verbose: True\n")
-
-    # Mock for `load_prompt_template`:
-    # This function is expected to load a prompt template file (e.g., 'extract_code_LLM.prompt').
-    # In a real scenario, this file would exist in a 'prompts' directory.
-    mock_load_template = MagicMock(return_value="Mocked Prompt: Extract {{language}} code from: {{llm_output}}")
-
-    # Mock for `llm_invoke`:
-    # This function handles the actual LLM API call.
-    # It's expected to return a dictionary containing the LLM's result (parsed into
-    # an `ExtractedCode` Pydantic model), the cost, and the model name.
-    # The `extracted_code` from the LLM mock should include backticks and language identifier
-    # to test the cleaning step within the `postprocess` function.
-    mock_llm_response_code_from_llm = """```python
-def sophisticated_extraction(data):
-    # This code is supposedly extracted by an LLM
-    processed_data = data.upper()  # Example processing
-    return processed_data
-
-result = sophisticated_extraction("test data from llm")
-print(result)
-```"""
-    mock_extracted_code_pydantic_obj = ExtractedCode(extracted_code=mock_llm_response_code_from_llm)
-    mock_llm_invoke_return_value = {
-        'result': mock_extracted_code_pydantic_obj,
-        'cost': 0.00025,  # Example cost in dollars
-        'model_name': 'mock-llm-extractor-v1'
-    }
-    mock_llm_invoke_function = MagicMock(return_value=mock_llm_invoke_return_value)
-
-    # Patch the internal dependencies within the 'pdd.postprocess' module's namespace.
-    # This ensures that when `postprocess` calls `load_prompt_template` or `llm_invoke`,
-    # our mocks are used instead of the real implementations.
-    with patch('pdd.postprocess.load_prompt_template', mock_load_template):
-        with patch('pdd.postprocess.llm_invoke', mock_llm_invoke_function):
-            extracted_code_llm, cost_llm, model_llm = postprocess(
-                llm_output=llm_output_text_with_code,
-                language=target_language,
-                strength=0.9,
-                temperature=0.0,
-                time=0.5,
-                verbose=True
-            )
-
-            print("[bold green]Output for Scenario 2:[/bold green]")
-            print(f"  Extracted Code:\n[yellow]{extracted_code_llm}[/yellow]")
-            print(f"  Total Cost: ${cost_llm:.6f} (cost is in dollars)")
-            print(f"  Model Name: '{model_llm}'")
-
-            # --- Verification of Mock Calls (for developer understanding) ---
-            # Check that `load_prompt_template` was called correctly.
-            mock_load_template.assert_called_once_with("extract_code_LLM")
-
-            # Check that `llm_invoke` was called correctly.
-            mock_llm_invoke_function.assert_called_once()
-            # Inspect the arguments passed to the mocked llm_invoke
-            call_args_to_llm_invoke = mock_llm_invoke_function.call_args[1] # kwargs
-            assert call_args_to_llm_invoke['prompt'] == mock_load_template.return_value
-            assert call_args_to_llm_invoke['input_json'] == {
-                "llm_output": llm_output_text_with_code,
-                "language": target_language
-            }
-            assert call_args_to_llm_invoke['strength'] == 0.9
-            assert call_args_to_llm_invoke['temperature'] == 0.0
-            assert call_args_to_llm_invoke['time'] == 0.5
-            assert call_args_to_llm_invoke['verbose'] is True
-            assert call_args_to_llm_invoke['output_pydantic'] == ExtractedCode
-            print("[dim]  (Mocked LLM calls verified successfully)[/dim]")
-
-    print("\n[bold underline blue]Demonstration finished.[/bold underline blue]")
-    print("\n[italic]Important Notes:[/italic]")
-    print("  - For Scenario 2 (LLM-based extraction), `load_prompt_template` and `llm_invoke` were mocked.")
-    print("    In a real-world scenario:")
-    print("    - `load_prompt_template('extract_code_LLM')` would attempt to load a file named ")
-    print("      `extract_code_LLM.prompt` (typically from a 'prompts' directory configured within the `pdd` package).")
-    print("    - `llm_invoke` would make an actual API call to a Large Language Model, which requires")
-    print("      API keys and network access.")
-    print("  - The `time` parameter (0-1) for `postprocess` (and `llm_invoke`) generally controls the")
-    print("    'thinking effort' or computational resources allocated to the LLM, potentially affecting")
-    print("    which underlying LLM model is chosen and the quality/cost of the result.")
-    print("  - No actual files (like prompt files or output files) are created or read by this example script,")
-    print("    particularly in the './output' directory, due to the use of mocks for file-dependent operations.")
-
-if __name__ == "__main__":
-    main()
-</postprocess_example>
+                % Here is an example how to postprocess the model output result:
+                <processing.output_postprocessor><include>processing/output_postprocessor.py</include></processing.output_postprocessor>
             </internal_example_modules>
         </example_string_of_includes>
     </example_2>
+
+    <example_3>
+        <example_input_prompt>
+            % You are an expert Python engineer. Build an admin-only HTTP endpoint.
+            % The function must be decorated with @require_admin from utils.auth_helpers.
+            % Use utils.db_helpers to query Firestore and utils.error_handling for structured errors.
+        </example_input_prompt>
+        <example_available_includes>
+            File: context/auth_helpers_example.py
+            Summary: Example usage of authentication/authorization helpers such as require_admin
+            File: context/db_helpers_example.py
+            Summary: Example usage of Firestore database helper utilities for queries and pagination
+            File: context/error_handling_example.py
+            Summary: Example usage of structured error handling and custom exceptions
+        </example_available_includes>
+        <example_string_of_includes>
+            % Here are examples of how to use internal modules:
+            <internal_example_modules>
+                <utils.auth_helpers><include>context/auth_helpers_example.py</include></utils.auth_helpers>
+                <utils.db_helpers><include>context/db_helpers_example.py</include></utils.db_helpers>
+                <utils.error_handling><include>context/error_handling_example.py</include></utils.error_handling>
+            </internal_example_modules>
+        </example_string_of_includes>
+    </example_3>
 </examples>
 
 <instructions>
@@ -1028,5 +241,9 @@ if __name__ == "__main__":
         Step 1. Select possible includes from the available_includes based on the input_prompt.
         Step 2. Explain why an include might or might not be necessary for the prompt.
         Step 3. Determine the minimum set of includes required to achieve the goal of the input_prompt.
-        Step 4. Generate the string_of_includes based on Step 3.
+        Step 4. Generate the string_of_includes based on Step 3, following <tag_naming_rules_for_step_4>.
+
+    IMPORTANT for Step 4: Transform each file path from available_includes (format: "File: PATH")
+    into proper include syntax: <CANONICAL_MODULE><include>PATH</include></CANONICAL_MODULE>
+    Do NOT echo the "File:" format - always use <include>...</include> tags.
 </instructions>