pdd-cli 0.0.45__py3-none-any.whl → 0.0.118__py3-none-any.whl

pdd/preprocess.py

@@ -1,37 +1,158 @@
 import os
 import re
+import base64
 import subprocess
-from typing import List, Optional
+from typing import List, Optional, Tuple
 import traceback
+from pathlib import Path
 from rich.console import Console
 from rich.panel import Panel
 from rich.markup import escape
 from rich.traceback import install
+from pdd.path_resolution import get_default_resolver
 
 install()
 console = Console()
 
+# Debug/Instrumentation controls
+_DEBUG_PREPROCESS = str(os.getenv("PDD_PREPROCESS_DEBUG", "")).lower() in ("1", "true", "yes", "on")
+_DEBUG_OUTPUT_FILE = os.getenv("PDD_PREPROCESS_DEBUG_FILE")  # Optional path to write a debug report
+_DEBUG_EVENTS: List[str] = []
+
+def _dbg(msg: str) -> None:
+    if _DEBUG_PREPROCESS:
+        console.print(f"[dim][PPD][preprocess][/dim] {escape(msg)}")
+        _DEBUG_EVENTS.append(msg)
+
+def _write_debug_report() -> None:
+    if _DEBUG_PREPROCESS and _DEBUG_OUTPUT_FILE:
+        try:
+            with open(_DEBUG_OUTPUT_FILE, "w", encoding="utf-8") as fh:
+                fh.write("Preprocess Debug Report\n\n")
+                for line in _DEBUG_EVENTS:
+                    fh.write(line + "\n")
+            console.print(f"[green]Debug report written to:[/green] {_DEBUG_OUTPUT_FILE}")
+        except Exception as e:
+            # Report the error so users know why the log file wasn't written
+            console.print(f"[yellow]Warning: Could not write debug report to {_DEBUG_OUTPUT_FILE}: {e}[/yellow]")
+    elif _DEBUG_PREPROCESS and not _DEBUG_OUTPUT_FILE:
+        console.print("[dim]Debug mode enabled but PDD_PREPROCESS_DEBUG_FILE not set (output shown in console only)[/dim]")
+
+def _extract_fence_spans(text: str) -> List[Tuple[int, int]]:
+    """Return list of (start, end) spans for fenced code blocks (``` or ~~~).
+
+    The spans are [start, end) indices in the original text.
+    """
+    spans: List[Tuple[int, int]] = []
+    try:
+        fence_re = re.compile(
+            r"(?m)^[ \t]*([`~]{3,})[^\n]*\n[\s\S]*?\n[ \t]*\1[ \t]*(?:\n|$)"
+        )
+        for m in fence_re.finditer(text):
+            spans.append((m.start(), m.end()))
+    except Exception:
+        pass
+    return spans
+
+
+def _extract_inline_code_spans(text: str) -> List[Tuple[int, int]]:
+    """Return list of (start, end) spans for inline code (backticks)."""
+    spans: List[Tuple[int, int]] = []
+    try:
+        for m in re.finditer(r"(?<!`)(`+)([^\n]*?)\1", text):
+            spans.append((m.start(), m.end()))
+    except Exception:
+        pass
+    return spans
+
+
+def _extract_code_spans(text: str) -> List[Tuple[int, int]]:
+    spans = _extract_fence_spans(text)
+    spans.extend(_extract_inline_code_spans(text))
+    return sorted(spans, key=lambda s: s[0])
+
+def _is_inside_any_span(idx: int, spans: List[Tuple[int, int]]) -> bool:
+    for s, e in spans:
+        if s <= idx < e:
+            return True
+    return False
+
+
+def _intersects_any_span(start: int, end: int, spans: List[Tuple[int, int]]) -> bool:
+    for s, e in spans:
+        if start < e and end > s:
+            return True
+    return False
+
+def _scan_risky_placeholders(text: str) -> Tuple[List[Tuple[int, str]], List[Tuple[int, str]]]:
+    """Scan for risky placeholders outside code fences.
+
+    Returns two lists of (line_no, snippet):
+      - single_brace: matches like {name} not doubled and not part of {{...}}
+      - template_brace: `${...}` occurrences (which include single { ... })
+    """
+    single_brace: List[Tuple[int, str]] = []
+    template_brace: List[Tuple[int, str]] = []
+    try:
+        fence_spans = _extract_fence_spans(text)
+        # Single-brace variable placeholders (avoid matching {{ or }})
+        for m in re.finditer(r"(?<!\{)\{([A-Za-z_][A-Za-z0-9_]*)\}(?!\})", text):
+            if not _is_inside_any_span(m.start(), fence_spans):
+                line_no = text.count("\n", 0, m.start()) + 1
+                single_brace.append((line_no, m.group(0)))
+        # JavaScript template placeholders like ${...}
+        for m in re.finditer(r"\$\{[^\}]+\}", text):
+            if not _is_inside_any_span(m.start(), fence_spans):
+                line_no = text.count("\n", 0, m.start()) + 1
+                template_brace.append((line_no, m.group(0)))
+    except Exception:
+        pass
+    return single_brace, template_brace
+
 def preprocess(prompt: str, recursive: bool = False, double_curly_brackets: bool = True, exclude_keys: Optional[List[str]] = None) -> str:
     try:
         if not prompt:
             console.print("[bold red]Error:[/bold red] Empty prompt provided")
             return ""
+        _DEBUG_EVENTS.clear()
+        _dbg(f"Start preprocess(recursive={recursive}, double_curly={double_curly_brackets}, exclude_keys={exclude_keys})")
+        _dbg(f"Initial length: {len(prompt)} characters")
         console.print(Panel("Starting prompt preprocessing", style="bold blue"))
         prompt = process_backtick_includes(prompt, recursive)
+        _dbg("After backtick includes processed")
         prompt = process_xml_tags(prompt, recursive)
+        _dbg("After XML-like tags processed")
         if double_curly_brackets:
             prompt = double_curly(prompt, exclude_keys)
+            _dbg("After double_curly execution")
+        # Scan for risky placeholders remaining outside code fences
+        singles, templates = _scan_risky_placeholders(prompt)
+        if singles:
+            _dbg(f"WARNING: Found {len(singles)} single-brace placeholders outside code fences (examples):")
+            for ln, frag in singles[:5]:
+                _dbg(f"  line {ln}: {frag}")
+        if templates:
+            _dbg(f"INFO: Found {len(templates)} template literals ${'{...'} outside code fences (examples):")
+            for ln, frag in templates[:5]:
+                _dbg(f"  line {ln}: {frag}")
         # Don't trim whitespace that might be significant for the tests
         console.print(Panel("Preprocessing complete", style="bold green"))
+        _dbg(f"Final length: {len(prompt)} characters")
+        _write_debug_report()
         return prompt
     except Exception as e:
         console.print(f"[bold red]Error during preprocessing:[/bold red] {str(e)}")
         console.print(Panel(traceback.format_exc(), title="Error Details", style="red"))
+        _dbg(f"Exception: {str(e)}")
+        _write_debug_report()
         return prompt
 
 def get_file_path(file_name: str) -> str:
-    base_path = './'
-    return os.path.join(base_path, file_name)
+    resolver = get_default_resolver()
+    resolved = resolver.resolve_include(file_name)
+    if not Path(file_name).is_absolute() and resolved == resolver.cwd / file_name:
+        return os.path.join("./", file_name)
+    return str(resolved)
 
 def process_backtick_includes(text: str, recursive: bool) -> str:
     # More specific pattern that doesn't match nested > characters
@@ -45,12 +166,17 @@ def process_backtick_includes(text: str, recursive: bool) -> str:
                 content = file.read()
                 if recursive:
                     content = preprocess(content, recursive=True, double_curly_brackets=False)
+                _dbg(f"Included via backticks: {file_path} (len={len(content)})")
                 return f"```{content}```"
         except FileNotFoundError:
             console.print(f"[bold red]Warning:[/bold red] File not found: {file_path}")
-            return match.group(0)
+            _dbg(f"Missing backtick include: {file_path}")
+            # First pass (recursive=True): leave the tag so a later env expansion can resolve it
+            # Second pass (recursive=False): replace with a visible placeholder
+            return match.group(0) if recursive else f"```[File not found: {file_path}]```"
         except Exception as e:
             console.print(f"[bold red]Error processing include:[/bold red] {str(e)}")
+            _dbg(f"Error processing backtick include {file_path}: {e}")
             return f"```[Error processing include: {file_path}]```"
     prev_text = ""
     current_text = text
@@ -62,9 +188,9 @@ def process_backtick_includes(text: str, recursive: bool) -> str:
 def process_xml_tags(text: str, recursive: bool) -> str:
     text = process_pdd_tags(text)
     text = process_include_tags(text, recursive)
-
-    text = process_shell_tags(text)
-    text = process_web_tags(text)
+    text = process_include_many_tags(text, recursive)
+    text = process_shell_tags(text, recursive)
+    text = process_web_tags(text, recursive)
     return text
 
 def process_include_tags(text: str, recursive: bool) -> str:
@@ -73,23 +199,74 @@ def process_include_tags(text: str, recursive: bool) -> str:
         file_path = match.group(1).strip()
         try:
             full_path = get_file_path(file_path)
-            console.print(f"Processing XML include: [cyan]{full_path}[/cyan]")
-            with open(full_path, 'r', encoding='utf-8') as file:
-                content = file.read()
-                if recursive:
-                    content = preprocess(content, recursive=True, double_curly_brackets=False)
-                return content
+            ext = os.path.splitext(file_path)[1].lower()
+            image_extensions = ['.png', '.jpg', '.jpeg', '.gif', '.webp', '.heic']
+            
+            if ext in image_extensions:
+                console.print(f"Processing image include: [cyan]{full_path}[/cyan]")
+                from PIL import Image
+                import io
+                import pillow_heif
+                
+                pillow_heif.register_heif_opener()
+
+                MAX_DIMENSION = 1024
+                with open(full_path, 'rb') as file:
+                    img = Image.open(file)
+                    img.load() # Force loading the image data before the file closes
+                    
+                    if img.width > MAX_DIMENSION or img.height > MAX_DIMENSION:
+                        img.thumbnail((MAX_DIMENSION, MAX_DIMENSION))
+                        console.print(f"Image resized to {img.size}")
+
+                # Handle GIFs: convert to a static PNG of the first frame
+                if ext == '.gif':
+                    img.seek(0)
+                    img = img.convert("RGB")
+                    img_format = 'PNG'
+                    mime_type = 'image/png'
+                elif ext == '.heic':
+                    img_format = 'JPEG'
+                    mime_type = 'image/jpeg'
+                else:
+                    img_format = 'JPEG' if ext in ['.jpg', '.jpeg'] else 'PNG'
+                    mime_type = f'image/{img_format.lower()}'
+
+                # Save the (potentially resized and converted) image to an in-memory buffer
+                buffer = io.BytesIO()
+                img.save(buffer, format=img_format)
+                content = buffer.getvalue()
+
+                encoded_string = base64.b64encode(content).decode('utf-8')
+                return f"data:{mime_type};base64,{encoded_string}"
+            else:
+                console.print(f"Processing XML include: [cyan]{full_path}[/cyan]")
+                with open(full_path, 'r', encoding='utf-8') as file:
+                    content = file.read()
+                    if recursive:
+                        content = preprocess(content, recursive=True, double_curly_brackets=False)
+                    _dbg(f"Included via XML tag: {file_path} (len={len(content)})")
+                    return content
         except FileNotFoundError:
             console.print(f"[bold red]Warning:[/bold red] File not found: {file_path}")
-            return f"[File not found: {file_path}]"
+            _dbg(f"Missing XML include: {file_path}")
+            # First pass (recursive=True): leave the tag so a later env expansion can resolve it
+            # Second pass (recursive=False): replace with a visible placeholder
+            return match.group(0) if recursive else f"[File not found: {file_path}]"
         except Exception as e:
             console.print(f"[bold red]Error processing include:[/bold red] {str(e)}")
+            _dbg(f"Error processing XML include {file_path}: {e}")
             return f"[Error processing include: {file_path}]"
     prev_text = ""
     current_text = text
     while prev_text != current_text:
         prev_text = current_text
-        current_text = re.sub(pattern, replace_include, current_text, flags=re.DOTALL)
+        code_spans = _extract_code_spans(current_text)
+        def replace_include_with_spans(match):
+            if _intersects_any_span(match.start(), match.end(), code_spans):
+                return match.group(0)
+            return replace_include(match)
+        current_text = re.sub(pattern, replace_include_with_spans, current_text, flags=re.DOTALL)
     return current_text
 
 def process_pdd_tags(text: str) -> str:
@@ -101,54 +278,120 @@ def process_pdd_tags(text: str) -> str:
         return "This is a test "
     return processed
 
-def process_shell_tags(text: str) -> str:
+def process_shell_tags(text: str, recursive: bool) -> str:
     pattern = r'<shell>(.*?)</shell>'
     def replace_shell(match):
         command = match.group(1).strip()
+        if recursive:
+            # Defer execution until after env var expansion
+            return match.group(0)
         console.print(f"Executing shell command: [cyan]{escape(command)}[/cyan]")
+        _dbg(f"Shell tag command: {command}")
         try:
             result = subprocess.run(command, shell=True, check=True, capture_output=True, text=True)
             return result.stdout
         except subprocess.CalledProcessError as e:
             error_msg = f"Command '{command}' returned non-zero exit status {e.returncode}."
             console.print(f"[bold red]Error:[/bold red] {error_msg}")
+            _dbg(f"Shell command error: {error_msg}")
             return f"Error: {error_msg}"
         except Exception as e:
             console.print(f"[bold red]Error executing shell command:[/bold red] {str(e)}")
+            _dbg(f"Shell execution exception: {e}")
             return f"[Shell execution error: {str(e)}]"
-    return re.sub(pattern, replace_shell, text, flags=re.DOTALL)
+    code_spans = _extract_code_spans(text)
+    def replace_shell_with_spans(match):
+        if _intersects_any_span(match.start(), match.end(), code_spans):
+            return match.group(0)
+        return replace_shell(match)
+    return re.sub(pattern, replace_shell_with_spans, text, flags=re.DOTALL)
 
-def process_web_tags(text: str) -> str:
+def process_web_tags(text: str, recursive: bool) -> str:
     pattern = r'<web>(.*?)</web>'
     def replace_web(match):
         url = match.group(1).strip()
+        if recursive:
+            # Defer network operations until after env var expansion
+            return match.group(0)
         console.print(f"Scraping web content from: [cyan]{url}[/cyan]")
+        _dbg(f"Web tag URL: {url}")
         try:
             try:
-                from firecrawl import FirecrawlApp
+                from firecrawl import Firecrawl
             except ImportError:
+                _dbg("firecrawl import failed; package not installed")
                 return f"[Error: firecrawl-py package not installed. Cannot scrape {url}]"
             api_key = os.environ.get('FIRECRAWL_API_KEY')
             if not api_key:
                 console.print("[bold yellow]Warning:[/bold yellow] FIRECRAWL_API_KEY not found in environment")
+                _dbg("FIRECRAWL_API_KEY not set")
                 return f"[Error: FIRECRAWL_API_KEY not set. Cannot scrape {url}]"
-            app = FirecrawlApp(api_key=api_key)
-            response = app.scrape_url(url, formats=['markdown'])
-            if hasattr(response, 'markdown'):
+            app = Firecrawl(api_key=api_key)
+            response = app.scrape(url, formats=['markdown'])
+            # Handle both dict response (new API) and object response (legacy)
+            if isinstance(response, dict) and 'markdown' in response:
+                _dbg(f"Web scrape returned markdown (len={len(response['markdown'])})")
+                return response['markdown']
+            elif hasattr(response, 'markdown'):
+                _dbg(f"Web scrape returned markdown (len={len(response.markdown)})")
                 return response.markdown
             else:
                 console.print(f"[bold yellow]Warning:[/bold yellow] No markdown content returned for {url}")
+                _dbg("Web scrape returned no markdown content")
                 return f"[No content available for {url}]"
         except Exception as e:
             console.print(f"[bold red]Error scraping web content:[/bold red] {str(e)}")
+            _dbg(f"Web scraping exception: {e}")
             return f"[Web scraping error: {str(e)}]"
-    return re.sub(pattern, replace_web, text, flags=re.DOTALL)
+    code_spans = _extract_code_spans(text)
+    def replace_web_with_spans(match):
+        if _intersects_any_span(match.start(), match.end(), code_spans):
+            return match.group(0)
+        return replace_web(match)
+    return re.sub(pattern, replace_web_with_spans, text, flags=re.DOTALL)
+
+def process_include_many_tags(text: str, recursive: bool) -> str:
+    """Process <include-many> blocks whose inner content is a comma- or newline-separated
+    list of file paths (typically provided via variables after env expansion)."""
+    pattern = r'<include-many>(.*?)</include-many>'
+    def replace_many(match):
+        inner = match.group(1)
+        if recursive:
+            # Wait for env expansion to materialize the list
+            return match.group(0)
+        # Split by newlines or commas
+        raw_items = [s.strip() for part in inner.split('\n') for s in part.split(',')]
+        paths = [p for p in raw_items if p]
+        contents: list[str] = []
+        for p in paths:
+            try:
+                full_path = get_file_path(p)
+                console.print(f"Including (many): [cyan]{full_path}[/cyan]")
+                with open(full_path, 'r', encoding='utf-8') as fh:
+                    contents.append(fh.read())
+                _dbg(f"Included (many): {p}")
+            except FileNotFoundError:
+                console.print(f"[bold red]Warning:[/bold red] File not found: {p}")
+                _dbg(f"Missing include-many: {p}")
+                contents.append(f"[File not found: {p}]")
+            except Exception as e:
+                console.print(f"[bold red]Error processing include-many:[/bold red] {str(e)}")
+                _dbg(f"Error processing include-many {p}: {e}")
+                contents.append(f"[Error processing include: {p}]")
+        return "\n".join(contents)
+    code_spans = _extract_code_spans(text)
+    def replace_many_with_spans(match):
+        if _intersects_any_span(match.start(), match.end(), code_spans):
+            return match.group(0)
+        return replace_many(match)
+    return re.sub(pattern, replace_many_with_spans, text, flags=re.DOTALL)
 
 def double_curly(text: str, exclude_keys: Optional[List[str]] = None) -> str:
     if exclude_keys is None:
         exclude_keys = []
     
     console.print("Doubling curly brackets...")
+    _dbg("double_curly invoked")
     
     # Special case handling for specific test patterns
     if "Mix of {excluded{inner}} nesting" in text and "excluded" in exclude_keys:
@@ -172,6 +415,14 @@ def double_curly(text: str, exclude_keys: Optional[List[str]] = None) -> str:
             "2": {{"id": "2", "name": "Resource Two"}}
         }}"""
     
+    # Protect ${IDENT} placeholders so we can safely double braces, then restore
+    # them as ${{IDENT}} to avoid PromptTemplate interpreting {IDENT}.
+    protected_vars: List[str] = []
+    def _protect_var(m):
+        protected_vars.append(m.group(0))
+        return f"__PDD_VAR_{len(protected_vars)-1}__"
+    text = re.sub(r"\$\{[A-Za-z_][A-Za-z0-9_]*\}", _protect_var, text)
+
     # First, protect any existing double curly braces
     text = re.sub(r'\{\{([^{}]*)\}\}', r'__ALREADY_DOUBLED__\1__END_ALREADY__', text)
     
@@ -188,6 +439,24 @@ def double_curly(text: str, exclude_keys: Optional[List[str]] = None) -> str:
     
     # Restore already doubled brackets
     text = re.sub(r'__ALREADY_DOUBLED__(.*?)__END_ALREADY__', r'{{\1}}', text)
+
+    # Restore protected ${IDENT} placeholders as ${{IDENT}} so single braces
+    # don't leak into PromptTemplate formatting. This is safe for JS template
+    # literals and prevents missing-key errors in later formatting steps.
+    def _restore_var(m):
+        idx = int(m.group(1))
+        if 0 <= idx < len(protected_vars):
+            original = protected_vars[idx]  # e.g., ${FOO}
+            try:
+                inner = re.match(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}", original)
+                if inner:
+                    # Build as concatenation to avoid f-string brace escaping confusion
+                    return "${{" + inner.group(1) + "}}"  # -> ${{FOO}}
+            except Exception:
+                pass
+            return original
+        return m.group(0)
+    text = re.sub(r"__PDD_VAR_(\d+)__", _restore_var, text)
     
     # Special handling for code blocks
     code_block_pattern = r'```([\w\s]*)\n([\s\S]*?)```'
@@ -213,4 +482,4 @@ def double_curly(text: str, exclude_keys: Optional[List[str]] = None) -> str:
     # Process code blocks
     text = re.sub(code_block_pattern, process_code_block, text, flags=re.DOTALL)
     
-    return text
+    return text

pdd/preprocess_main.py

@@ -1,15 +1,23 @@
 import csv
 import sys
+from pathlib import Path
 from typing import Tuple, Optional
 import click
 from rich import print as rprint
 
+from .config_resolution import resolve_effective_config
 from .construct_paths import construct_paths
 from .preprocess import preprocess
 from .xml_tagger import xml_tagger
-from . import DEFAULT_TIME, DEFAULT_STRENGTH
+from .architecture_sync import (
+    get_architecture_entry_for_prompt,
+    generate_tags_from_architecture,
+    has_pdd_tags,
+)
+
+
 def preprocess_main(
-    ctx: click.Context, prompt_file: str, output: Optional[str], xml: bool, recursive: bool, double: bool, exclude: list
+    ctx: click.Context, prompt_file: str, output: Optional[str], xml: bool, recursive: bool, double: bool, exclude: list, pdd_tags: bool = False
 ) -> Tuple[str, float, str]:
     """
     CLI wrapper for preprocessing prompts.
@@ -22,6 +30,7 @@ def preprocess_main(
     :param double: If True, curly brackets will be doubled.
     :param exclude: List of keys to exclude from curly bracket doubling.
     :return: Tuple containing the preprocessed prompt, total cost, and model name used.
+    :param pdd_tags: If True, inject PDD metadata tags from architecture.json.
     """
     try:
         # Construct file paths
@@ -33,17 +42,41 @@ def preprocess_main(
             quiet=ctx.obj.get("quiet", False),
             command="preprocess",
             command_options=command_options,
+            context_override=ctx.obj.get('context')
         )
 
         # Load prompt file
         prompt = input_strings["prompt_file"]
 
+        # Inject PDD metadata tags from architecture.json if requested
+        pdd_tags_injected = False
+        if pdd_tags:
+            prompt_filename = Path(prompt_file).name
+            arch_entry = get_architecture_entry_for_prompt(prompt_filename)
+
+            if arch_entry:
+                if has_pdd_tags(prompt):
+                    if not ctx.obj.get("quiet", False):
+                        rprint(f"[yellow]Prompt already has PDD tags, skipping injection.[/yellow]")
+                else:
+                    generated_tags = generate_tags_from_architecture(arch_entry)
+                    if generated_tags:
+                        prompt = generated_tags + '\n\n' + prompt
+                        pdd_tags_injected = True
+                        if not ctx.obj.get("quiet", False):
+                            rprint(f"[green]Injected PDD tags from architecture.json[/green]")
+            else:
+                if not ctx.obj.get("quiet", False):
+                    rprint(f"[yellow]No architecture entry found for '{prompt_filename}', skipping PDD tags.[/yellow]")
+
         if xml:
             # Use xml_tagger to add XML delimiters
-            strength = ctx.obj.get("strength", DEFAULT_STRENGTH)
-            temperature = ctx.obj.get("temperature", 0.0)
+            # Use centralized config resolution with proper priority: CLI > pddrc > defaults
+            effective_config = resolve_effective_config(ctx, resolved_config)
+            strength = effective_config["strength"]
+            temperature = effective_config["temperature"]
+            time = effective_config["time"]
             verbose = ctx.obj.get("verbose", False)
-            time = ctx.obj.get("time", DEFAULT_TIME)
             xml_tagged, total_cost, model_name = xml_tagger(
                 prompt,
                 strength,
@@ -64,6 +97,8 @@ def preprocess_main(
         # Provide user feedback
         if not ctx.obj.get("quiet", False):
             rprint("[bold green]Prompt preprocessing completed successfully.[/bold green]")
+            if pdd_tags_injected:
+                rprint("[bold]PDD metadata tags: injected from architecture.json[/bold]")
             if xml:
                 rprint(f"[bold]XML Tagging used: {model_name}[/bold]")
             else:
@@ -76,4 +111,4 @@ def preprocess_main(
     except Exception as e:
         if not ctx.obj.get("quiet", False):
             rprint(f"[bold red]Error during preprocessing:[/bold red] {e}")
-        sys.exit(1)
+        sys.exit(1)