data2prompt 0.1.0__py3-none-any.whl

data2prompt/cli.py

@@ -0,0 +1,146 @@
+import argparse
+from dataclasses import dataclass, field
+from typing import List, Set
+from pathlib import Path
+from argparse import Namespace
+
+from .constants import (
+    CORE_IGNORES,
+    CORE_IGNORE_FILES,
+    CORE_SKIP_EXTS,
+    DEFAULT_CSV_SAMPLE_SIZE,
+    DEFAULT_SQL_SAMPLE_SIZE,
+    DEFAULT_SQL_MAX_LINES,
+    DEFAULT_MAX_LINES,
+    DEFAULT_MAX_SHEETS,
+    DEFAULT_SEED,
+    DEFAULT_LINE_LENGTH_THRESHOLD,
+    DEFAULT_TRUNCATED_LINE_LENGTH,
+    DEFAULT_TABLE_CHAR_LIMIT,
+    DEFAULT_TABLE_TRUNCATED_SIZE,
+    DEFAULT_MAX_FILE_SIZE_KB,
+    DEFAULT_OUTPUT_FILE,
+    DEFAULT_FORMAT,
+    SUPPORTED_FORMATS
+)
+
+@dataclass
+class Config:
+    """Data Transfer Object for application configuration."""
+    output: str
+    format: str
+    csv_sample_size: int
+    seed: int
+    sql_sample_size: int
+    sql_max_lines: int
+    max_lines: int
+    max_sheets: int
+    line_length_threshold: int
+    truncated_line_length: int
+    table_limit: int
+    table_truncate: int
+    ignore_folders: Set[str] = field(default_factory=set)
+    ignore_files: Set[str] = field(default_factory=set)
+    max_file_size: int = 0
+    skip_exts: Set[str] = field(default_factory=set)
+    use_gitignore: bool = True
+
+def setup_cli() -> Config:
+    """Configures the Command Line Interface (CLI) for the tool.
+
+    Defines all available flags and their help descriptions.
+
+    Returns:
+        Config: A type-safe configuration object.
+    """
+    parser = argparse.ArgumentParser(
+        description="📊 Data2Prompt: High-tech prompt packaging for Data Scientists."
+    )
+    
+    # Output settings
+    parser.add_argument('-o', '--output', default=DEFAULT_OUTPUT_FILE,
+                        help=f'Base name of the generated file (default: {DEFAULT_OUTPUT_FILE})')
+    
+    parser.add_argument('-f', '--format', choices=list(SUPPORTED_FORMATS.keys()), default=DEFAULT_FORMAT,
+                        help=f'Output format: xml or markdown (default: {DEFAULT_FORMAT})')
+    
+    # CSV sampling settings
+    parser.add_argument('-s', '--csv-sample-size', type=int, default=DEFAULT_CSV_SAMPLE_SIZE,
+                        help=f'Number of random rows to sample from CSVs (default: {DEFAULT_CSV_SAMPLE_SIZE})')
+    parser.add_argument('--seed', type=int, default=DEFAULT_SEED,
+                        help=f'Random seed for consistent CSV sampling (default: {DEFAULT_SEED})')
+    
+    # SQL sampling settings
+    parser.add_argument('--sql-sample-size', type=int, default=DEFAULT_SQL_SAMPLE_SIZE,
+                        help=f'Number of INSERT statements to keep in SQL files (default: {DEFAULT_SQL_SAMPLE_SIZE})')
+    
+    parser.add_argument('--sql-max-lines', type=int, default=DEFAULT_SQL_MAX_LINES,
+                        help=f'Max non-data lines to keep in SQL files (default: {DEFAULT_SQL_MAX_LINES})')
+    
+    # Notebook settings
+    parser.add_argument('--max-lines', type=int, default=DEFAULT_MAX_LINES,
+                        help=f'Max lines of text output to keep per notebook cell (default: {DEFAULT_MAX_LINES})')
+    
+    # Excel settings
+    parser.add_argument('--max-sheets', type=int, default=DEFAULT_MAX_SHEETS,
+                        help=f'Max number of sheets to process in Excel files (default: {DEFAULT_MAX_SHEETS})')
+    
+    # Line Truncation settings
+    parser.add_argument('--line-length-threshold', type=int, default=DEFAULT_LINE_LENGTH_THRESHOLD,
+                        help=f'Max characters per line before truncation (default: {DEFAULT_LINE_LENGTH_THRESHOLD})')
+    parser.add_argument('--truncated-line-length', type=int, default=DEFAULT_TRUNCATED_LINE_LENGTH,
+                        help=f'Length to truncate long lines to (default: {DEFAULT_TRUNCATED_LINE_LENGTH})')
+
+    # Table Truncation settings
+    parser.add_argument('--table-limit', type=int, default=DEFAULT_TABLE_CHAR_LIMIT,
+                        help=f'Max characters for a single table/sheet after sampling (default: {DEFAULT_TABLE_CHAR_LIMIT})')
+    parser.add_argument('--table-truncate', type=int, default=DEFAULT_TABLE_TRUNCATED_SIZE,
+                        help=f'Length to truncate large tables to (default: {DEFAULT_TABLE_TRUNCATED_SIZE})')
+
+    # Exclusions
+    parser.add_argument('--ignore-folders', nargs='+', default=[],
+                        help='Additional folders to skip entirely')
+    
+    parser.add_argument('--ignore-files', nargs='+', default=[],
+                        help='Additional files to skip entirely')
+    
+    parser.add_argument('--max-file-size', type=int, default=DEFAULT_MAX_FILE_SIZE_KB,
+                        help=f'Max file size in KB to read entirely (default: {DEFAULT_MAX_FILE_SIZE_KB}KB)')
+    
+    # file formats to ignore
+    parser.add_argument('--skip-exts', nargs='+', default=[],
+                        help='Additional file extensions to skip content for')
+    
+    parser.add_argument('--no-gitignore', action='store_false', dest='use_gitignore',
+                        help='Disable automatic .gitignore detection and filtering')
+    
+    args = parser.parse_args()
+    
+    # --- Argument Merging Logic ---
+    # We combine the user's terminal input with our CORE constants.
+    # This ensures that even if a user provides custom ignores, essential items
+    # like '.git' or binary extensions are still respected.
+    
+    # Combine base name with format-specific extension
+    extension = SUPPORTED_FORMATS.get(args.format, SUPPORTED_FORMATS.get(DEFAULT_FORMAT))
+    final_output_name = f"{args.output}{extension}"
+
+    return Config(
+        output=final_output_name,
+        format=args.format,
+        csv_sample_size=args.csv_sample_size,
+        seed=args.seed,
+        sql_sample_size=args.sql_sample_size,
+        sql_max_lines=args.sql_max_lines,
+        max_lines=args.max_lines,
+        max_sheets=args.max_sheets,
+        line_length_threshold=args.line_length_threshold,
+        truncated_line_length=args.truncated_line_length,
+        table_limit=args.table_limit,
+        table_truncate=args.table_truncate,
+        ignore_folders=set(args.ignore_folders) | CORE_IGNORES,
+        ignore_files=set(args.ignore_files) | CORE_IGNORE_FILES,
+        max_file_size=args.max_file_size,
+        skip_exts=set(args.skip_exts) | CORE_SKIP_EXTS,
+        use_gitignore=args.use_gitignore
+    )

data2prompt/constants.py

@@ -0,0 +1,94 @@
+# --- Core Defaults & Constants ---
+
+# Folders matching these names are excluded from both the project tree and content processing.
+CORE_IGNORES = {
+    '.git', '__pycache__', 'venv', '.vscode', '.ipynb_checkpoints',
+    'node_modules', '.idea', 'dist', 'build', '.mypy_cache',
+    '.pytest_cache', 'target', '.docker', '.aws', '.gcloud',
+    '__MACOSX'
+}
+
+# Specific filenames that should be excluded from the entire process.
+CORE_IGNORE_FILES = set()
+
+# Files with these extensions will have their names listed in the project tree,
+# but their actual content will be skipped.
+CORE_SKIP_EXTS = {
+    # Data & Databases
+    '.pbix', '.db', '.sqlite', '.sqlite3', '.parquet', '.pkl', '.pickle', '.feather', '.h5',
+    # Compressed & Binary
+    '.zip', '.tar', '.gz', '.7z', '.rar', '.exe', '.dll', '.so', '.bin',
+    # Media
+    '.png', '.jpg', '.jpeg', '.gif', '.svg', '.pdf', '.mp4', '.mp3', '.mov',
+    # Environment & Secrets
+    '.env', '.venv', '.pyc', '.ds_store'
+}
+
+# Default values for CLI arguments and processing functions
+DEFAULT_CSV_SAMPLE_SIZE = 15                # Controls the number of rows per csv file.
+DEFAULT_SQL_SAMPLE_SIZE = 15                # Controls the number of INSERT/data rows kept per table in SQL files.
+DEFAULT_SQL_MAX_LINES = 50                  # Caps the total number of non-data lines (comments, setup, etc.) in SQL files.
+DEFAULT_MAX_LINES = 40                      # Max lines of text output to keep per notebook cell.
+DEFAULT_MAX_SHEETS = 10                     # Max number of sheets to process in Excel files.
+DEFAULT_SEED = 42                           # Random seed for consistent sampling.
+DEFAULT_LINE_LENGTH_THRESHOLD = 4000        # Max characters allowed per line before truncation is triggered.
+DEFAULT_TRUNCATED_LINE_LENGTH = 1000        # Number of characters to keep when a line is truncated.
+DEFAULT_TABLE_CHAR_LIMIT = 50000            # Max characters allowed for a single table/sheet representation after sampling.
+DEFAULT_TABLE_TRUNCATED_SIZE = 20000        # Number of characters to keep when a table/sheet is truncated due to size.
+DEFAULT_MAX_FILE_SIZE_KB = 70               # maximum file size of unhandled type to keep enitrely (if file is larger than that only the first 10kb will be shown)
+DEFAULT_OUTPUT_FILE = 'PROMPT'              # default output base name (extension added via --format)
+DEFAULT_FORMAT = 'markdown'                 # default output format
+
+# Mapping of format types to their respective file extensions
+SUPPORTED_FORMATS = {
+    'xml': '.xml',
+    'markdown': '.md'
+}
+
+# A unique identifier added to the top of every generated file to prevent recursive scanning.
+GENERATION_FLAG = "DATA2PROMPT_GENERATED_CONTENT"
+
+# --- LLM Structured Output Constants ---
+# Refactored System Instructions (Repomix Style)
+SYSTEM_INSTRUCTIONS_MARKDOWN = """## purpose: \nThis document is a structured representation of a codebase and data schema. It is designed to be consumed by a Large Language Model.
+The output is organized into sections:
+1. Directory Structure: List of all files in this project.
+2. Files: The content of each file, clearly labeled with its path using '## File: {path}' headers.
+For all standard files, content is wrapped in markdown code blocks using dynamic backtick depth to ensure robust nesting.
+For notebooks, individual cells are clearly labeled with cell numbers, types, and their respective file paths.
+For Excel files, individual sheets are clearly labeled with sheet names, numbers, and their respective file paths."""
+
+SYSTEM_INSTRUCTIONS_XML = """<purpose>\nThis document is a structured representation of a codebase and data schema. It is designed to be consumed by a Large Language Model.
+The output is organized into XML tags:
+1. <directory_structure>: List of all files in this project.
+2. <files>: Contains the repository's files.
+3. <file>: Represents a single file with a 'path' attribute.
+4. <cell>: Used within notebooks to encapsulate individual cells, featuring 'path', 'number', and 'type' attributes.
+5. <sheet>: Used within Excel files to encapsulate individual sheets, featuring 'name', 'number', and 'path' attributes.\n</purpose>"""
+
+# Updated Tags
+TAG_DIRECTORY_STRUCTURE = "directory_structure"
+TAG_FILES = "files"
+TAG_FILE = "file"
+TAG_CONTENT = "content" # Used for notebook cells
+
+# --- UI & Aesthetic Constants ---
+MATRIX_DARK_GREEN = (0, 150, 0)
+MATRIX_NEON_GREEN = (0, 255, 0)
+STARTUP_ANIMATION_DURATION = 0.9
+ANIMATION_FRAME_DELAY = 0.03
+
+# Scroll Bar Characters
+SCROLL_THUMB = "█"
+SCROLL_TRACK = "│"
+
+# ASCII Art for the application header
+ASCII_ART =  [
+"                                                                                                                   ",
+"  ██╗      ██████╗   █████╗  ████████╗  █████╗  ██████╗  ██████╗  ██████╗   ██████╗  ███╗   ███╗ ██████╗  ████████╗",
+"  ╚██╗     ██╔══██╗ ██╔══██╗ ╚══██╔══╝ ██╔══██╗ ╚════██╗ ██╔══██╗ ██╔══██╗ ██╔═══██╗ ████╗ ████║ ██╔══██╗ ╚══██╔══╝",
+"   ╚██╗    ██║  ██║ ███████║    ██║    ███████║  █████╔╝ ██████╔╝ ██████╔╝ ██║   ██║ ██╔████╔██║ ██████╔╝    ██║   ",
+"   ██╔╝    ██║  ██║ ██╔══██║    ██║    ██╔══██║ ██╔═══╝  ██╔═══╝  ██╔══██╗ ██║   ██║ ██║╚██╔╝██║ ██╔═══╝     ██║   ",
+"  ██╔╝     ██████╔╝ ██║  ██║    ██║    ██║  ██║ ███████╗ ██║      ██║  ██║ ╚██████╔╝ ██║ ╚═╝ ██║ ██║         ██║   ",
+"  ╚═╝      ╚═════╝  ╚═╝  ╚═╝    ╚═╝    ╚═╝  ╚═╝ ╚══════╝ ╚═╝      ╚═╝  ╚═╝  ╚═════╝  ╚═╝     ╚═╝ ╚═╝         ╚═╝   "
+    ]

data2prompt/main.py

@@ -0,0 +1,172 @@
+import warnings
+
+# Suppress known noisy warnings globally for a cleaner TUI experience
+# We do this before importing pandas to ensure the filters are in place
+warnings.filterwarnings("ignore", category=UserWarning, module="openpyxl")
+
+import pandas as pd
+# Now that pandas is imported, we can reference its error types
+warnings.filterwarnings("ignore", category=pd.errors.DtypeWarning)
+from pathlib import Path
+from typing import Set
+from .cli import setup_cli, Config
+from .parsers import registry, ParserResult, flatten_ir
+from .utils import ProjectScanner, count_tokens, check_connectivity
+from .ui import ui
+from .output import get_generator
+
+def get_ui_action(ext: str, skip_exts: Set[str]) -> str:
+    """Determines the UI action string based on file extension."""
+    if ext in skip_exts: return "Skipping"
+    elif ext == '.csv': return "Sampling"
+    elif ext == '.ipynb': return "Cleaning"
+    elif ext == '.sql': return "Parsing"
+    elif ext in ['.xlsx', '.xls']: return "Extracting"
+    return "Reading"
+
+def process_target_file(file_path: Path, config: Config) -> ParserResult:
+    """Handles a single file and returns its content, tokens, and metadata."""
+    ext = file_path.suffix.lower()
+    
+    if ext in config.skip_exts:
+        return ParserResult(
+            content=f"*Note: Content skipped for ({ext}) file based on exclusion rules.*\n",
+            tokens=0,
+            type=f"Excluded ({ext})",
+            status="Skipped (Exclusion)",
+            stats_update={"excluded_count": 1}
+        )
+    
+    parser = registry.get_parser(ext)
+    return parser.parse(file_path, config)
+
+def main():
+    """
+    The main entry point for the Data2Prompt CLI.
+    Orchestrates the argument parsing, file discovery, content processing, and Markdown generation.
+    """
+    config = setup_cli() # Retrieve user settings from the terminal
+    
+    project_path = Path.cwd()
+    scanner = ProjectScanner(
+        project_path=project_path,
+        ignore_folders=config.ignore_folders,
+        ignore_files=config.ignore_files,
+        output_file=config.output,
+        use_gitignore=config.use_gitignore
+    )
+    
+    # Collect all files first to set progress bar total
+    all_files = scanner.scan()
+    total_steps = 1 + 1 + len(all_files) + 1
+
+    # Initialize UI and start process
+    ui.on_start("[cyan]Starting process...[/cyan]", total=total_steps)
+
+    stats = {
+        "file_count": 0,
+        "csv_count": 0,
+        "notebook_count": 0,
+        "sql_count": 0,
+        "excel_count": 0,
+        "excel_sheets_count": 0,
+        "truncated_count": 0,
+        "binary_count": 0,
+        "excluded_count": 0
+    }
+    
+    # For the summary table
+    processed_files_info = []
+
+    with ui.progress_bar("[cyan]Starting process...[/cyan]", total=total_steps) as handler:
+        # 1. Checking connectivity
+        handler.on_progress("[cyan]Checking online connectivity...[/cyan]")
+        is_online = check_connectivity()
+        status_msg = "[green]Online[/green]" if is_online else "[yellow]Offline (using fallback)[/yellow]"
+        handler.on_progress(f"[cyan]Checking online connectivity... {status_msg}[/cyan]", advance=1)
+
+        # 2. Generating project tree
+        handler.on_progress("[cyan]Generating project tree...[/cyan]")
+        tree_text = scanner.generate_tree()
+        handler.on_progress("[cyan]Generating project tree...[/cyan]", advance=1)
+
+        # 2. Processing files
+        files_data = []
+        
+        for file_path in all_files:
+            relative_path = file_path.relative_to(project_path)
+            ext = file_path.suffix.lower()
+            stats["file_count"] += 1
+            
+            # Determine action for progress bar - show only filename
+            action = get_ui_action(ext, config.skip_exts)
+            handler.on_progress(f"[cyan]{action}[/cyan] [bold]{file_path.name}[/bold] [cyan]...[/cyan]")
+            
+            result = process_target_file(file_path, config)
+            if result.skip_file:
+                handler.on_progress(f"[cyan]{action}[/cyan] [bold]{file_path.name}[/bold] [cyan]...[/cyan]", advance=1)
+                continue
+
+            # Collect file data for the generator
+            files_data.append({
+                "path": str(relative_path),
+                "content": result.content,
+                "type": result.type,
+                "tokens": result.tokens,
+                "status": result.status
+            })
+            
+            # Update stats
+            for key, value in result.stats_update.items():
+                stats[key] += value
+            
+            processed_files_info.append({
+                "name": str(relative_path),
+                "type": result.type,
+                "tokens": result.tokens,
+                "status": result.status
+            })
+            
+            handler.on_progress(f"[cyan]{action}[/cyan] [bold]{file_path.name}[/bold] [cyan]...[/cyan]", advance=1)
+
+        # 3. Compiling project context
+        handler.on_progress("[cyan]Compiling project context...[/cyan]")
+        
+        # We need a temporary token count for the final report
+        # The generator will handle the final string construction
+        # We use flatten_ir to convert structured content to strings for token counting
+        temp_content = "\n".join([flatten_ir(f["content"]) for f in files_data]) + tree_text
+        total_tokens, method = count_tokens(temp_content)
+        
+        generator = get_generator(config.format)
+        final_output = generator.generate(
+            project_name=project_path.name,
+            tree_text=tree_text,
+            files_data=files_data,
+            stats=stats,
+            total_tokens=total_tokens,
+            token_method=method,
+            config=config
+        )
+
+        with open(config.output, 'w', encoding='utf-8') as f:
+            f.write(final_output)
+        handler.on_progress("[cyan]Compiling project context...[/cyan]", advance=1)
+
+    # Final File Size Check
+    file_size_kb = Path(config.output).stat().st_size / 1024
+    
+    # Display Final Report (Interactive Summary + Success Panel)
+    ui.print_final_report(processed_files_info, config.output, file_size_kb, total_tokens, stats, method)
+    
+    if file_size_kb > 2000:
+        ui.print_warning_panel(
+            "[bold yellow]WARNING:[/bold yellow] File is over 2MB. This might be too large for some context windows.\n"
+            "[bold cyan]Suggestion:[/bold cyan] Reduce --csv-sample-size, --sql-sample-size or --max-lines."
+        )
+
+if __name__ == "__main__":
+    main()
+
+# Alias for backward compatibility with stale entry point scripts
+run_packager = main

data2prompt/output.py

@@ -0,0 +1,235 @@
+import os
+import pandas as pd
+from abc import ABC, abstractmethod
+from typing import List, Dict, Any, TYPE_CHECKING
+from pathlib import Path
+from xml.sax.saxutils import escape, quoteattr
+
+if TYPE_CHECKING:
+    from .cli import Config
+
+from .constants import (
+    TAG_DIRECTORY_STRUCTURE,
+    TAG_FILES,
+    TAG_FILE,
+    TAG_CONTENT,
+    SYSTEM_INSTRUCTIONS_MARKDOWN,
+    SYSTEM_INSTRUCTIONS_XML,
+    GENERATION_FLAG
+)
+from .utils import get_dynamic_wrapper
+from .parsers import NotebookCellIR, TableIR, enforce_table_limit
+
+class OutputGenerator(ABC):
+    @abstractmethod
+    def generate(self, 
+                 project_name: str, 
+                 tree_text: str, 
+                 files_data: List[Dict[str, Any]], 
+                 stats: Dict[str, Any],
+                 total_tokens: int,
+                 token_method: str,
+                 config: 'Config' = None) -> str:
+        pass
+
+class MarkdownGenerator(OutputGenerator):
+    def generate(self, 
+                 project_name: str, 
+                 tree_text: str, 
+                 files_data: List[Dict[str, Any]], 
+                 stats: Dict[str, Any],
+                 total_tokens: int,
+                 token_method: str,
+                 config: 'Config' = None) -> str:
+        
+        timestamp = pd.Timestamp.now().strftime('%Y-%m-%d %H:%M')
+        method_label = "o200k_base" if token_method == "o200k_base" else "regex_fallback" if token_method == "regex_fallback" else "word_count"
+        
+        lines = [
+            f"<!-- {GENERATION_FLAG} -->",
+            "",
+            f"# codebase: {project_name}",
+            "",
+            SYSTEM_INSTRUCTIONS_MARKDOWN,
+            "",
+            f"> Generated on: {timestamp}",
+            f"> Tokens: {total_tokens} (est. via {method_label})",
+            "",
+            "# Directory Structure",
+            "```text",
+            tree_text,
+            "```",
+            "",
+            "# Files",
+            "",
+            "This section contains the contents of the repository's files.",
+            ""
+        ]
+        
+        for file_info in files_data:
+            rel_path = file_info['path']
+            # Normalize path to match directory structure (always use backslashes)
+            display_path = rel_path.replace(os.sep, '\\')
+            content = file_info['content']
+            ext = Path(rel_path).suffix.lower()
+            
+            lines.append(f"## File: {display_path}")
+            
+            if isinstance(content, list) and content and isinstance(content[0], NotebookCellIR):
+                # Render Notebook IR
+                for cell in content:
+                    lines.append(f"### Cell {cell.number} ({cell.type}) - {display_path}")
+                    wrapper = get_dynamic_wrapper(cell.source)
+                    lang = 'python' if cell.type == 'code' else 'markdown'
+                    lines.append(f"{wrapper}{lang}")
+                    lines.append(cell.source)
+                    lines.append(wrapper)
+                    
+                    if cell.outputs:
+                        lines.append("\n**Outputs:**")
+                        lines.append("```text")
+                        lines.append(cell.outputs)
+                        lines.append("```")
+                    lines.append("")
+            
+            elif isinstance(content, list) and content and isinstance(content[0], TableIR):
+                # Render Table IR (CSV/Excel)
+                for table in content:
+                    table_parts = []
+                    
+                    # Handle Excel Sheet Metadata
+                    if table.sheet_number is not None:
+                        lines.append(f"### Sheet {table.sheet_number}: {table.name} - {table.file_path}")
+                    
+                    if table.header_note:
+                        table_parts.append(table.header_note)
+                    
+                    if not table.df.empty:
+                        table_parts.append(table.df.to_markdown(index=False))
+                    
+                    if table.footer_note:
+                        table_parts.append(table.footer_note)
+                    
+                    table_text = "\n".join(table_parts)
+                    if config:
+                        table_text = enforce_table_limit(table_text, config.table_limit, config.table_truncate)
+                    
+                    lines.append(table_text)
+                    
+                    # Close Sheet block if applicable
+                    if table.sheet_number is not None:
+                        lines.append("---")
+                    
+                    lines.append("")
+            
+            else:
+                # Standard files or fallback string content
+                str_content = str(content)
+                wrapper = get_dynamic_wrapper(str_content)
+                lang = ext[1:] if ext and ext != '.md' else 'markdown' if ext == '.md' else 'text'
+                lines.append(f"{wrapper}{lang}")
+                lines.append(str_content)
+                lines.append(wrapper)
+            
+            lines.append("")
+            
+        return "\n".join(lines)
+
+class XMLGenerator(OutputGenerator):
+    def generate(self, 
+                 project_name: str, 
+                 tree_text: str, 
+                 files_data: List[Dict[str, Any]], 
+                 stats: Dict[str, Any],
+                 total_tokens: int,
+                 token_method: str,
+                 config: 'Config' = None) -> str:
+        
+        timestamp = pd.Timestamp.now().strftime('%Y-%m-%d %H:%M')
+        method_label = "o200k_base" if token_method == "o200k_base" else "regex_fallback" if token_method == "regex_fallback" else "word_count"
+        
+        lines = [
+            f"<!-- {GENERATION_FLAG} -->",
+            "",
+            f'<codebase name={quoteattr(project_name)}>',
+            "",
+            SYSTEM_INSTRUCTIONS_XML,
+            "",
+            "<metadata>",
+            f"    <generated_on>{timestamp}</generated_on>",
+            f'    <total_tokens method="{method_label}">{total_tokens}</total_tokens>',
+            "</metadata>",
+            "",
+            f"<{TAG_DIRECTORY_STRUCTURE}>",
+            escape(tree_text),
+            f"</{TAG_DIRECTORY_STRUCTURE}>",
+            "",
+            f"<{TAG_FILES}>",
+            "This section contains the contents of the repository's files.",
+            ""
+        ]
+        
+        for file_info in files_data:
+            rel_path = file_info['path']
+            # Normalize path to match directory structure (always use backslashes)
+            display_path = rel_path.replace(os.sep, '\\')
+            content = file_info['content']
+            
+            lines.append(f'<{TAG_FILE} path="{display_path}">')
+            
+            if isinstance(content, list) and content and isinstance(content[0], NotebookCellIR):
+                # Render Notebook IR to XML
+                for cell in content:
+                    lines.append(f'    <cell path="{display_path}" index="{cell.number}" type="{cell.type}">')
+                    lines.append(f'        <{TAG_CONTENT}>')
+                    lines.append(escape(cell.source))
+                    lines.append(f'        </{TAG_CONTENT}>')
+                    if cell.outputs:
+                        lines.append('        <outputs>')
+                        lines.append(escape(cell.outputs))
+                        lines.append('        </outputs>')
+                    lines.append('    </cell>')
+            
+            elif isinstance(content, list) and content and isinstance(content[0], TableIR):
+                # Render Table IR to XML
+                for table in content:
+                    # Handle Excel Sheet Metadata
+                    if table.sheet_number is not None:
+                        lines.append(f'<sheet name="{table.name}" sheet_number="{table.sheet_number}" path="{table.file_path}">')
+                    
+                    table_parts = []
+                    if table.header_note:
+                        table_parts.append(table.header_note)
+                    
+                    if not table.df.empty:
+                        table_parts.append(table.df.to_markdown(index=False))
+                    
+                    if table.footer_note:
+                        table_parts.append(table.footer_note)
+                    
+                    table_text = "\n".join(table_parts)
+                    if config:
+                        table_text = enforce_table_limit(table_text, config.table_limit, config.table_truncate)
+                    
+                    lines.append(escape(table_text))
+                    
+                    # Close Sheet block if applicable
+                    if table.sheet_number is not None:
+                        lines.append('</sheet>')
+            
+            else:
+                # Standard files or fallback string content
+                lines.append(str(content))
+            
+            lines.append(f"</{TAG_FILE}>")
+            lines.append("")
+            
+        lines.append(f"</{TAG_FILES}>")
+        lines.append("</codebase>")
+        
+        return "\n".join(lines)
+
+def get_generator(format_type: str) -> OutputGenerator:
+    if format_type.lower() == 'markdown':
+        return MarkdownGenerator()
+    return XMLGenerator()