chapman-datagen 0.1.0__tar.gz

chapman_datagen-0.1.0/PKG-INFO

@@ -0,0 +1,21 @@
+Metadata-Version: 2.4
+Name: chapman-datagen
+Version: 0.1.0
+Summary: A highly generalizable prompt-based dataset generator and Hugging Face uploader
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: openai>=1.0.0
+Requires-Dist: pandas>=1.5.0
+Requires-Dist: pyarrow>=10.0.0
+Requires-Dist: huggingface_hub>=0.15.0
+Requires-Dist: tqdm>=4.65.0
+Requires-Dist: jinja2>=3.0.0
+Requires-Dist: datasets>=2.0.0
+
+# chapman-datagen
+
+A highly generalizable prompt-based dataset generator and Hugging Face uploader.

chapman_datagen-0.1.0/README.md

@@ -0,0 +1,3 @@
+# chapman-datagen
+
+A highly generalizable prompt-based dataset generator and Hugging Face uploader.

chapman_datagen-0.1.0/chapman_datagen/__init__.py

@@ -0,0 +1,29 @@
+from chapman_datagen.config import (
+    PipelineConfig,
+    APIConfig,
+    InputConfig,
+    StepConfig,
+    OutputConfig,
+    HuggingFaceConfig
+)
+from chapman_datagen.pipeline import run_pipeline, run_generation_pipeline_async
+from chapman_datagen.generator import Generator
+from chapman_datagen.chunker import chunk_jsonl_to_parquet, write_parquet_chunks
+from chapman_datagen.uploader import upload_to_huggingface
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "PipelineConfig",
+    "APIConfig",
+    "InputConfig",
+    "StepConfig",
+    "OutputConfig",
+    "HuggingFaceConfig",
+    "run_pipeline",
+    "run_generation_pipeline_async",
+    "Generator",
+    "chunk_jsonl_to_parquet",
+    "write_parquet_chunks",
+    "upload_to_huggingface"
+]

chapman_datagen-0.1.0/chapman_datagen/chunker.py

@@ -0,0 +1,126 @@
+import os
+import io
+import math
+import shutil
+import logging
+import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+def estimate_rows_per_chunk(df: pd.DataFrame, max_chunk_size_mb: float) -> int:
+    """
+    Estimates the number of rows that can fit into a single Parquet file under the limit.
+    Uses the first 100 rows to estimate the average compressed size per row.
+    """
+    num_rows = len(df)
+    test_rows = min(num_rows, 100)
+    if test_rows == 0:
+        return 1000
+
+    test_df = df.iloc[:test_rows]
+    buffer = io.BytesIO()
+    try:
+        test_df.to_parquet(buffer, index=False)
+        compressed_size = len(buffer.getvalue())
+    except Exception as e:
+        logger.warning(f"Failed to estimate Parquet size in memory: {e}. Falling back to default row estimation.")
+        # Fallback estimation based on memory usage
+        compressed_size = test_df.memory_usage(deep=True).sum() * 0.3 # assume 30% compression
+
+    avg_row_size_bytes = compressed_size / test_rows
+    target_size_bytes = max_chunk_size_mb * 1024 * 1024
+    
+    # Use a safety factor of 0.85 to handle text length variability
+    safety_factor = 0.85
+    estimated_rows = int((target_size_bytes * safety_factor) / avg_row_size_bytes)
+    
+    # Ensure at least 1 row per chunk
+    return max(1, estimated_rows)
+
+def write_parquet_chunks(df: pd.DataFrame, output_dir: str, max_chunk_size_mb: float) -> list[str]:
+    """
+    Splits the dataframe and writes to parquet chunks of size <= max_chunk_size_mb.
+    Includes a safety checker that recursively splits a chunk if its final disk size exceeds the limit.
+    """
+    if os.path.exists(output_dir):
+        # Clear existing directory to avoid stale files
+        shutil.rmtree(output_dir)
+    os.makedirs(output_dir, exist_ok=True)
+
+    if df.empty:
+        logger.warning("Empty DataFrame provided. Writing empty parquet file.")
+        empty_path = os.path.join(output_dir, "data-00000-of-00001.parquet")
+        df.to_parquet(empty_path, index=False)
+        return [empty_path]
+
+    rows_per_chunk = estimate_rows_per_chunk(df, max_chunk_size_mb)
+    logger.info(f"Estimated rows per chunk: {rows_per_chunk} (for target size {max_chunk_size_mb} MB)")
+
+    # Initial slice mapping
+    initial_chunks = []
+    num_rows = len(df)
+    num_initial_chunks = math.ceil(num_rows / rows_per_chunk)
+    
+    for i in range(num_initial_chunks):
+        start_idx = i * rows_per_chunk
+        end_idx = min(start_idx + rows_per_chunk, num_rows)
+        initial_chunks.append(df.iloc[start_idx:end_idx])
+
+    # Queue of dataframes to write.
+    # Each item is a tuple: (dataframe, chunk_id)
+    write_queue = [(chunk_df, i) for i, chunk_df in enumerate(initial_chunks)]
+    written_files = []
+    
+    max_chunk_bytes = max_chunk_size_mb * 1024 * 1024
+    
+    while write_queue:
+        current_df, chunk_id = write_queue.pop(0)
+        
+        # Temp file name to check size
+        temp_name = f"data-chunk-{chunk_id}.parquet"
+        temp_path = os.path.join(output_dir, temp_name)
+        
+        current_df.to_parquet(temp_path, index=False)
+        file_size = os.path.getsize(temp_path)
+        
+        if file_size > max_chunk_bytes and len(current_df) > 1:
+            # The chunk exceeded the max size! Split it in half.
+            os.remove(temp_path)
+            mid = len(current_df) // 2
+            logger.warning(
+                f"Chunk {chunk_id} with {len(current_df)} rows size on disk ({file_size / (1024*1024):.2f} MB) "
+                f"exceeded maximum size limit {max_chunk_size_mb} MB. Splitting in half."
+            )
+            write_queue.insert(0, (current_df.iloc[mid:], f"{chunk_id}b"))
+            write_queue.insert(0, (current_df.iloc[:mid], f"{chunk_id}a"))
+        else:
+            written_files.append((temp_path, current_df))
+
+    # Rename files to follow standard dataset format: data-00000-of-0000N.parquet
+    total_chunks = len(written_files)
+    final_paths = []
+    
+    for i, (temp_path, chunk_df) in enumerate(written_files):
+        filename = f"data-{i:05d}-of-{total_chunks:05d}.parquet"
+        final_path = os.path.join(output_dir, filename)
+        os.rename(temp_path, final_path)
+        
+        actual_size_mb = os.path.getsize(final_path) / (1024 * 1024)
+        logger.info(f"Wrote chunk {i+1}/{total_chunks}: {filename} (size: {actual_size_mb:.2f} MB, rows: {len(chunk_df)})")
+        final_paths.append(final_path)
+
+    return final_paths
+
+def chunk_jsonl_to_parquet(jsonl_path: str, output_dir: str, max_chunk_size_mb: float) -> list[str]:
+    """
+    Utility function that loads a JSONL file, parses it to a DataFrame,
+    and runs the parquet chunking logic.
+    """
+    logger.info(f"Reading JSONL file from {jsonl_path}...")
+    try:
+        df = pd.read_json(jsonl_path, lines=True)
+    except Exception as e:
+        logger.error(f"Failed to read JSONL file: {e}")
+        raise ValueError(f"Could not load JSONL file at {jsonl_path}: {e}")
+        
+    return write_parquet_chunks(df, output_dir, max_chunk_size_mb)

chapman_datagen-0.1.0/chapman_datagen/cli.py

@@ -0,0 +1,193 @@
+import os
+import sys
+import argparse
+import logging
+import asyncio
+
+from chapman_datagen.config import PipelineConfig
+from chapman_datagen.pipeline import run_pipeline, run_generation_pipeline_async
+from chapman_datagen.chunker import chunk_jsonl_to_parquet
+from chapman_datagen.uploader import upload_to_huggingface
+
+def setup_logging(verbose: bool):
+    level = logging.DEBUG if verbose else logging.INFO
+    logging.basicConfig(
+        level=level,
+        format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S"
+    )
+    # Silence third-party logs a bit unless in debug mode
+    if not verbose:
+        logging.getLogger("openai").setLevel(logging.WARNING)
+        logging.getLogger("httpx").setLevel(logging.WARNING)
+        logging.getLogger("urllib3").setLevel(logging.WARNING)
+        logging.getLogger("huggingface_hub").setLevel(logging.WARNING)
+
+def handle_run(args):
+    """
+    Handles 'run' subcommand: executes input -> generate -> chunk -> upload.
+    """
+    if not os.path.exists(args.config):
+        print(f"Error: Config file not found at '{args.config}'", file=sys.stderr)
+        sys.exit(1)
+        
+    config = PipelineConfig.from_json_file(args.config)
+    
+    # Overrides
+    if args.api_key:
+        config.api.api_key = args.api_key
+    if args.base_url:
+        config.api.base_url = args.base_url
+    if args.hf_token:
+        if config.huggingface:
+            config.huggingface.token = args.hf_token
+        else:
+            print("Warning: Hugging Face token provided but 'huggingface' section missing in config.json. Upload will be skipped.")
+    if args.repo_id:
+        if config.huggingface:
+            config.huggingface.repo_id = args.repo_id
+        else:
+            print("Warning: Hugging Face repo_id provided but 'huggingface' section missing in config.json. Upload will be skipped.")
+            
+    run_pipeline(config)
+
+def handle_generate(args):
+    """
+    Handles 'generate' subcommand: only executes inputs and steps to generate output JSONL.
+    """
+    if not os.path.exists(args.config):
+        print(f"Error: Config file not found at '{args.config}'", file=sys.stderr)
+        sys.exit(1)
+        
+    config = PipelineConfig.from_json_file(args.config)
+    
+    if args.api_key:
+        config.api.api_key = args.api_key
+    if args.base_url:
+        config.api.base_url = args.base_url
+        
+    logging.info("Starting generation phase only...")
+    asyncio.run(run_generation_pipeline_async(config))
+    logging.info(f"Generation phase complete. Output saved to {config.output.jsonl_path}")
+
+def handle_process(args):
+    """
+    Handles 'process' subcommand: converts JSONL to Parquet chunks.
+    """
+    jsonl_path = args.jsonl
+    parquet_dir = args.parquet_dir
+    max_chunk_size = args.max_chunk_size
+    
+    if not jsonl_path or not os.path.exists(jsonl_path):
+        # Check if config is provided to fallback
+        if args.config and os.path.exists(args.config):
+            config = PipelineConfig.from_json_file(args.config)
+            jsonl_path = jsonl_path or config.output.jsonl_path
+            parquet_dir = parquet_dir or config.output.parquet_dir
+            max_chunk_size = max_chunk_size or config.output.max_parquet_chunk_size_mb
+            
+    if not jsonl_path or not os.path.exists(jsonl_path):
+        print("Error: Valid JSONL path must be provided via --jsonl or config.json", file=sys.stderr)
+        sys.exit(1)
+    if not parquet_dir:
+        print("Error: Output Parquet directory must be provided via --parquet-dir or config.json", file=sys.stderr)
+        sys.exit(1)
+    if max_chunk_size is None or max_chunk_size <= 0:
+        max_chunk_size = 20.0
+
+    logging.info(f"Starting Parquet chunking phase for {jsonl_path}...")
+    chunk_jsonl_to_parquet(
+        jsonl_path=jsonl_path,
+        output_dir=parquet_dir,
+        max_chunk_size_mb=max_chunk_size
+    )
+    logging.info(f"Parquet chunking phase complete. Saved to: {parquet_dir}")
+
+def handle_upload(args):
+    """
+    Handles 'upload' subcommand: uploads Parquet chunks directory to HF.
+    """
+    parquet_dir = args.parquet_dir
+    repo_id = args.repo_id
+    hf_token = args.hf_token
+    
+    if not parquet_dir or not os.path.exists(parquet_dir):
+        # Fallback to config.json
+        if args.config and os.path.exists(args.config):
+            config = PipelineConfig.from_json_file(args.config)
+            parquet_dir = parquet_dir or config.output.parquet_dir
+            if config.huggingface:
+                repo_id = repo_id or config.huggingface.repo_id
+                hf_token = hf_token or config.huggingface.token
+                
+    if not parquet_dir or not os.path.exists(parquet_dir):
+        print("Error: Valid Parquet directory must be provided via --parquet-dir or config.json", file=sys.stderr)
+        sys.exit(1)
+    if not repo_id:
+        print("Error: Hugging Face repo_id must be provided via --repo-id or config.json", file=sys.stderr)
+        sys.exit(1)
+        
+    logging.info(f"Starting Hugging Face upload phase for {parquet_dir}...")
+    repo_url = upload_to_huggingface(
+        folder_path=parquet_dir,
+        repo_id=repo_id,
+        token=hf_token
+    )
+    logging.info(f"Upload complete! Repository URL: {repo_url}")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="chapman-datagen: A Highly Generalizable Dataset Generator and Hugging Face Uploader."
+    )
+    parser.add_argument("-v", "--verbose", action="store_true", help="Enable verbose logging.")
+    
+    subparsers = parser.add_subparsers(dest="command", help="Available subcommands")
+    
+    # 1. run command
+    run_parser = subparsers.add_parser("run", help="Run the entire pipeline (generate -> chunk -> upload)")
+    run_parser.add_argument("--config", "-c", default="config.json", help="Path to config.json file (default: config.json)")
+    run_parser.add_argument("--api-key", help="Override OpenAI API key")
+    run_parser.add_argument("--base-url", help="Override OpenAI API base URL")
+    run_parser.add_argument("--hf-token", help="Override Hugging Face Hub token")
+    run_parser.add_argument("--repo-id", help="Override Hugging Face dataset repository ID")
+    
+    # 2. generate command
+    gen_parser = subparsers.add_parser("generate", help="Only generate the dataset (output raw JSONL)")
+    gen_parser.add_argument("--config", "-c", default="config.json", help="Path to config.json file (default: config.json)")
+    gen_parser.add_argument("--api-key", help="Override OpenAI API key")
+    gen_parser.add_argument("--base-url", help="Override OpenAI API base URL")
+
+    # 3. process command
+    proc_parser = subparsers.add_parser("process", help="Convert JSONL into size-bounded Parquet chunks")
+    proc_parser.add_argument("--config", "-c", help="Path to config.json file (optional fallback)")
+    proc_parser.add_argument("--jsonl", "-j", help="Path to JSONL source file")
+    proc_parser.add_argument("--parquet-dir", "-p", help="Output directory for Parquet chunks")
+    proc_parser.add_argument("--max-size", type=float, dest="max_chunk_size", help="Max size of Parquet chunk in MB")
+
+    # 4. upload command
+    up_parser = subparsers.add_parser("upload", help="Upload Parquet directory to Hugging Face")
+    up_parser.add_argument("--config", "-c", help="Path to config.json file (optional fallback)")
+    up_parser.add_argument("--parquet-dir", "-p", help="Path to Parquet chunks directory")
+    up_parser.add_argument("--repo-id", "-r", help="Hugging Face repo name (e.g. username/dataset)")
+    up_parser.add_argument("--hf-token", help="Hugging Face Hub API Token")
+
+    args = parser.parse_args()
+    
+    if not args.command:
+        parser.print_help()
+        sys.exit(0)
+        
+    setup_logging(args.verbose)
+    
+    if args.command == "run":
+        handle_run(args)
+    elif args.command == "generate":
+        handle_generate(args)
+    elif args.command == "process":
+        handle_process(args)
+    elif args.command == "upload":
+        handle_upload(args)
+
+if __name__ == "__main__":
+    main()

chapman_datagen-0.1.0/chapman_datagen/config.py

@@ -0,0 +1,156 @@
+import os
+import json
+from dataclasses import dataclass, field
+from typing import List, Dict, Any, Optional
+
+@dataclass
+class APIConfig:
+    api_key: Optional[str] = field(default=None)
+    base_url: str = field(default="https://api.openai.com/v1")
+    api_type: str = field(default="chat")  # "chat" or "completion"
+    model: str = field(default="gpt-4o-mini")
+    temperature: float = field(default=0.7)
+    max_tokens: int = field(default=1000)
+    concurrency_limit: int = field(default=10)
+
+    def __post_init__(self):
+        # Fallback to env variables if not set
+        if not self.api_key:
+            self.api_key = os.environ.get("OPENAI_API_KEY")
+        if self.base_url == "https://api.openai.com/v1" and os.environ.get("OPENAI_BASE_URL"):
+            self.base_url = os.environ.get("OPENAI_BASE_URL")
+        
+        # Validation
+        if self.api_type not in ("chat", "completion"):
+            raise ValueError(f"api_type must be either 'chat' or 'completion', got '{self.api_type}'")
+        if self.concurrency_limit <= 0:
+            raise ValueError("concurrency_limit must be greater than 0")
+
+@dataclass
+class InputConfig:
+    type: str  # "list", "file", "synthetic", "huggingface"
+    data: Optional[List[Dict[str, Any]]] = field(default=None)  # for type "list"
+    path: Optional[str] = field(default=None)  # for type "file"
+    criteria: Optional[str] = field(default=None)  # for type "synthetic"
+    num_prompts: Optional[int] = field(default=None)  # for type "synthetic"
+    generator_model: Optional[str] = field(default=None)  # for type "synthetic"
+    repo_id: Optional[str] = field(default=None)  # for type "huggingface"
+    split: Optional[str] = field(default=None)  # for type "huggingface"
+    subset: Optional[str] = field(default=None)  # for type "huggingface"
+    limit: Optional[int] = field(default=None)  # general limit for loading samples
+
+    def __post_init__(self):
+        valid_types = ("list", "file", "synthetic", "huggingface")
+        if self.type not in valid_types:
+            raise ValueError(f"input.type must be one of {valid_types}, got '{self.type}'")
+        
+        if self.type == "list" and not self.data:
+            raise ValueError("input.data must be provided when input.type is 'list'")
+        if self.type == "file" and not self.path:
+            raise ValueError("input.path must be provided when input.type is 'file'")
+        if self.type == "huggingface" and not self.repo_id:
+            raise ValueError("input.repo_id must be provided when input.type is 'huggingface'")
+        if self.type == "synthetic":
+            if not self.criteria:
+                raise ValueError("input.criteria must be provided when input.type is 'synthetic'")
+            if not self.num_prompts or self.num_prompts <= 0:
+                raise ValueError("input.num_prompts must be greater than 0 for synthetic input")
+        if self.limit is not None and self.limit <= 0:
+            raise ValueError("input.limit must be greater than 0")
+
+@dataclass
+class StepConfig:
+    prompt_template: str
+    type: str = field(default="llm")
+    system_prompt: Optional[str] = field(default=None)
+    output_field: str = field(default="response")
+    cot_field: Optional[str] = field(default=None)  # field to save reasoning/thinking
+    loop_field: Optional[str] = field(default=None)  # field to loop over (e.g. "messages")
+    loop_filter_role: Optional[str] = field(default=None)  # role to filter and rewrite (e.g. "assistant")
+    model: Optional[str] = field(default=None)
+    temperature: Optional[float] = field(default=None)
+    max_tokens: Optional[int] = field(default=None)
+
+    def __post_init__(self):
+        if self.type != "llm":
+            raise ValueError(f"step.type currently only supports 'llm', got '{self.type}'")
+        if not self.prompt_template:
+            raise ValueError("step.prompt_template cannot be empty")
+        if not self.output_field:
+            raise ValueError("step.output_field cannot be empty")
+        if self.cot_field == "":
+            raise ValueError("step.cot_field cannot be an empty string")
+        if self.loop_field == "":
+            raise ValueError("step.loop_field cannot be an empty string")
+
+@dataclass
+class OutputConfig:
+    jsonl_path: str
+    parquet_dir: str
+    max_parquet_chunk_size_mb: float = field(default=20.0)
+
+    def __post_init__(self):
+        if not self.jsonl_path:
+            raise ValueError("output.jsonl_path cannot be empty")
+        if not self.parquet_dir:
+            raise ValueError("output.parquet_dir cannot be empty")
+        if self.max_parquet_chunk_size_mb <= 0:
+            raise ValueError("output.max_parquet_chunk_size_mb must be greater than 0")
+
+@dataclass
+class HuggingFaceConfig:
+    repo_id: str
+    token: Optional[str] = field(default=None)
+    private: bool = field(default=False)
+
+    def __post_init__(self):
+        if not self.repo_id:
+            raise ValueError("huggingface.repo_id cannot be empty")
+        if not self.token:
+            self.token = os.environ.get("HF_TOKEN")
+
+@dataclass
+class PipelineConfig:
+    api: APIConfig
+    input: InputConfig
+    steps: List[StepConfig]
+    output: OutputConfig
+    huggingface: Optional[HuggingFaceConfig] = field(default=None)
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, Any]) -> "PipelineConfig":
+        # Parse api
+        api_data = d.get("api", {})
+        api = APIConfig(**api_data)
+
+        # Parse input
+        input_data = d.get("input")
+        if not input_data:
+            raise ValueError("Pipeline configuration must have an 'input' section")
+        input_cfg = InputConfig(**input_data)
+
+        # Parse steps
+        steps_data = d.get("steps")
+        if not steps_data:
+            raise ValueError("Pipeline configuration must have a 'steps' list")
+        if not isinstance(steps_data, list):
+            raise ValueError("'steps' must be a list of step configurations")
+        steps = [StepConfig(**s) for s in steps_data]
+
+        # Parse output
+        output_data = d.get("output")
+        if not output_data:
+            raise ValueError("Pipeline configuration must have an 'output' section")
+        output = OutputConfig(**output_data)
+
+        # Parse huggingface
+        hf_data = d.get("huggingface")
+        hf = HuggingFaceConfig(**hf_data) if hf_data else None
+
+        return cls(api=api, input=input_cfg, steps=steps, output=output, huggingface=hf)
+
+    @classmethod
+    def from_json_file(cls, path: str) -> "PipelineConfig":
+        with open(path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        return cls.from_dict(data)

chapman_datagen-0.1.0/chapman_datagen/generator.py

@@ -0,0 +1,129 @@
+import asyncio
+import random
+import logging
+from typing import Optional, List, Dict, Any
+from openai import AsyncOpenAI, RateLimitError, APIConnectionError, APIStatusError
+
+logger = logging.getLogger(__name__)
+
+class Generator:
+    def __init__(self, api_key: Optional[str], base_url: str, api_type: str = "chat"):
+        self.api_type = api_type
+        # Initialize client. If api_key is None, AsyncOpenAI will look up OPENAI_API_KEY from env automatically.
+        self.client = AsyncOpenAI(
+            api_key=api_key or "mock-key",  # Fallback to prevent client validation crash if mocking/stubbing
+            base_url=base_url
+        )
+        if api_key is None and not asyncio.iscoroutinefunction(self.client.with_options):
+            # If no API key is specified and it's not in the env, we set a dummy to avoid initialization error,
+            # but log a warning.
+            import os
+            if "OPENAI_API_KEY" not in os.environ:
+                logger.warning("No API Key provided in configuration or OPENAI_API_KEY environment variable. Request might fail.")
+
+    async def generate(
+        self,
+        prompt: str,
+        system_prompt: Optional[str] = None,
+        messages: Optional[List[Dict[str, str]]] = None,
+        model: str = "gpt-4o-mini",
+        temperature: float = 0.7,
+        max_tokens: int = 1000,
+        max_retries: int = 5,
+        initial_backoff: float = 1.0,
+    ) -> dict[str, str]:
+        """
+        Executes a prompt generation using either Chat Completions or Text Completions API.
+        Includes robust retry logic with exponential backoff and jitter.
+        Returns a dictionary containing "content" and "reasoning_content" keys.
+        """
+        retries = 0
+        backoff = initial_backoff
+
+        while True:
+            try:
+                if self.api_type == "chat":
+                    chat_messages = []
+                    if system_prompt:
+                        chat_messages.append({"role": "system", "content": system_prompt})
+                    
+                    if messages:
+                        # Append the pre-structured chat messages directly
+                        for msg in messages:
+                            chat_messages.append({
+                                "role": msg.get("role", "user"),
+                                "content": msg.get("content", "")
+                            })
+                        # Only append prompt as a separate instruction if it differs from the last message's content
+                        last_content = chat_messages[-1].get("content") if chat_messages else None
+                        if prompt and prompt != last_content:
+                            chat_messages.append({"role": "user", "content": prompt})
+                    else:
+                        chat_messages.append({"role": "user", "content": prompt})
+
+                    response = await self.client.chat.completions.create(
+                        model=model,
+                        messages=chat_messages,
+                        temperature=temperature,
+                        max_tokens=max_tokens,
+                        frequency_penalty=1.2,
+                        presence_penalty=0.5,
+                    )
+                    choice = response.choices[0]
+                    message = choice.message
+                    content = message.content or ""
+                    
+                    # Extract reasoning content (DeepSeek R1/OpenAI o-series standard)
+                    reasoning_content = getattr(message, "reasoning_content", None)
+                    if not reasoning_content:
+                        reasoning_content = getattr(message, "reasoning", None)
+                    if not reasoning_content and hasattr(message, "model_extra"):
+                        extra = message.model_extra or {}
+                        reasoning_content = extra.get("reasoning_content") or extra.get("reasoning")
+
+                    return {
+                        "content": content,
+                        "reasoning_content": reasoning_content or ""
+                    }
+                
+                elif self.api_type == "completion":
+                    full_prompt = prompt
+                    if system_prompt:
+                        full_prompt = f"{system_prompt}\n\n{prompt}"
+
+                    response = await self.client.completions.create(
+                        model=model,
+                        prompt=full_prompt,
+                        temperature=temperature,
+                        max_tokens=max_tokens,
+                    )
+                    choice = response.choices[0]
+                    content = choice.text or ""
+                    return {
+                        "content": content,
+                        "reasoning_content": ""
+                    }
+                
+                else:
+                    raise ValueError(f"Unsupported api_type: {self.api_type}")
+
+            except (RateLimitError, APIConnectionError, APIStatusError, Exception) as e:
+                # If we have reached max retries, propagate the error
+                if retries >= max_retries:
+                    logger.error(f"Max retries ({max_retries}) reached. Raising error: {e}")
+                    raise e
+                
+                # Check if it's a non-retryable API error (e.g. 401 Unauthorized, 404 Not Found)
+                if isinstance(e, APIStatusError) and e.status_code in (400, 401, 403, 404):
+                    logger.error(f"Non-retryable API Error {e.status_code}: {e.message}. Aborting request.")
+                    raise e
+
+                # Determine backoff time with jitter
+                sleep_time = backoff + (random.random() * 0.5 * backoff)
+                logger.warning(
+                    f"Request failed due to: {e}. Retrying in {sleep_time:.2f} seconds (Attempt {retries + 1}/{max_retries})..."
+                )
+                
+                await asyncio.sleep(sleep_time)
+                retries += 1
+                backoff *= 2  # Exponential backoff

chapman_datagen-0.1.0/chapman_datagen/pipeline.py

@@ -0,0 +1,466 @@
+import os
+import json
+import logging
+import asyncio
+import pandas as pd
+from typing import List, Dict, Any, Optional
+import tqdm.asyncio
+
+from chapman_datagen.config import PipelineConfig, StepConfig
+from chapman_datagen.generator import Generator
+from chapman_datagen.chunker import chunk_jsonl_to_parquet
+from chapman_datagen.uploader import upload_to_huggingface
+
+logger = logging.getLogger(__name__)
+
+def load_inputs(config: PipelineConfig, generator: Generator) -> List[Dict[str, Any]]:
+    """
+    Loads inputs based on the input configuration.
+    Supports lists, local files (CSV, JSON, JSONL, Parquet), synthetic LLM prompt generation,
+    or loading from the Hugging Face Hub (huggingface type).
+    """
+    input_cfg = config.input
+    logger.info(f"Loading inputs of type: {input_cfg.type}")
+
+    records: List[Dict[str, Any]] = []
+
+    if input_cfg.type == "list":
+        records = input_cfg.data or []
+
+    elif input_cfg.type == "file":
+        path = input_cfg.path
+        if not path or not os.path.exists(path):
+            raise FileNotFoundError(f"Input file not found at: {path}")
+
+        # Support CSV, Parquet, JSON, and JSONL via Pandas
+        ext = os.path.splitext(path)[1].lower()
+        try:
+            if ext == ".csv":
+                df = pd.read_csv(path)
+            elif ext == ".parquet":
+                df = pd.read_parquet(path)
+            elif ext in (".jsonl", ".json"):
+                # Guess lines=True/False
+                try:
+                    df = pd.read_json(path, lines=True)
+                except Exception:
+                    df = pd.read_json(path, lines=False)
+            else:
+                # Default fallback
+                df = pd.read_json(path, lines=True)
+            
+            # Convert NaN to None for clean JSON serialization
+            df = df.where(pd.notnull(df), None)
+            records = df.to_dict(orient="records")
+        except Exception as e:
+            raise ValueError(f"Failed to read input file {path}: {e}")
+
+    elif input_cfg.type == "huggingface":
+        # Load from Hugging Face Hub using the `datasets` library
+        try:
+            from datasets import load_dataset
+        except ImportError:
+            raise ImportError(
+                "The 'datasets' library is required to load Hugging Face datasets. "
+                "Please install it using: pip install datasets"
+            )
+        
+        logger.info(f"Loading Hugging Face dataset '{input_cfg.repo_id}' (split: '{input_cfg.split or 'train'}', subset: '{input_cfg.subset or 'default'}')...")
+        try:
+            ds = load_dataset(
+                input_cfg.repo_id,
+                name=input_cfg.subset,
+                split=input_cfg.split or "train"
+            )
+            records = ds.to_list()
+        except Exception as e:
+            raise ValueError(f"Failed to load dataset '{input_cfg.repo_id}' from Hugging Face: {e}")
+
+    elif input_cfg.type == "synthetic":
+        # Generate prompts using the LLM based on criteria
+        criteria = input_cfg.criteria
+        num_prompts = input_cfg.num_prompts or 10
+        model = input_cfg.generator_model or config.api.model
+        
+        logger.info(f"Generating {num_prompts} synthetic prompts based on criteria: '{criteria}'")
+        
+        system_prompt = "You are a synthetic dataset generator."
+        prompt = (
+            f"We are generating a dataset based on the following criteria:\n"
+            f"'{criteria}'\n\n"
+            f"Please generate a JSON list containing {num_prompts} distinct user prompts that match this criteria. "
+            f"The output must be a valid JSON array of strings, for example:\n"
+            f"[\n  \"Prompt 1\",\n  \"Prompt 2\"\n]\n"
+            f"Return ONLY the raw JSON (no markdown block, no explanation, no headers)."
+        )
+        
+        try:
+            loop = asyncio.get_running_loop()
+        except RuntimeError:
+            loop = None
+
+        if loop and loop.is_running():
+            # If loop is already running, run it in task
+            future = asyncio.run_coroutine_threadsafe(
+                generator.generate(prompt, system_prompt=system_prompt, model=model),
+                loop
+            )
+            response_dict = future.result()
+        else:
+            response_dict = asyncio.run(
+                generator.generate(prompt, system_prompt=system_prompt, model=model)
+            )
+        response_text = response_dict["content"]
+
+        # Parse response
+        response_text = response_text.strip()
+        if response_text.startswith("```"):
+            # Strip markdown code blocks
+            lines = response_text.splitlines()
+            if lines[0].startswith("```"):
+                lines = lines[1:]
+            if lines[-1].startswith("```"):
+                lines = lines[:-1]
+            response_text = "\n".join(lines).strip()
+            
+        try:
+            parsed = json.loads(response_text)
+            if not isinstance(parsed, list):
+                raise ValueError("Parsed JSON is not a list")
+            
+            # Format as records
+            for item in parsed:
+                if isinstance(item, dict):
+                    records.append(item)
+                else:
+                    records.append({"prompt": str(item)})
+        except Exception as e:
+            logger.error(f"Failed to parse synthetic prompts LLM response: {response_text}")
+            raise ValueError(f"Failed to parse generated prompts as JSON: {e}")
+    else:
+        raise ValueError(f"Unknown input type: {input_cfg.type}")
+
+    # Apply general sample limit if specified
+    if input_cfg.limit and input_cfg.limit > 0:
+        logger.info(f"Limiting loaded records to the first {input_cfg.limit} samples.")
+        records = records[:input_cfg.limit]
+
+    return records
+
+
+def extract_cot_from_text(text: str) -> tuple[str, str]:
+    """
+    Parses <think>...</think> or <reasoning>...</reasoning> blocks from text.
+    Returns a tuple of (cleaned_text, extracted_cot).
+    """
+    import re
+    # Pattern for <think>...</think>
+    think_pattern = re.compile(r'<think>(.*?)</think>', re.DOTALL)
+    think_match = think_pattern.search(text)
+    if think_match:
+        cot = think_match.group(1).strip()
+        cleaned = think_pattern.sub("", text).strip()
+        return cleaned, cot
+        
+    # Pattern for <reasoning>...</reasoning>
+    reasoning_pattern = re.compile(r'<reasoning>(.*?)</reasoning>', re.DOTALL)
+    reasoning_match = reasoning_pattern.search(text)
+    if reasoning_match:
+        cot = reasoning_match.group(1).strip()
+        cleaned = reasoning_pattern.sub("", text).strip()
+        return cleaned, cot
+        
+    return text, ""
+
+
+async def execute_step_for_record(
+    record: Dict[str, Any],
+    step: StepConfig,
+    generator: Generator,
+    global_model: str,
+    global_temp: float,
+    global_max_tokens: int,
+    semaphore: asyncio.Semaphore
+) -> Dict[str, Any]:
+    """
+    Executes a single step on a record under the semaphore lock.
+    If loop_field is specified, it loops over items in the field (e.g. conversation messages)
+    and rewrites them sequentially.
+    """
+    # Override settings if step defines them
+    model = step.model or global_model
+    temp = step.temperature if step.temperature is not None else global_temp
+    max_tokens = step.max_tokens if step.max_tokens is not None else global_max_tokens
+
+    if step.loop_field:
+        # Loop mode: rewrite elements within a list field in place
+        items = record.get(step.loop_field)
+        if not items or not isinstance(items, list):
+            logger.warning(f"loop_field '{step.loop_field}' not found or is not a list in record. Skipping loop execution.")
+            return record
+            
+        rewritten_items = []
+        cots = []
+
+        # We process the loop sequentially to build context step-by-step
+        for idx, item in enumerate(items):
+            if not isinstance(item, dict):
+                rewritten_items.append(item)
+                continue
+
+            role = item.get("role")
+            if role != step.loop_filter_role:
+                # Keep non-matching items (e.g. user messages) exactly unchanged
+                rewritten_items.append(item.copy())
+                continue
+
+            # We want to perform generation with the prefix history up to the current item.
+            # To do that, we construct a snapshot of the messages up to this point,
+            # but using the already rewritten history for previous assistant responses.
+            prefix_messages = rewritten_items.copy()
+            
+            # Format the prompt using both global record keys and helper variables
+            local_context = record.copy()
+            # If the template references '{messages}', feed the JSON prefix string
+            local_context["messages"] = json.dumps(prefix_messages)
+            local_context["original_response"] = item.get("content", "")
+            local_context["current_prompt"] = prefix_messages[-1].get("content", "") if prefix_messages else ""
+            
+            # Format system prompt and template
+            try:
+                formatted_prompt = step.prompt_template.format(**local_context)
+            except KeyError as e:
+                # If formatting fails, fallback to passing the original response directly
+                formatted_prompt = f"Original Response: {item.get('content', '')}"
+                
+            formatted_system = None
+            if step.system_prompt:
+                try:
+                    formatted_system = step.system_prompt.format(**local_context)
+                except KeyError as e:
+                    formatted_system = step.system_prompt
+
+            logger.info(f"Loop Step: Processing conversation turn {idx+1}/{len(items)} for role '{role}'...")
+            import time
+            start_time = time.time()
+            async with semaphore:
+                res_dict = await generator.generate(
+                    prompt=formatted_prompt,
+                    system_prompt=formatted_system,
+                    messages=prefix_messages,  # Pass native messages directly to Chat API
+                    model=model,
+                    temperature=temp,
+                    max_tokens=max_tokens
+                )
+            duration = time.time() - start_time
+
+            content = res_dict["content"]
+            reasoning_content = res_dict["reasoning_content"]
+            logger.info(f"Loop Step: Turn {idx+1} complete in {duration:.2f}s (generated {len(content)} chars, CoT: {len(reasoning_content) if reasoning_content else 0} chars).")
+
+            if step.cot_field:
+                if not reasoning_content:
+                    content, reasoning_content = extract_cot_from_text(content)
+                else:
+                    # API returned reasoning_content separately, but content may still
+                    # contain inline <think> tags — strip them so they don't pollute
+                    # the conversation history passed to subsequent turns.
+                    content, _ = extract_cot_from_text(content)
+                if reasoning_content:
+                    cots.append(reasoning_content)
+            else:
+                # Even without a cot_field, always strip think tags from content
+                # before storing it back into the conversation history.
+                content, _ = extract_cot_from_text(content)
+
+            # Update item content with the rewritten response (CoT stripped)
+            new_item = item.copy()
+            new_item["content"] = content
+            rewritten_items.append(new_item)
+
+        # Update the main record with the fully rewritten list
+        record[step.output_field] = rewritten_items
+        if step.cot_field and cots:
+            # Join all step CoTs together for database visibility
+            record[step.cot_field] = "\n\n=== Step CoT ===\n\n".join(cots)
+        return record
+
+    else:
+        # Standard mode: single generation
+        async with semaphore:
+            # Format the templates using the current record values
+            try:
+                formatted_prompt = step.prompt_template.format(**record)
+            except KeyError as e:
+                logger.error(f"Formatting failed for template '{step.prompt_template}' with record keys: {list(record.keys())}. Missing key: {e}")
+                raise KeyError(f"Missing variable {e} in record for step template '{step.prompt_template}'")
+            
+            formatted_system = None
+            if step.system_prompt:
+                try:
+                    formatted_system = step.system_prompt.format(**record)
+                except KeyError as e:
+                    formatted_system = step.system_prompt  # Fallback to literal string if formatting fails
+
+            # Generate response (returns dict with content and reasoning_content keys)
+            res_dict = await generator.generate(
+                prompt=formatted_prompt,
+                system_prompt=formatted_system,
+                model=model,
+                temperature=temp,
+                max_tokens=max_tokens
+            )
+            
+            response_text = res_dict["content"]
+            reasoning_content = res_dict["reasoning_content"]
+
+            if step.cot_field:
+                if not reasoning_content:
+                    # If no native reasoning content was returned by the API, parse it using XML-like tags
+                    response_text, reasoning_content = extract_cot_from_text(response_text)
+                
+                record[step.cot_field] = reasoning_content
+                record[step.output_field] = response_text
+            else:
+                record[step.output_field] = response_text
+                
+            return record
+
+
+async def process_record_pipeline(
+    record: Dict[str, Any],
+    steps: List[StepConfig],
+    generator: Generator,
+    config: PipelineConfig,
+    semaphore: asyncio.Semaphore,
+    output_file: Any,
+    write_lock: asyncio.Lock
+) -> Dict[str, Any]:
+    """
+    Passes a record through all pipeline steps sequentially,
+    then writes the result incrementally to the JSONL output file.
+    """
+    current_record = record.copy()
+    
+    for step in steps:
+        current_record = await execute_step_for_record(
+            record=current_record,
+            step=step,
+            generator=generator,
+            global_model=config.api.model,
+            global_temp=config.api.temperature,
+            global_max_tokens=config.api.max_tokens,
+            semaphore=semaphore
+        )
+    
+    # Save incrementally to JSONL file under the lock
+    async with write_lock:
+        output_file.write(json.dumps(current_record) + "\n")
+        output_file.flush()
+        
+    return current_record
+
+
+async def run_generation_pipeline_async(config: PipelineConfig) -> List[Dict[str, Any]]:
+    """
+    Executes the generation phase asynchronously.
+    """
+    # 1. Initialize generator client
+    generator = Generator(
+        api_key=config.api.api_key,
+        base_url=config.api.base_url,
+        api_type=config.api.api_type
+    )
+
+    # 2. Load inputs
+    records = load_inputs(config, generator)
+    if not records:
+        logger.warning("No input records found to process.")
+        return []
+
+    logger.info(f"Loaded {len(records)} records. Starting execution steps...")
+
+    # Ensure output file directory exists
+    jsonl_path = config.output.jsonl_path
+    os.makedirs(os.path.dirname(os.path.abspath(jsonl_path)) or ".", exist_ok=True)
+
+    # 3. Process records in parallel with concurrency semaphore
+    semaphore = asyncio.Semaphore(config.api.concurrency_limit)
+    write_lock = asyncio.Lock()
+    
+    # Open the file in write mode to clear previous runs
+    processed_records = []
+    with open(jsonl_path, "w", encoding="utf-8") as f:
+        tasks = [
+            process_record_pipeline(
+                record=record,
+                steps=config.steps,
+                generator=generator,
+                config=config,
+                semaphore=semaphore,
+                output_file=f,
+                write_lock=write_lock
+            )
+            for record in records
+        ]
+
+        logger.info(f"Running pipeline with concurrency limit={config.api.concurrency_limit}...")
+        # Use tqdm to show async progress bar
+        for task in tqdm.asyncio.tqdm.as_completed(tasks, total=len(records), desc="Generating dataset"):
+            res = await task
+            processed_records.append(res)
+            
+    logger.info(f"Generation phase complete. Wrote {len(processed_records)} records to {jsonl_path}")
+    return processed_records
+
+
+def run_pipeline(config: PipelineConfig) -> List[Dict[str, Any]]:
+    """
+    Programmatic entrypoint to run the entire dataset pipeline:
+    1. Runs generation steps asynchronously (incremental JSONL writing).
+    2. Chunks the JSONL to Parquet files under max_parquet_chunk_size_mb.
+    3. Uploads Parquet directory to Hugging Face if config.huggingface is provided.
+    """
+    logger.info("Starting chapman-datagen pipeline...")
+
+    try:
+        loop = asyncio.get_running_loop()
+    except RuntimeError:
+        loop = None
+
+    if loop and loop.is_running():
+        # Running in environment with active loop (e.g. Jupyter notebook or another framework)
+        # Check if we are running in an existing loop or create a task
+        try:
+            import nest_asyncio
+            nest_asyncio.apply()
+        except ImportError:
+            pass
+        results = loop.run_until_complete(run_generation_pipeline_async(config))
+    else:
+        results = asyncio.run(run_generation_pipeline_async(config))
+
+    # Chunks Parquet
+    logger.info("Starting Parquet chunking phase...")
+    parquet_files = chunk_jsonl_to_parquet(
+        jsonl_path=config.output.jsonl_path,
+        output_dir=config.output.parquet_dir,
+        max_chunk_size_mb=config.output.max_parquet_chunk_size_mb
+    )
+    logger.info(f"Successfully chunked into {len(parquet_files)} Parquet files.")
+
+    # Upload to HF if configured
+    if config.huggingface:
+        logger.info("Starting Hugging Face uploading phase...")
+        repo_url = upload_to_huggingface(
+            folder_path=config.output.parquet_dir,
+            repo_id=config.huggingface.repo_id,
+            token=config.huggingface.token,
+            private=config.huggingface.private
+        )
+        logger.info(f"Pipeline execution complete. Dataset uploaded to: {repo_url}")
+    else:
+        logger.info("No Hugging Face repository configured. Skipping upload phase.")
+        logger.info("Pipeline execution complete.")
+
+    return results

chapman_datagen-0.1.0/chapman_datagen/uploader.py

@@ -0,0 +1,47 @@
+import os
+import logging
+from typing import Optional
+from huggingface_hub import HfApi
+
+logger = logging.getLogger(__name__)
+
+def upload_to_huggingface(
+    folder_path: str,
+    repo_id: str,
+    token: Optional[str] = None,
+    private: bool = False
+) -> str:
+    """
+    Creates/updates a Hugging Face dataset repository and uploads all files in `folder_path` to it.
+    Returns the URL of the uploaded repository.
+    """
+    # Hugging Face Hub token resolution
+    # If not explicitly provided, HfApi will default to HF_TOKEN env variable or local cache
+    api = HfApi(token=token)
+
+    logger.info(f"Authenticating and checking/creating Hugging Face dataset repository: {repo_id}")
+    try:
+        api.create_repo(
+            repo_id=repo_id,
+            repo_type="dataset",
+            private=private,
+            exist_ok=True
+        )
+    except Exception as e:
+        logger.error(f"Failed to create/verify Hugging Face repository {repo_id}: {e}")
+        raise ValueError(f"Hugging Face repository verification failed: {e}")
+
+    logger.info(f"Uploading files from directory {folder_path} to HF dataset repo {repo_id}...")
+    try:
+        api.upload_folder(
+            folder_path=folder_path,
+            repo_id=repo_id,
+            repo_type="dataset",
+        )
+    except Exception as e:
+        logger.error(f"Failed to upload folder {folder_path} to Hugging Face: {e}")
+        raise ValueError(f"Hugging Face upload failed: {e}")
+
+    repo_url = f"https://huggingface.co/datasets/{repo_id}"
+    logger.info(f"Successfully uploaded dataset to: {repo_url}")
+    return repo_url

chapman_datagen-0.1.0/chapman_datagen.egg-info/PKG-INFO

@@ -0,0 +1,21 @@
+Metadata-Version: 2.4
+Name: chapman-datagen
+Version: 0.1.0
+Summary: A highly generalizable prompt-based dataset generator and Hugging Face uploader
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: openai>=1.0.0
+Requires-Dist: pandas>=1.5.0
+Requires-Dist: pyarrow>=10.0.0
+Requires-Dist: huggingface_hub>=0.15.0
+Requires-Dist: tqdm>=4.65.0
+Requires-Dist: jinja2>=3.0.0
+Requires-Dist: datasets>=2.0.0
+
+# chapman-datagen
+
+A highly generalizable prompt-based dataset generator and Hugging Face uploader.

chapman_datagen-0.1.0/chapman_datagen.egg-info/SOURCES.txt

@@ -0,0 +1,15 @@
+README.md
+pyproject.toml
+chapman_datagen/__init__.py
+chapman_datagen/chunker.py
+chapman_datagen/cli.py
+chapman_datagen/config.py
+chapman_datagen/generator.py
+chapman_datagen/pipeline.py
+chapman_datagen/uploader.py
+chapman_datagen.egg-info/PKG-INFO
+chapman_datagen.egg-info/SOURCES.txt
+chapman_datagen.egg-info/dependency_links.txt
+chapman_datagen.egg-info/entry_points.txt
+chapman_datagen.egg-info/requires.txt
+chapman_datagen.egg-info/top_level.txt

chapman_datagen-0.1.0/chapman_datagen.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

chapman_datagen-0.1.0/chapman_datagen.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+chapman-datagen = chapman_datagen.cli:main

chapman_datagen-0.1.0/chapman_datagen.egg-info/requires.txt

@@ -0,0 +1,7 @@
+openai>=1.0.0
+pandas>=1.5.0
+pyarrow>=10.0.0
+huggingface_hub>=0.15.0
+tqdm>=4.65.0
+jinja2>=3.0.0
+datasets>=2.0.0

chapman_datagen-0.1.0/chapman_datagen.egg-info/top_level.txt

@@ -0,0 +1 @@
+chapman_datagen

chapman_datagen-0.1.0/pyproject.toml

@@ -0,0 +1,32 @@
+[build-system]
+requires = ["setuptools>=61.0.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "chapman-datagen"
+version = "0.1.0"
+description = "A highly generalizable prompt-based dataset generator and Hugging Face uploader"
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "MIT" }
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "openai>=1.0.0",
+    "pandas>=1.5.0",
+    "pyarrow>=10.0.0",
+    "huggingface_hub>=0.15.0",
+    "tqdm>=4.65.0",
+    "jinja2>=3.0.0",
+    "datasets>=2.0.0",
+]
+
+[project.scripts]
+chapman-datagen = "chapman_datagen.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["chapman_datagen*"]

chapman_datagen-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+