qmdr 1.0.0

package/finetune/dataset/generate_data_offline.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "datasets",
+# ]
+# ///
+"""
+Generate QMD training data by transforming s-emanuilov/query-expansion dataset
+and adding synthetic hyde passages. No API calls needed.
+"""
+
+import json
+import random
+from pathlib import Path
+
+from dataset.schema import normalize_output_items, parse_output_text
+
+# HyDE passage templates for different query types
+HYDE_TEMPLATES = {
+    "how_to": [
+        "To {action}, you need to {steps}. This can be done by {method}.",
+        "The recommended way to {action} is to first {step1}, then {step2}.",
+        "{Topic} can be achieved by {method}. Make sure to {consideration}.",
+    ],
+    "what_is": [
+        "{Topic} is a {category} that {description}. It is commonly used for {use_case}.",
+        "{Topic} refers to {definition}. Key features include {features}.",
+    ],
+    "config": [
+        "To configure {topic}, set the {setting} option to {value}. You can also customize {other}.",
+        "Configuration for {topic} is done in the {file} file. Key settings include {settings}.",
+    ],
+    "error": [
+        "The {error} error occurs when {cause}. To fix this, {solution}.",
+        "If you encounter {error}, check that {check}. Common solutions include {solutions}.",
+    ],
+    "general": [
+        "{Topic} provides {benefit} for {use_case}. It works by {mechanism}.",
+        "When working with {topic}, consider {considerations}. Best practices include {practices}.",
+    ],
+}
+
+
+def classify_query(query: str) -> str:
+    """Classify query type for hyde template selection."""
+    q = query.lower()
+    if any(
+        w in q for w in ["how to", "how do", "setup", "install", "configure", "create"]
+    ):
+        return "how_to"
+    if any(w in q for w in ["what is", "what are", "definition", "meaning"]):
+        return "what_is"
+    if any(w in q for w in ["config", "setting", "option"]):
+        return "config"
+    if any(w in q for w in ["error", "issue", "problem", "fix", "debug"]):
+        return "error"
+    return "general"
+
+
+def extract_topic(query: str) -> str:
+    """Extract main topic from query."""
+    # Remove common prefixes
+    for prefix in [
+        "how to ",
+        "how do i ",
+        "what is ",
+        "what are ",
+        "configure ",
+        "setup ",
+    ]:
+        if query.lower().startswith(prefix):
+            return query[len(prefix) :].strip()
+    return query
+
+
+def generate_hyde(query: str, expansions: list[str]) -> str:
+    """Generate a hypothetical document passage by combining expansions naturally."""
+    topic = extract_topic(query)
+    query_type = classify_query(query)
+
+    # Use the longest, most descriptive expansion as the base
+    sorted_exp = sorted(expansions, key=len, reverse=True)
+    main_exp = sorted_exp[0] if sorted_exp else topic
+
+    # Build a natural passage based on query type
+    if query_type == "how_to":
+        templates = [
+            f"To {topic}, start by reviewing the requirements and dependencies. {main_exp.capitalize()} is the recommended approach. Make sure all prerequisites are met before proceeding.",
+            f"The process of {topic} involves several steps. First, {main_exp}. Follow the official documentation for detailed instructions.",
+            f"When you need to {topic}, the most effective method is to {main_exp}. This ensures compatibility and follows best practices.",
+        ]
+    elif query_type == "what_is":
+        templates = [
+            f"{topic.capitalize()} refers to {main_exp}. It is widely used in various applications and provides significant benefits.",
+            f"The concept of {topic} encompasses {main_exp}. Understanding this is essential for effective implementation.",
+            f"{topic.capitalize()} is defined as {main_exp}. This plays a crucial role in modern development practices.",
+        ]
+    elif query_type == "config":
+        templates = [
+            f"Configuration for {topic} requires setting the appropriate parameters. {main_exp.capitalize()} should be adjusted based on your specific requirements.",
+            f"To configure {topic}, modify the settings in your configuration file. Key options include those related to {main_exp}.",
+            f"The {topic} configuration can be customized by {main_exp}. Default values work for most use cases.",
+        ]
+    elif query_type == "error":
+        templates = [
+            f"The {topic} issue typically occurs when dependencies are misconfigured. To resolve this, {main_exp}. Check your environment settings.",
+            f"If you encounter problems with {topic}, verify that {main_exp}. Common solutions include updating dependencies and checking permissions.",
+            f"Debugging {topic} requires understanding the root cause. Often, {main_exp} resolves the issue. Review logs for details.",
+        ]
+    else:
+        templates = [
+            f"{topic.capitalize()} is an important concept that relates to {main_exp}. It provides functionality for various use cases in software development.",
+            f"Understanding {topic} is essential for modern development. Key aspects include {main_exp}. This knowledge helps in building robust applications.",
+            f"The topic of {topic} covers {main_exp}. Proper implementation follows established patterns and best practices.",
+        ]
+
+    return random.choice(templates)
+
+
+def transform_to_qmd_format(query: str, expansions: list[str]) -> str:
+    """Transform s-emanuilov format to QMD lex/vec/hyde format."""
+    lines = []
+
+    # Generate hyde line first
+    hyde = generate_hyde(query, expansions)
+    lines.append(f"hyde: {hyde}")
+
+    # Generate lex lines (keyword-focused, shorter)
+    lex_candidates = []
+    for exp in expansions:
+        # Shorter versions for lex
+        words = exp.split()
+        if len(words) <= 4:
+            lex_candidates.append(exp)
+        else:
+            # Take key phrases
+            lex_candidates.append(" ".join(words[:3]))
+
+    # Add 1-2 lex lines
+    for lex in lex_candidates[:2]:
+        if lex.lower() != query.lower():
+            lines.append(f"lex: {lex}")
+
+    # Generate vec lines (semantic, complete phrases)
+    vec_candidates = [exp for exp in expansions if len(exp.split()) >= 3]
+    if not vec_candidates:
+        vec_candidates = expansions
+
+    # Add 1-2 vec lines
+    for vec in vec_candidates[:2]:
+        if vec.lower() != query.lower():
+            lines.append(f"vec: {vec}")
+
+    return "\n".join(lines)
+
+
+def main():
+    try:
+        from datasets import load_dataset
+    except ImportError:
+        print("Installing datasets...")
+        import subprocess
+
+        subprocess.run(["uv", "pip", "install", "datasets"], check=True)
+        from datasets import load_dataset
+
+    print("Loading s-emanuilov/query-expansion dataset...")
+    dataset = load_dataset("s-emanuilov/query-expansion", split="train")
+
+    print(f"Loaded {len(dataset)} examples")
+
+    # Transform each example
+    output_path = Path("data/qmd_expansion.jsonl")
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    examples = []
+    for item in dataset:
+        query = item["query"]
+        expansions = item["expansions"]
+
+        output = transform_to_qmd_format(query, expansions)
+        output_items = normalize_output_items(parse_output_text(output))
+        examples.append({"query": query, "output": output_items})
+
+    # Shuffle
+    random.seed(42)
+    random.shuffle(examples)
+
+    # Write output
+    with open(output_path, "w") as f:
+        for ex in examples:
+            f.write(json.dumps(ex) + "\n")
+
+    print(f"Generated {len(examples)} examples to {output_path}")
+
+    # Show sample
+    print("\nSample output:")
+    print("-" * 50)
+    sample = examples[0]
+    print(f"Input: {sample['query']}")
+    print(f"Output: {sample['output']}")
+
+
+if __name__ == "__main__":
+    main()

package/finetune/dataset/generate_diverse.py

@@ -0,0 +1,441 @@
+#!/usr/bin/env python3
+"""
+Generate diverse QMD training examples for underrepresented categories.
+
+This script creates additional training examples focused on:
+- Trivia, Geography, Philosophy, History (as requested)
+- Temporal/Recency queries (important for evals)
+- Named entity queries (critical for entity preservation scoring)
+"""
+
+import json
+import random
+from pathlib import Path
+from datetime import datetime, timedelta
+
+from dataset.schema import normalize_output_items, parse_output_text
+
+# Additional diverse query categories
+TRIVIA_QUERIES = [
+    "world capitals quiz",
+    "trivia facts about space",
+    "did you know history",
+    "random science facts",
+    "famous inventions timeline",
+    "world records list",
+    "fun geography facts",
+    "historical trivia questions",
+    "animal trivia facts",
+    "sports trivia records",
+]
+
+GEOGRAPHY_QUERIES = [
+    "largest countries by area",
+    "rivers that cross multiple countries",
+    "highest mountain peaks",
+    "desert climate zones",
+    "island nations list",
+    "capital cities europe",
+    "population by continent",
+    "time zones map",
+    "latitude longitude coordinates",
+    "borders between countries",
+    "ocean currents patterns",
+    "tectonic plate boundaries",
+    "climate zones earth",
+]
+
+PHILOSOPHY_QUERIES = [
+    "stoicism daily practice",
+    "existentialism meaning life",
+    "utilitarianism ethics explained",
+    "kant categorical imperative",
+    "free will determinism debate",
+    "nietzsche will to power",
+    "socrates method questioning",
+    "plato theory forms",
+    "aristotle virtue ethics",
+    "descartes cogito ergo sum",
+    "logic propositional calculus",
+    "epistemology knowledge theory",
+    "metaphysics existence reality",
+]
+
+HISTORY_QUERIES = [
+    "ancient civilizations timeline",
+    "roman empire fall reasons",
+    "medieval period events",
+    "renaissance art movement",
+    "industrial revolution inventions",
+    "world war i causes",
+    "cold war key events",
+    "french revolution timeline",
+    "american civil war battles",
+    "egyptian pharaohs dynasty",
+    "bronze age collapse",
+    "byzantine empire history",
+    "vietnam war timeline",
+]
+
+SCIENCE_QUERIES = [
+    "quantum mechanics basics",
+    "theory of relativity explained",
+    "dna structure discovery",
+    "photosynthesis process steps",
+    "black holes physics",
+    "plate tectonics theory",
+    "evolution natural selection",
+    "periodic table elements",
+    "cell biology fundamentals",
+    "climate change evidence",
+]
+
+ARTS_CULTURE_QUERIES = [
+    "impressionist painters list",
+    "shakespeare plays summary",
+    "classical music composers",
+    "modern art movements",
+    "film noir characteristics",
+    "jazz history origins",
+    "renaissance sculpture techniques",
+    "photography composition rules",
+    "poetry forms haiku",
+    "baroque art characteristics",
+    "street art graffiti history",
+]
+
+HEALTH_MEDICINE_QUERIES = [
+    "symptoms of vitamin deficiency",
+    "how vaccines work immune system",
+    "blood pressure normal range",
+    "sleep hygiene tips",
+    "intermittent fasting benefits",
+    "anxiety coping strategies",
+    "stretching exercises back pain",
+    "heart disease prevention",
+    "diabetes type 2 management",
+    "meditation mental health",
+    "nutrition macros explained",
+    "first aid basics",
+]
+
+BUSINESS_FINANCE_QUERIES = [
+    "compound interest calculator",
+    "stock market basics beginners",
+    "startup funding stages",
+    "tax deductions small business",
+    "budgeting methods 50 30 20",
+    "cryptocurrency explained simply",
+    "inflation effects on savings",
+    "retirement planning strategies",
+    "passive income ideas",
+    "venture capital vs angel investors",
+    "balance sheet basics",
+    "supply chain management",
+]
+
+SPORTS_QUERIES = [
+    "marathon training schedule",
+    "weightlifting proper form",
+    "swimming stroke techniques",
+    "tennis serve mechanics",
+    "basketball dribbling drills",
+    "soccer formations tactics",
+    "golf swing fundamentals",
+    "yoga poses beginners",
+    "running injury prevention",
+    "cycling gear ratios",
+    "rock climbing grades",
+    "surfing wave types",
+]
+
+TRAVEL_QUERIES = [
+    "best time visit japan",
+    "travel packing checklist",
+    "budget backpacking europe",
+    "visa requirements usa",
+    "jet lag remedies",
+    "road trip planning tips",
+    "solo travel safety",
+    "airport security rules",
+    "travel insurance coverage",
+    "language apps learning",
+    "hostel vs hotel comparison",
+    "travel photography tips",
+]
+
+FOOD_COOKING_QUERIES = [
+    "bread baking techniques",
+    "knife skills basics",
+    "fermentation at home",
+    "meal prep weekly",
+    "spice combinations guide",
+    "pasta making fresh",
+    "coffee brewing methods",
+    "wine pairing basics",
+    "vegetarian protein sources",
+    "food storage guidelines",
+    "sourdough starter maintenance",
+    "grilling temperature chart",
+]
+
+PSYCHOLOGY_QUERIES = [
+    "cognitive biases list",
+    "attachment theory styles",
+    "maslow hierarchy needs",
+    "growth mindset vs fixed",
+    "emotional intelligence components",
+    "memory techniques mnemonics",
+    "habit formation science",
+    "stress response fight flight",
+    "personality types myers briggs",
+    "motivation intrinsic extrinsic",
+    "decision making psychology",
+    "procrastination causes solutions",
+]
+
+ENVIRONMENT_NATURE_QUERIES = [
+    "renewable energy types",
+    "carbon footprint reduction",
+    "composting basics home",
+    "endangered species list",
+    "recycling symbols meaning",
+    "ocean plastic pollution",
+    "deforestation effects",
+    "sustainable living tips",
+    "wildlife conservation efforts",
+    "solar panel installation",
+    "water conservation methods",
+    "biodiversity importance",
+]
+
+MATH_QUERIES = [
+    "calculus derivatives explained",
+    "probability basics statistics",
+    "linear algebra matrices",
+    "geometry proofs theorems",
+    "logarithms rules properties",
+    "trigonometry identities",
+    "set theory basics",
+    "prime numbers properties",
+    "fractions decimals conversion",
+    "algebra equations solving",
+    "graph theory fundamentals",
+    "combinatorics permutations",
+]
+
+LANGUAGE_QUERIES = [
+    "spanish verb conjugation",
+    "japanese hiragana katakana",
+    "french pronunciation rules",
+    "german cases grammar",
+    "mandarin tones guide",
+    "latin phrases common",
+    "arabic alphabet basics",
+    "english idioms meanings",
+    "sign language basics",
+    "etymology word origins",
+    "grammar punctuation rules",
+    "writing style guides",
+]
+
+DIY_CRAFTS_QUERIES = [
+    "woodworking joints types",
+    "knitting patterns beginners",
+    "home repair basics",
+    "sewing machine threading",
+    "painting techniques acrylic",
+    "pottery wheel basics",
+    "electronics soldering guide",
+    "gardening soil preparation",
+    "candle making supplies",
+    "leather crafting tools",
+    "origami folding instructions",
+    "furniture restoration tips",
+]
+
+# Temporal/Recency queries (matches evals/queries.txt requirements)
+TEMPORAL_TEMPLATES = [
+    "latest {topic} updates",
+    "recent {topic} changes {year}",
+    "what changed in {topic} {year}",
+    "{topic} changelog {year}",
+    "{topic} new features {year}",
+    "{topic} latest version release",
+    "{topic} recent news {month}",
+]
+
+TEMPORAL_TOPICS = [
+    "Shopify",
+    "React",
+    "Kubernetes",
+    "Docker",
+    "TypeScript",
+    "Python",
+    "AWS",
+    "GitHub",
+    "Next.js",
+    "Vue",
+    "AI",
+    "machine learning",
+    "climate tech",
+    "space exploration",
+]
+
+# Named entity queries (critical for entity preservation testing)
+NAMED_ENTITY_QUERIES = [
+    "who is TDS motorsports",
+    "React hooks tutorial",
+    "Docker container networking",
+    "Kubernetes pod deployment",
+    "AWS Lambda functions setup",
+    "Stripe payment integration",
+    "GitHub Actions workflow",
+    "Vercel deployment guide",
+    "Supabase auth configuration",
+    "Twilio SMS API",
+    "Datadog monitoring setup",
+    "Sentry error tracking",
+    "Terraform AWS provider",
+    "Ansible playbook examples",
+]
+
+
+# Generate temporal queries with recent dates
+def generate_temporal_queries():
+    queries = []
+    current_year = datetime.now().year
+    months = [
+        "January",
+        "February",
+        "March",
+        "April",
+        "May",
+        "June",
+        "July",
+        "August",
+        "September",
+        "October",
+        "November",
+        "December",
+    ]
+
+    for template in TEMPORAL_TEMPLATES:
+        for topic in TEMPORAL_TOPICS:
+            if "{year}" in template:
+                # Use current year and previous year
+                for year in [current_year, current_year - 1]:
+                    queries.append(template.format(topic=topic, year=year))
+            elif "{month}" in template:
+                # Use recent months
+                for month in months[-3:]:  # Last 3 months
+                    queries.append(template.format(topic=topic, month=month))
+            else:
+                queries.append(template.format(topic=topic))
+
+    return list(set(queries))  # Remove duplicates
+
+
+def generate_expansion(query: str) -> str:
+    """Generate a realistic expansion for a query."""
+    # This is a template-based generator - in production, use Claude API
+    lex_variations = [
+        f"{query} guide",
+        f"{query} documentation",
+        f"{query} tutorial",
+        f"{query} examples",
+        f"{query} best practices",
+    ]
+
+    vec_variations = [
+        f"how to {query}",
+        f"guide for {query}",
+        f"learn about {query}",
+        f"understanding {query}",
+        f"complete {query} reference",
+    ]
+
+    # Select 2-3 lex and 2 vec variations
+    selected_lex = random.sample(lex_variations, min(3, len(lex_variations)))
+    selected_vec = random.sample(vec_variations, min(2, len(vec_variations)))
+
+    # Generate hyde passage
+    hyde = f"This comprehensive guide covers everything you need to know about {query}. It includes practical examples, best practices, and troubleshooting tips for beginners and advanced users alike."
+
+    output_lines = []
+    for lex in selected_lex:
+        output_lines.append(f"lex: {lex}")
+    for vec in selected_vec:
+        output_lines.append(f"vec: {vec}")
+    output_lines.append(f"hyde: {hyde}")
+
+    return "\n".join(output_lines)
+
+
+def main():
+    """Generate diverse examples and append to training data."""
+    output_file = Path("data/qmd_expansion_diverse_addon.jsonl")
+
+    all_queries = (
+        TRIVIA_QUERIES
+        + GEOGRAPHY_QUERIES
+        + PHILOSOPHY_QUERIES
+        + HISTORY_QUERIES
+        + SCIENCE_QUERIES
+        + ARTS_CULTURE_QUERIES
+        + HEALTH_MEDICINE_QUERIES
+        + BUSINESS_FINANCE_QUERIES
+        + SPORTS_QUERIES
+        + TRAVEL_QUERIES
+        + FOOD_COOKING_QUERIES
+        + PSYCHOLOGY_QUERIES
+        + ENVIRONMENT_NATURE_QUERIES
+        + MATH_QUERIES
+        + LANGUAGE_QUERIES
+        + DIY_CRAFTS_QUERIES
+        + generate_temporal_queries()
+        + NAMED_ENTITY_QUERIES
+    )
+
+    print(f"Generating {len(all_queries)} diverse training examples...")
+    print(f"  - Trivia: {len(TRIVIA_QUERIES)}")
+    print(f"  - Geography: {len(GEOGRAPHY_QUERIES)}")
+    print(f"  - Philosophy: {len(PHILOSOPHY_QUERIES)}")
+    print(f"  - History: {len(HISTORY_QUERIES)}")
+    print(f"  - Science: {len(SCIENCE_QUERIES)}")
+    print(f"  - Arts/Culture: {len(ARTS_CULTURE_QUERIES)}")
+    print(f"  - Health/Medicine: {len(HEALTH_MEDICINE_QUERIES)}")
+    print(f"  - Business/Finance: {len(BUSINESS_FINANCE_QUERIES)}")
+    print(f"  - Sports: {len(SPORTS_QUERIES)}")
+    print(f"  - Travel: {len(TRAVEL_QUERIES)}")
+    print(f"  - Food/Cooking: {len(FOOD_COOKING_QUERIES)}")
+    print(f"  - Psychology: {len(PSYCHOLOGY_QUERIES)}")
+    print(f"  - Environment: {len(ENVIRONMENT_NATURE_QUERIES)}")
+    print(f"  - Math: {len(MATH_QUERIES)}")
+    print(f"  - Language: {len(LANGUAGE_QUERIES)}")
+    print(f"  - DIY/Crafts: {len(DIY_CRAFTS_QUERIES)}")
+    print(f"  - Temporal: {len(generate_temporal_queries())}")
+    print(f"  - Named Entities: {len(NAMED_ENTITY_QUERIES)}")
+
+    examples = []
+    for query in all_queries:
+        expansion = generate_expansion(query)
+        output_items = normalize_output_items(parse_output_text(expansion))
+        examples.append(
+            {"query": query, "output": output_items, "category": "diverse_addon"}
+        )
+
+    # Write to file
+    output_file.parent.mkdir(parents=True, exist_ok=True)
+    with open(output_file, "w") as f:
+        for ex in examples:
+            f.write(json.dumps(ex) + "\n")
+
+    print(f"\nSaved {len(examples)} diverse examples to {output_file}")
+    print("\nTo use these examples:")
+    print(f"  cat {output_file} >> data/qmd_expansion_v2.jsonl")
+    print("  uv run dataset/prepare_data.py --add-short 2")
+
+
+if __name__ == "__main__":
+    main()