fc-data 0.2.0__py3-none-any.whl

datasmith/github/render.py

@@ -0,0 +1,173 @@
+"""Problem statement rendering with Jinja2 templates."""
+
+from __future__ import annotations
+
+import os
+import re
+
+from jinja2 import Environment, FileSystemLoader
+
+from datasmith.github.models import PR, IssueExpanded
+from datasmith.utils import get_logger
+
+logger = get_logger("github.render")
+
+_TEMPLATES_DIR = os.path.join(os.path.dirname(__file__), "templates")
+
+
+def _get_env() -> Environment:
+    """Return a Jinja2 environment pointing at the templates directory."""
+    return Environment(
+        loader=FileSystemLoader(_TEMPLATES_DIR),
+        autoescape=False,
+        trim_blocks=True,
+        lstrip_blocks=True,
+    )
+
+
+class Anonymizer:
+    """Replace usernames, emails, and other PII with deterministic placeholders."""
+
+    _SIGNOFF = re.compile(
+        r"((?:Signed-off-by|Co-authored-by)\s*:\s*)(.*?)(\s*<[^>]+>)",
+        re.IGNORECASE,
+    )
+    _EMAIL = re.compile(r"[\w.+-]+@[\w-]+\.[\w.]+")
+    _MENTION = re.compile(r"@([\w-]+)")
+    _GITHUB_URL = re.compile(r"(https?://github\.com/)([\w.-]+)")
+    _USER_IMAGES = re.compile(r"https?://user-images\.githubusercontent\.com/[^\s)>\]]+")
+    _HOME_PATH = re.compile(r"(/(?:home|Users)/|[Cc]:\\Users\\)([\w.-]+)")
+    _IP_ADDR = re.compile(r"\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b")
+
+    def __init__(self, known_usernames: set[str] | None = None) -> None:
+        self._map: dict[str, str] = {}
+        self._counter = 0
+        self._known_usernames = known_usernames or set()
+
+    def _placeholder(self, username: str) -> str:
+        """Return a deterministic placeholder for *username* (case-insensitive)."""
+        key = username.lower()
+        if key not in self._map:
+            self._counter += 1
+            self._map[key] = f"user_{self._counter}"
+        return self._map[key]
+
+    def anonymize(self, text: str) -> str:
+        """Strip PII and replace identifiers with deterministic placeholders.
+
+        Processing order:
+
+        1. Signed-off-by / Co-authored-by lines (before email stripping)
+        2. Email addresses → ``[email]``
+        3. ``@user`` mentions → ``@user_N``
+        4. GitHub profile/repo URLs (username segment)
+        5. ``user-images.githubusercontent.com`` URLs → ``[image]``
+        6. Home-directory paths (``/home/x``, ``/Users/x``, ``C:\\Users\\x``)
+        7. IP addresses → ``[ip]``
+        8. Known bare usernames (word-boundary, case-insensitive)
+        """
+        # 1. Sign-off lines (must precede email stripping so we can match the <email>)
+        text = self._SIGNOFF.sub(lambda m: f"{m.group(1)}[name] <[email]>", text)
+
+        # 2. Emails
+        text = self._EMAIL.sub("[email]", text)
+
+        # 3. @mentions
+        text = self._MENTION.sub(lambda m: f"@{self._placeholder(m.group(1))}", text)
+
+        # 4. GitHub user/org URLs
+        text = self._GITHUB_URL.sub(lambda m: f"{m.group(1)}{self._placeholder(m.group(2))}", text)
+
+        # 5. User-uploaded image URLs
+        text = self._USER_IMAGES.sub("[image]", text)
+
+        # 6. Home-directory paths
+        text = self._HOME_PATH.sub(lambda m: f"{m.group(1)}{self._placeholder(m.group(2))}", text)
+
+        # 7. IP addresses
+        text = self._IP_ADDR.sub("[ip]", text)
+
+        # 8. Known bare usernames (longest first to avoid partial replacement)
+        for uname in sorted(self._known_usernames, key=len, reverse=True):
+            placeholder = self._placeholder(uname)
+            text = re.sub(rf"\b{re.escape(uname)}\b", placeholder, text, flags=re.IGNORECASE)
+
+        return text
+
+
+def render_problem_statement(
+    pr: PR,
+    issues: list[IssueExpanded] | None = None,
+    repo_description: str = "",
+    anonymize: bool = False,
+    known_usernames: set[str] | None = None,
+    extract: bool = True,
+    initial_observations: str | None = None,
+) -> str:
+    """Render the full problem statement for a FormulaCode task.
+
+    Parameters
+    ----------
+    pr:
+        The pull request providing the initial observations.
+    issues:
+        Optional list of linked issues to include.
+    repo_description:
+        A short description of the repository.
+    anonymize:
+        If ``True``, replace usernames and emails with placeholders.
+    known_usernames:
+        Additional usernames to scrub even without an ``@`` prefix.
+        Only used when *anonymize* is ``True``.
+    extract:
+        If ``True`` (default), use :class:`ProblemExtractor` to separate
+        problem observations from solution details, preventing information
+        leakage.  Falls back to raw ``pr.body`` on failure.
+        Ignored when *initial_observations* is provided.
+    initial_observations:
+        If provided, use this text directly as the problem observations and
+        skip the :class:`ProblemExtractor` call entirely.  Pass the output of
+        ``ProblemExtraction.to_problem_markdown()`` here when the extraction
+        has already been performed outside this function.
+    """
+    env = _get_env()
+    anon = Anonymizer(known_usernames=known_usernames) if anonymize else None
+
+    # Extract problem observations (strip solution details) or use raw body
+    if initial_observations is not None:
+        # Caller already ran ProblemExtractor — use the supplied text directly.
+        pass
+    elif extract:
+        try:
+            from datasmith.agents.extractors import ProblemExtractor
+
+            extractor = ProblemExtractor()
+            extraction = extractor.extract_problem(
+                pr_title=getattr(pr, "title", ""),
+                pr_body=getattr(pr, "body", ""),
+            )
+            initial_observations = extraction.to_problem_markdown() or getattr(pr, "body", "")
+        except Exception:
+            logger.warning("ProblemExtractor failed, falling back to raw PR body")
+            initial_observations = getattr(pr, "body", "")
+    else:
+        initial_observations = getattr(pr, "body", "")
+
+    # Render issues section
+    issues_text = ""
+    if issues:
+        tpl = env.get_template("issues.md.j2")
+        issues_text = tpl.render(issues=issues)
+
+    # Render final
+    tpl = env.get_template("final.md.j2")
+    rendered = tpl.render(
+        repo_description=repo_description,
+        initial_observations=initial_observations,
+        issues=issues_text,
+    )
+
+    if anon:
+        rendered = anon.anonymize(rendered)
+
+    return rendered

datasmith/github/search.py

@@ -0,0 +1,66 @@
+"""GitHub Code Search helpers for repo discovery."""
+
+from __future__ import annotations
+
+from datasmith.utils import get_logger
+
+logger = get_logger("github.search")
+
+
+async def search_repos_by_file(
+    gh: object,
+    filename: str = "asv.conf.json",
+    min_stars: int = 100,
+) -> list[tuple[str, str]]:
+    """Discover repos containing *filename* via the GitHub Code Search API.
+
+    Returns a deduplicated list of ``(owner, repo)`` tuples for repos that are
+    not forks, not archived, and have at least *min_stars* stars.
+    """
+    from datasmith.github.client import GitHubClient
+
+    if not isinstance(gh, GitHubClient):
+        raise TypeError(f"Expected GitHubClient, got {type(gh).__name__}")
+
+    seen: set[str] = set()
+    results: list[tuple[str, str]] = []
+
+    async for item in gh.search_code(f"filename:{filename}"):
+        repo_data = item.get("repository", {})
+        full_name: str = repo_data.get("full_name", "")
+        if not full_name or full_name in seen:
+            continue
+        seen.add(full_name)
+
+        # Skip forks (available in search response)
+        if repo_data.get("fork", False):
+            continue
+
+        # Skip the ASV tool itself
+        if full_name == "airspeed-velocity/asv":
+            continue
+
+        owner, repo = full_name.split("/", 1)
+        results.append((owner, repo))
+
+    logger.info("Code search found %d candidate repos for filename:%s", len(results), filename)
+
+    # Fetch full metadata to filter by archived/disabled/stars
+    filtered: list[tuple[str, str]] = []
+    for owner, repo in results:
+        resp = await gh._request("GET", f"/repos/{owner}/{repo}")
+        if resp is None:
+            continue
+        data = resp.json()
+        if data.get("archived", False) or data.get("disabled", False):
+            logger.debug("%s/%s: archived/disabled — skipped", owner, repo)
+            continue
+        stars = data.get("stargazers_count", 0)
+        if stars < min_stars:
+            logger.debug("%s/%s: %d stars — below threshold", owner, repo, stars)
+            continue
+        filtered.append((owner, repo))
+        logger.debug("%s/%s: %d stars — included", owner, repo, stars)
+
+    logger.info("Filtered to %d repos with >= %d stars", len(filtered), min_stars)
+    return filtered

datasmith/github/templates/comment.md.j2

@@ -0,0 +1,5 @@
+**@{{ user_login }}** — {{ timestamp }}
+
+{{ body }}
+
+Links mentioned: {{ links_str }}

datasmith/github/templates/final.md.j2

@@ -0,0 +1,66 @@
+**Objective:**
+You are a performance optimization expert. Speed up the repository **while maintaining correctness**.
+
+**Tooling:**
+The micromamba environment includes **Pytest** for testing and **Airspeed Velocity (ASV)** for benchmarking measurements and profiling.
+
+**Process:**
+
+**1. Scan & Baseline**
+
+- Read the code and any hints.
+- Map likely bottlenecks.
+- Establish a **baseline** by running the **relevant** ASV benchmarks.
+
+**2. Benchmark (ASV)**
+- Read through relevant benchmarks.
+- Prefer targeted runs using `--bench=<regex>`; full-suite runs are too time-consuming and are discouraged.
+- Commands:
+
+  ```bash
+  # Always pin to current interpreter
+  asv run --python=same --bench="<regex>"
+  ```
+
+* Find benchmarks via `asv_benchmarks.txt` or in the directory containing the ASV benchmarks.
+* You may run multiple benchmarks at once using regexes.
+
+**3. Profile Hotspots**
+
+* Profile **relevant** benchmarks to locate hot paths.
+* Use ASV's built-in profiling support.
+
+  ```bash
+  asv profile --python=same --config=<path-to-asv.*.json> <benchmark_name>
+  ```
+
+**4. Optimize**
+
+* Make targeted changes that address the hot paths while maintaining correctness.
+* Always follow the **Operating Principles** below.
+
+**Operating Principles**
+
+* **One change/command at a time** (code edit, ASV run, profiling).
+* **Baseline first**, then iterate.
+* **Target the hot paths** shown by profiling.
+* **Evidence-driven**: justify changes with benchmark/profile data.
+* **Correctness first**: never trade correctness for speed.
+{% if repo_description %}
+
+**Repository Description**
+
+{{ repo_description }}
+{% endif %}
+
+**Task Description**
+
+Your main goal is to optimize the code to run as fast as possible. Use the following information if needed to understand the problem:
+
+{{ initial_observations }}
+
+{% if issues %}
+**Relevant Issues**
+
+{{ issues }}
+{% endif %}

datasmith/github/templates/issues.md.j2

@@ -0,0 +1,21 @@
+{% for it in issues %}
+# Issue #{{ it.number }}: {{ it.title }}
+{% if it.description %}
+Description:
+{{ it.description }}
+{% endif %}
+
+{% if it.comments %}
+Comments:
+{% for c in it.comments %}
+ - {{ c }}
+{% endfor %}
+{% endif %}
+
+{% if it.cross_references %}
+Cross-references:
+{% for x in it.cross_references %}
+ - {{ x }}
+{% endfor %}
+{% endif %}
+{% endfor %}

datasmith/github/templates/repo.md.j2

@@ -0,0 +1 @@
+This repository is called {{ repo_name }}. {{ repo_name }} is written primarily in {{ repo_language }} and is described as a "{{ repo_description }}"{% if repo_topics %}, with topics including {{ repo_topics }}{% endif %}.

datasmith/preflight.py

@@ -0,0 +1,162 @@
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+
+from datasmith.utils import get_logger
+
+logger = get_logger("preflight")
+
+
+def _check(name: str, condition: bool, detail: str = "") -> bool:
+    status = "OK" if condition else "FAIL"
+    msg = f"  [{status}] {name}"
+    if detail:
+        msg += f" — {detail}"
+    print(msg)
+    return condition
+
+
+def run_preflight() -> bool:  # noqa: C901
+    """Verify all prerequisites for the DataSmith pipeline."""
+    print("DataSmith Preflight Check")
+    print("=" * 40)
+
+    all_ok = True
+
+    # 1. Environment variables
+    print("\n== Environment ==")
+    supabase_url = os.environ.get("SUPABASE_URL", "")
+    all_ok &= _check(
+        "SUPABASE_URL", bool(supabase_url), supabase_url[:30] + "..." if len(supabase_url) > 30 else supabase_url
+    )
+
+    supabase_key = os.environ.get("SUPABASE_KEY", "")
+    all_ok &= _check("SUPABASE_KEY", bool(supabase_key), "***" if supabase_key else "")
+
+    gh_tokens = os.environ.get("GH_TOKENS", os.environ.get("GH_TOKEN", ""))
+    token_count = len([t for t in gh_tokens.split(",") if t.strip()]) if gh_tokens else 0
+    all_ok &= _check("GH_TOKENS", token_count > 0, f"{token_count} token(s)")
+
+    hf_token_path = os.environ.get("HF_TOKEN_PATH", "/mnt/sdd3/llama_atharvas/huggingface/token")
+    hf_exists = Path(hf_token_path).exists()
+    all_ok &= _check("HF_TOKEN", hf_exists, hf_token_path)
+
+    # 2. LLM Backend
+    print("\n== LLM Backend ==")
+    dspy_model = os.environ.get("DSPY_MODEL", "openai/gpt-oss-120b")
+    dspy_api_base = os.environ.get("DSPY_API_BASE", "http://localhost:30001/v1")
+    all_ok &= _check("DSPY_MODEL", bool(dspy_model), dspy_model)
+    all_ok &= _check("DSPY_API_BASE", bool(dspy_api_base), dspy_api_base)
+
+    if dspy_model and dspy_api_base:
+        try:
+            import httpx
+
+            # Strip "openai/" prefix for model matching
+            model_id = dspy_model.removeprefix("openai/")
+            models_url = f"{dspy_api_base}/models"
+            resp = httpx.get(models_url, timeout=10)
+            resp.raise_for_status()
+            models_data = resp.json().get("data", [])
+            model_ids = [m.get("id", "") for m in models_data]
+            found = model_id in model_ids
+            all_ok &= _check("Model available", found, f"{model_id} in {len(model_ids)} model(s)")
+
+            if found:
+                # Test completion
+                chat_url = f"{dspy_api_base}/chat/completions"
+                payload = {
+                    "model": model_id,
+                    "messages": [{"role": "user", "content": "Say OK"}],
+                    "max_tokens": 64,
+                }
+                comp_resp = httpx.post(chat_url, json=payload, timeout=60)
+                comp_resp.raise_for_status()
+                choices = comp_resp.json().get("choices", [])
+                content = ""
+                if choices:
+                    msg = choices[0].get("message") or {}
+                    if isinstance(msg, dict):
+                        content = msg.get("content") or msg.get("reasoning") or ""
+                    finish = choices[0].get("finish_reason", "")
+                else:
+                    finish = ""
+                detail = repr(content[:40]) if content else f"finish_reason={finish}"
+                all_ok &= _check("Test completion", True, detail)
+        except Exception as e:
+            all_ok &= _check("LLM server", False, str(e)[:80])
+
+    # 3. Supabase connection
+    print("\n== Supabase ==")
+    try:
+        import datasmith.utils.db as db_mod
+        from datasmith.utils.db import get_client
+
+        db_mod._client = None  # Force fresh connection
+        client = get_client()
+        # Try a simple query
+        client.table("repositories").select("owner").limit(1).execute()
+        all_ok &= _check("Connection", True)
+        db_mod._client = None
+    except Exception as e:
+        all_ok &= _check("Connection", False, str(e)[:80])
+
+    # 4. Docker
+    print("\n== Docker ==")
+    try:
+        from python_on_whales import DockerClient
+
+        docker = DockerClient()
+        docker.version()
+        all_ok &= _check("Docker daemon", True)
+    except Exception as e:
+        all_ok &= _check("Docker daemon", False, str(e)[:80])
+
+    # 5. GitHub tokens
+    print("\n== GitHub ==")
+    if token_count > 0:
+        try:
+            import httpx
+
+            total_remaining = 0
+            failed_tokens = 0
+            tokens = [t.strip() for t in gh_tokens.split(",") if t.strip()]
+            for token in tokens:
+                resp = httpx.get(
+                    "https://api.github.com/rate_limit",
+                    headers={"Authorization": f"Bearer {token}"},
+                    timeout=10,
+                )
+                if resp.status_code == 200:
+                    total_remaining += resp.json()["rate"]["remaining"]
+                else:
+                    failed_tokens += 1
+
+            if failed_tokens == len(tokens):
+                all_ok &= _check("API access", False, "all tokens failed")
+            else:
+                detail = f"remaining={total_remaining} across {len(tokens) - failed_tokens}/{len(tokens)} token(s)"
+                all_ok &= _check("API access", True, detail)
+        except Exception as e:
+            all_ok &= _check("API access", False, str(e)[:80])
+    else:
+        all_ok &= _check("API access", False, "no tokens")
+
+    # Summary
+    print("\n" + "=" * 40)
+    if all_ok:
+        print("All checks passed!")
+    else:
+        print("Some checks failed. Fix issues above before running the pipeline.")
+
+    return all_ok
+
+
+if __name__ == "__main__":
+    from datasmith import setup_environment
+
+    setup_environment()
+    ok = run_preflight()
+    sys.exit(0 if ok else 1)

datasmith/publish/__init__.py

@@ -0,0 +1,13 @@
+"""ds.publish — DockerHub + HuggingFace publishing pipeline."""
+
+from datasmith.publish.huggingface import HuggingFacePublisher
+from datasmith.publish.pipeline import publish_pipeline
+from datasmith.publish.records import records_from_parquet, records_from_supabase, records_to_parquet
+
+__all__ = [
+    "HuggingFacePublisher",
+    "publish_pipeline",
+    "records_from_parquet",
+    "records_from_supabase",
+    "records_to_parquet",
+]

datasmith/publish/huggingface.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+import os
+import tempfile
+from pathlib import Path
+
+from datasmith.github.models import FormulaCodeRecord
+from datasmith.publish.records import records_to_parquet
+from datasmith.utils import get_logger
+
+logger = get_logger("publish.huggingface")
+
+
+class HuggingFacePublisher:
+    """Publish FormulaCode records to HuggingFace as versioned Parquet datasets."""
+
+    def __init__(
+        self,
+        repo_id: str = "formulacode/formulacode",
+        token_path: str = "",
+    ) -> None:
+        self._repo_id = repo_id
+        self._token_path = token_path or os.environ.get("HF_TOKEN_PATH", "/mnt/sdd3/llama_atharvas/huggingface/token")
+        self._token: str | None = None
+
+    def _get_token(self) -> str:
+        if self._token is None:
+            path = Path(self._token_path)
+            if path.exists():
+                self._token = path.read_text().strip()
+            else:
+                self._token = os.environ.get("HF_TOKEN", "")
+            if not self._token:
+                raise ValueError(f"HuggingFace token not found at {self._token_path} or HF_TOKEN env var")
+        return self._token
+
+    def publish(self, records: list[FormulaCodeRecord], version: str) -> None:
+        """Upload records as Parquet to HuggingFace Hub."""
+        from huggingface_hub import HfApi
+
+        token = self._get_token()
+        parquet_bytes = records_to_parquet(records)
+        if not parquet_bytes:
+            logger.warning("No records to publish")
+            return
+
+        api = HfApi(token=token)
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            parquet_path = os.path.join(tmpdir, f"{version}.parquet")
+            with open(parquet_path, "wb") as f:
+                f.write(parquet_bytes)
+
+            api.upload_file(
+                path_or_fileobj=parquet_path,
+                path_in_repo=f"data/{version}.parquet",
+                repo_id=self._repo_id,
+                repo_type="dataset",
+                commit_message=f"Add {version} data ({len(records)} records)",
+            )
+
+        logger.info("Published %d records as %s to %s", len(records), version, self._repo_id)
+
+    def create_dataset_card(self, version: str) -> str:
+        """Generate a YAML dataset card string."""
+        card = f"""---
+language:
+- en
+license: apache-2.0
+tags:
+- code
+- benchmarks
+- performance
+- optimization
+pretty_name: FormulaCode
+size_categories:
+- 1K<n<10K
+---
+
+# FormulaCode Dataset
+
+Performance optimization benchmark dataset.
+
+## Version: {version}
+
+## Schema
+
+| Field | Type | Description |
+|-------|------|-------------|
+| task_id | string | Unique task identifier (owner__repo-issue_number) |
+| owner | string | Repository owner |
+| repo | string | Repository name |
+| issue_number | int | PR number |
+| gt_hash | string | Ground truth merge commit SHA |
+| base_commit | string | Base commit SHA |
+| date | string | Merge date |
+| instructions | string | Task instructions / problem statement |
+| classification | string | Optimization type category |
+| difficulty | string | easy / medium / hard |
+| container_name | string | Docker container name |
+| patch | string | Ground truth patch |
+| image_name | string | Docker image name |
+"""
+        return card

datasmith/publish/pipeline.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from datasmith.publish.huggingface import HuggingFacePublisher
+from datasmith.publish.records import records_from_supabase
+from datasmith.utils import get_client, get_logger
+
+logger = get_logger("publish.pipeline")
+
+
+async def publish_pipeline(
+    start_date: str,
+    end_date: str,
+    dockerhub_push: bool = True,
+    hf_publish: bool = True,
+) -> int:
+    """Full publishing pipeline: query DB -> push DockerHub -> upload HuggingFace -> mark published.
+
+    Returns the number of records published.
+    """
+    records = records_from_supabase(start_date=start_date, end_date=end_date)
+    if not records:
+        logger.info("No unpublished records found for %s to %s", start_date, end_date)
+        return 0
+
+    logger.info("Found %d unpublished records", len(records))
+
+    version = f"formulacode@{datetime.now(tz=timezone.utc).strftime('%Y-%m')}"
+
+    # DockerHub push (optional)
+    if dockerhub_push:
+        from datasmith.docker.publish import DockerHubPublisher
+
+        publisher = DockerHubPublisher()
+        for record in records:
+            if record.container_name:
+                try:
+                    publisher.push(record.container_name)
+                except Exception:
+                    logger.warning("Failed to push %s", record.container_name)
+
+    # HuggingFace publish (optional)
+    if hf_publish:
+        hf = HuggingFacePublisher()
+        hf.publish(records, version)
+
+    # Mark as published in Supabase
+    client = get_client()
+    now = datetime.now(tz=timezone.utc).isoformat()
+    for record in records:
+        try:
+            client.table("pull_requests").update({"published_at": now}).eq("owner", record.owner).eq(
+                "repo", record.repo
+            ).eq("issue_number", record.issue_number).execute()
+        except Exception:
+            logger.warning("Failed to mark %s as published", record.task_id)
+
+    logger.info("Published %d records as %s", len(records), version)
+    return len(records)