devpost-scraper 0.1.0__tar.gz

devpost_scraper-0.1.0/.env.example

@@ -0,0 +1,13 @@
+# Backboard API key — get yours at https://app.backboard.io
+BACKBOARD_API_KEY=
+
+# Persisted automatically after first run — do not delete
+DEVPOST_ASSISTANT_ID=
+
+# Devpost session cookie (_devpost) for authenticated endpoints (e.g. /participants)
+# Copy the _devpost cookie value from your browser DevTools after logging in
+DEVPOST_SESSION=
+
+# GitHub personal access token for higher API rate limits (5000/hr vs 60/hr)
+# Generate at https://github.com/settings/tokens (no scopes needed, public data only)
+GITHUB_TOKEN=

devpost_scraper-0.1.0/.gitignore

@@ -0,0 +1,11 @@
+.env
+*.csv
+.cursor
+backboard*.md
+uv.lock
+start.sh
+.venv/
+__pycache__/
+.backboard/
+dist/
+*.egg-info/

devpost_scraper-0.1.0/PKG-INFO

@@ -0,0 +1,101 @@
+Metadata-Version: 2.4
+Name: devpost-scraper
+Version: 0.1.0
+Summary: CLI for extracting Devpost data with Backboard tool-calling and exporting results to CSV.
+Requires-Python: >=3.11
+Requires-Dist: backboard-sdk>=1.5.9
+Requires-Dist: beautifulsoup4>=4.12.0
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: pydantic>=2.7.0
+Requires-Dist: python-dotenv>=1.0.1
+Description-Content-Type: text/markdown
+
+# Devpost Scraper
+
+CLI for extracting Devpost project data with a Backboard assistant that can call a Devpost MCP tool server and export structured results to CSV.
+
+## Requirements
+
+- Python 3.11+
+- `uv`
+- Node.js / `npx` available on your machine
+- A Backboard API key
+
+## Environment
+
+Create a `.env` file from `.env.example` and set:
+
+- `BACKBOARD_API_KEY`
+- `BACKBOARD_MODEL` (optional)
+- `DEVPOST_ASSISTANT_NAME` (optional)
+
+## MCP server
+
+This project is designed to use a Devpost MCP server with this configuration:
+
+```json
+{
+  "mcpServers": {
+    "devpost": {
+      "command": "npx",
+      "args": ["devpost-mcp-server"]
+    }
+  }
+}
+```
+
+## Install
+
+```bash
+uv sync
+```
+
+## Run
+
+```bash
+uv run devpost-scraper "ai agents" --output ai_agents.csv
+uv run devpost-scraper "developer tools" "climate tech" --output results.csv
+```
+
+You can also use the startup script:
+
+```bash
+./start.sh "ai agents" --output ai_agents.csv
+```
+
+## What it does
+
+1. Creates or reuses a Backboard assistant configured for Devpost extraction.
+2. Creates a thread for the run.
+3. Sends a prompt that asks the assistant to use the Devpost MCP toolset.
+4. Handles tool-calling loops until the assistant returns completed structured content.
+5. Parses the structured JSON result.
+6. Writes the extracted rows to CSV.
+
+## Expected output shape
+
+Each extracted row should contain fields like:
+
+- `search_term`
+- `project_title`
+- `tagline`
+- `project_url`
+- `hackathon_name`
+- `hackathon_url`
+- `summary`
+- `built_with`
+- `prizes`
+- `submission_date`
+- `team_size`
+
+## Notes
+
+- The CLI is intentionally API-heavy and UI-free.
+- The Backboard assistant must have access to the Devpost MCP tools in the environment where it runs.
+- If your Backboard account or environment requires additional tool registration, wire that into the assistant creation flow in the client module.
+
+## Development
+
+```bash
+uv run python -m devpost_scraper.cli "ai agents" --output out.csv
+```

devpost_scraper-0.1.0/README.md

@@ -0,0 +1,89 @@
+# Devpost Scraper
+
+CLI for extracting Devpost project data with a Backboard assistant that can call a Devpost MCP tool server and export structured results to CSV.
+
+## Requirements
+
+- Python 3.11+
+- `uv`
+- Node.js / `npx` available on your machine
+- A Backboard API key
+
+## Environment
+
+Create a `.env` file from `.env.example` and set:
+
+- `BACKBOARD_API_KEY`
+- `BACKBOARD_MODEL` (optional)
+- `DEVPOST_ASSISTANT_NAME` (optional)
+
+## MCP server
+
+This project is designed to use a Devpost MCP server with this configuration:
+
+```json
+{
+  "mcpServers": {
+    "devpost": {
+      "command": "npx",
+      "args": ["devpost-mcp-server"]
+    }
+  }
+}
+```
+
+## Install
+
+```bash
+uv sync
+```
+
+## Run
+
+```bash
+uv run devpost-scraper "ai agents" --output ai_agents.csv
+uv run devpost-scraper "developer tools" "climate tech" --output results.csv
+```
+
+You can also use the startup script:
+
+```bash
+./start.sh "ai agents" --output ai_agents.csv
+```
+
+## What it does
+
+1. Creates or reuses a Backboard assistant configured for Devpost extraction.
+2. Creates a thread for the run.
+3. Sends a prompt that asks the assistant to use the Devpost MCP toolset.
+4. Handles tool-calling loops until the assistant returns completed structured content.
+5. Parses the structured JSON result.
+6. Writes the extracted rows to CSV.
+
+## Expected output shape
+
+Each extracted row should contain fields like:
+
+- `search_term`
+- `project_title`
+- `tagline`
+- `project_url`
+- `hackathon_name`
+- `hackathon_url`
+- `summary`
+- `built_with`
+- `prizes`
+- `submission_date`
+- `team_size`
+
+## Notes
+
+- The CLI is intentionally API-heavy and UI-free.
+- The Backboard assistant must have access to the Devpost MCP tools in the environment where it runs.
+- If your Backboard account or environment requires additional tool registration, wire that into the assistant creation flow in the client module.
+
+## Development
+
+```bash
+uv run python -m devpost_scraper.cli "ai agents" --output out.csv
+```

devpost_scraper-0.1.0/pyproject.toml

@@ -0,0 +1,27 @@
+[project]
+name = "devpost-scraper"
+version = "0.1.0"
+description = "CLI for extracting Devpost data with Backboard tool-calling and exporting results to CSV."
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+  "backboard-sdk>=1.5.9",
+  "beautifulsoup4>=4.12.0",
+  "httpx>=0.27.0",
+  "pydantic>=2.7.0",
+  "python-dotenv>=1.0.1",
+]
+
+[project.scripts]
+devpost-scraper = "devpost_scraper.cli:main"
+devpost-participants = "devpost_scraper.cli:participants_main"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/devpost_scraper"]
+
+[tool.uv]
+package = true

devpost_scraper-0.1.0/src/devpost_scraper/backboard_client.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+import json
+import os
+from typing import Any, Awaitable, Callable, Mapping
+
+from backboard import BackboardClient
+from backboard.exceptions import BackboardAPIError
+
+ToolHandler = Callable[[dict[str, Any]], Awaitable[dict[str, Any]]]
+
+
+class BackboardClientError(Exception):
+    """Raised when a Backboard operation fails."""
+
+
+def build_client() -> BackboardClient:
+    api_key = os.getenv("BACKBOARD_API_KEY", "").strip()
+    if not api_key:
+        raise BackboardClientError(
+            "Missing required environment variable `BACKBOARD_API_KEY`."
+        )
+    return BackboardClient(api_key=api_key)
+
+
+async def ensure_assistant(
+    client: BackboardClient,
+    *,
+    assistant_id: str | None,
+    name: str,
+    system_prompt: str,
+    tools: list[dict[str, Any]],
+) -> str:
+    if assistant_id:
+        return assistant_id
+    assistant = await client.create_assistant(
+        name=name,
+        system_prompt=system_prompt,
+        tools=tools,
+    )
+    return str(assistant.assistant_id)
+
+
+async def _collect_stream(stream: Any) -> dict[str, Any]:
+    """Drain a streaming add_message response into a unified result dict."""
+    content_parts: list[str] = []
+    tool_calls: list[Any] = []
+    run_id: str | None = None
+    status = "completed"
+
+    async for chunk in stream:
+        t = chunk.get("type")
+        if t == "content_streaming":
+            content_parts.append(chunk.get("content", ""))
+        elif t == "tool_submit_required":
+            status = "REQUIRES_ACTION"
+            run_id = chunk.get("run_id")
+            tool_calls = chunk.get("tool_calls", [])
+        elif t == "run_ended":
+            if chunk.get("status") not in (None, "completed"):
+                raise BackboardClientError(
+                    f"Run ended with status: {chunk.get('status')}"
+                )
+
+    return {
+        "content": "".join(content_parts) or None,
+        "status": status,
+        "tool_calls": tool_calls,
+        "run_id": run_id,
+    }
+
+
+async def run_in_thread(
+    client: BackboardClient,
+    *,
+    assistant_id: str,
+    user_message: str,
+    tool_handlers: Mapping[str, ToolHandler],
+    llm_provider: str = "openai",
+    model_name: str = "gpt-4o-mini",
+    max_tool_rounds: int = 6,
+) -> str:
+    """Create a thread, send a message via streaming, execute the tool loop."""
+    thread = await client.create_thread(assistant_id)
+
+    stream = await client.add_message(
+        thread_id=thread.thread_id,
+        content=user_message,
+        stream=True,
+        llm_provider=llm_provider,
+        model_name=model_name,
+    )
+    result = await _collect_stream(stream)
+
+    rounds = 0
+    while result["status"] == "REQUIRES_ACTION":
+        rounds += 1
+        if rounds > max_tool_rounds:
+            raise BackboardClientError(
+                f"Tool loop exceeded {max_tool_rounds} rounds — aborting."
+            )
+        if not result["run_id"]:
+            raise BackboardClientError("REQUIRES_ACTION without run_id.")
+        if not result["tool_calls"]:
+            raise BackboardClientError("REQUIRES_ACTION without tool_calls.")
+
+        tool_outputs = []
+        for tc in result["tool_calls"]:
+            name = tc["function"]["name"] if isinstance(tc, dict) else tc.function.name
+            args_raw = (
+                tc["function"].get("arguments", "{}")
+                if isinstance(tc, dict)
+                else (tc.function.arguments or "{}")
+            )
+            args = args_raw if isinstance(args_raw, dict) else json.loads(args_raw or "{}")
+            tc_id = tc["id"] if isinstance(tc, dict) else tc.id
+
+            handler = tool_handlers.get(name)
+            if handler is None:
+                raise BackboardClientError(f"No handler registered for tool `{name}`.")
+
+            call_result = await handler(args)
+            tool_outputs.append({"tool_call_id": tc_id, "output": json.dumps(call_result)})
+
+        stream = await client.submit_tool_outputs(
+            thread_id=thread.thread_id,
+            run_id=result["run_id"],
+            tool_outputs=tool_outputs,
+            stream=True,
+        )
+        result = await _collect_stream(stream)
+
+    if not result["content"]:
+        raise BackboardClientError("Run completed without content.")
+    return result["content"]

devpost_scraper-0.1.0/src/devpost_scraper/cli.py

@@ -0,0 +1,364 @@
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse
+
+from dotenv import load_dotenv, set_key
+
+from devpost_scraper.backboard_client import (
+    BackboardClientError,
+    build_client,
+    ensure_assistant,
+    run_in_thread,
+)
+from devpost_scraper.csv_export import write_projects
+from devpost_scraper.models import DevpostProject, HackathonParticipant
+from devpost_scraper.scraper import (
+    find_author_email,
+    find_participant_email,
+    get_hackathon_participants,
+    get_project_details,
+    search_projects,
+)
+
+_ENV_FILE = Path(".env")
+_ASSISTANT_ID_KEY = "DEVPOST_ASSISTANT_ID"
+
+# The assistant's ONLY job is to search and return raw project URLs.
+# Python handles all enrichment directly — no tool loop explosion.
+_SYSTEM_PROMPT = """\
+You are a Devpost search assistant. Given a search term:
+
+1. Call search_devpost_projects for page 1 and page 2.
+2. Deduplicate results by URL.
+3. Return ONLY a valid JSON array — no prose, no markdown, no code fences.
+
+Each element: {"title": "...", "tagline": "...", "url": "...", "built_with": "..."}
+built_with is a comma-separated string of technology names.
+Never call the same tool with the same arguments twice.\
+"""
+
+_TOOLS: list[dict[str, Any]] = [
+    {
+        "type": "function",
+        "function": {
+            "name": "search_devpost_projects",
+            "description": "Search Devpost for hackathon projects matching a query.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "query": {"type": "string", "description": "Search query term"},
+                    "page": {"type": "integer", "description": "Page number (default 1)"},
+                },
+                "required": ["query"],
+            },
+        },
+    },
+]
+
+
+async def _handle_search(args: dict[str, Any]) -> dict[str, Any]:
+    query = args["query"]
+    page = int(args.get("page") or 1)
+    print(f"  [tool] search_devpost_projects(query={query!r}, page={page})", file=sys.stderr)
+    return await search_projects(query=query, page=page)
+
+
+_TOOL_HANDLERS = {
+    "search_devpost_projects": _handle_search,
+}
+
+
+async def _load_or_create_assistant(client: Any) -> str:
+    load_dotenv(_ENV_FILE, override=True)
+    stored_id = os.getenv(_ASSISTANT_ID_KEY, "").strip()
+    if stored_id:
+        print(f"[info] Reusing assistant {stored_id}", file=sys.stderr)
+        return stored_id
+
+    print("[info] Creating Backboard assistant…", file=sys.stderr)
+    aid = await ensure_assistant(
+        client,
+        assistant_id=None,
+        name="devpost-scraper-v3",
+        system_prompt=_SYSTEM_PROMPT,
+        tools=_TOOLS,
+    )
+    _ENV_FILE.touch(exist_ok=True)
+    set_key(str(_ENV_FILE), _ASSISTANT_ID_KEY, str(aid))
+    print(f"[info] Created assistant {aid} — saved to .env", file=sys.stderr)
+    return str(aid)
+
+
+def _parse_search_results(raw: str) -> list[dict[str, Any]]:
+    raw = raw.strip()
+    if raw.startswith("```"):
+        raw = "\n".join(
+            line for line in raw.splitlines()
+            if not line.strip().startswith("```")
+        ).strip()
+    try:
+        data = json.loads(raw)
+    except json.JSONDecodeError as exc:
+        raise SystemExit(
+            f"[error] Assistant returned invalid JSON: {exc}\n\nRaw:\n{raw}"
+        ) from exc
+    if not isinstance(data, list):
+        raise SystemExit(f"[error] Expected JSON array, got {type(data).__name__}")
+    return [item for item in data if isinstance(item, dict) and item.get("url")]
+
+
+async def _enrich_project(
+    item: dict[str, Any],
+    search_term: str,
+) -> DevpostProject:
+    url = item["url"]
+
+    # detail page enrichment
+    details: dict[str, Any] = {}
+    try:
+        details = await get_project_details(url=url)
+        print(f"  [enrich] details {url}", file=sys.stderr)
+    except Exception as exc:
+        print(f"  [warn] details failed for {url}: {exc}", file=sys.stderr)
+
+    # email chain
+    email_data: dict[str, Any] = {}
+    try:
+        email_data = await find_author_email(project_url=url)
+        if email_data.get("email"):
+            print(f"  [email] {email_data['email']} ← {url}", file=sys.stderr)
+        else:
+            print(f"  [email] (none found) ← {url}", file=sys.stderr)
+    except Exception as exc:
+        print(f"  [warn] email failed for {url}: {exc}", file=sys.stderr)
+
+    author_urls: list[str] = email_data.get("author_profile_urls", [])
+
+    return DevpostProject(
+        search_term=search_term,
+        title=details.get("title") or item.get("title", ""),
+        tagline=details.get("tagline") or item.get("tagline", ""),
+        url=url,
+        hackathon_name=details.get("hackathon_name", ""),
+        hackathon_url=details.get("hackathon_url", ""),
+        summary=details.get("summary", ""),
+        built_with=details.get("built_with") or item.get("built_with", ""),
+        prizes=details.get("prizes", ""),
+        team_size=details.get("team_size", ""),
+        author_profile_url=author_urls[0] if author_urls else "",
+        email=email_data.get("email", ""),
+    )
+
+
+async def run(search_terms: list[str], output: str | None) -> None:
+    load_dotenv(_ENV_FILE, override=True)
+    client = build_client()
+    assistant_id = await _load_or_create_assistant(client)
+
+    all_projects: list[DevpostProject] = []
+
+    for term in search_terms:
+        print(f"\n[info] Searching Devpost for: {term!r}", file=sys.stderr)
+        raw = await run_in_thread(
+            client,
+            assistant_id=assistant_id,
+            user_message=(
+                f"Search Devpost for: {term!r}\n"
+                "Collect page 1 and page 2. Return a JSON array of projects."
+            ),
+            tool_handlers=_TOOL_HANDLERS,
+            llm_provider=os.getenv("BACKBOARD_LLM_PROVIDER", "openai"),
+            model_name=os.getenv("BACKBOARD_MODEL", "gpt-4o-mini"),
+        )
+        items = _parse_search_results(raw)
+        print(f"[info] Found {len(items)} projects — enriching…", file=sys.stderr)
+
+        # Enrich sequentially to be polite to external sites
+        projects: list[DevpostProject] = []
+        for item in items:
+            project = await _enrich_project(item, search_term=term)
+            projects.append(project)
+
+        print(f"[info] Collected {len(projects)} projects for {term!r}", file=sys.stderr)
+        all_projects.extend(projects)
+
+    print(f"\n[info] Total projects: {len(all_projects)}", file=sys.stderr)
+    write_projects(all_projects, output)
+    if output:
+        print(f"[info] Wrote → {output}", file=sys.stderr)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        prog="devpost-scraper",
+        description="Extract Devpost project data and export to CSV.",
+    )
+    parser.add_argument(
+        "search_terms",
+        nargs="+",
+        metavar="TERM",
+        help="One or more search terms to query on Devpost",
+    )
+    parser.add_argument(
+        "--output", "-o",
+        metavar="FILE",
+        default=None,
+        help="Output CSV file path (default: stdout)",
+    )
+    args = parser.parse_args()
+    asyncio.run(run(search_terms=args.search_terms, output=args.output))
+
+
+if __name__ == "__main__":
+    main()
+
+
+_PARTICIPANTS_JWT_KEY = "DEVPOST_SESSION"
+
+
+async def _run_participants(
+    hackathon_url: str,
+    jwt_token: str,
+    output: str | None,
+    no_email: bool,
+) -> None:
+    all_participants: list[HackathonParticipant] = []
+    page = 1
+
+    print(f"[info] Fetching participants from {hackathon_url}", file=sys.stderr)
+
+    while True:
+        data = await get_hackathon_participants(hackathon_url, jwt_token, page=page)
+        batch = data.get("participants", [])
+        has_more = data.get("has_more", False)
+
+        if not batch:
+            print(f"[info] No participants on page {page}, stopping.", file=sys.stderr)
+            break
+
+        print(f"[info] Page {page}: {len(batch)} participants", file=sys.stderr)
+
+        for raw in batch:
+            profile_url = raw.get("profile_url", "")
+            email = ""
+            github_url = ""
+            linkedin_url = ""
+
+            if not no_email and profile_url:
+                try:
+                    email_data = await find_participant_email(profile_url)
+                    email = email_data.get("email", "")
+                    github_url = email_data.get("github_url", "")
+                    linkedin_url = email_data.get("linkedin_url", "")
+                    parts = [f for f in [email, github_url, linkedin_url] if f]
+                    if parts:
+                        print(f"  [found] {', '.join(parts)} ← {profile_url}", file=sys.stderr)
+                    else:
+                        print(f"  [none] ← {profile_url}", file=sys.stderr)
+                except Exception as exc:
+                    print(f"  [warn] enrich failed for {profile_url}: {exc}", file=sys.stderr)
+
+            all_participants.append(
+                HackathonParticipant(
+                    hackathon_url=hackathon_url,
+                    username=raw.get("username", ""),
+                    name=raw.get("name", ""),
+                    specialty=raw.get("specialty", ""),
+                    profile_url=profile_url,
+                    github_url=github_url,
+                    linkedin_url=linkedin_url,
+                    email=email,
+                )
+            )
+
+        if not has_more:
+            break
+        page += 1
+
+    print(f"\n[info] Total participants: {len(all_participants)}", file=sys.stderr)
+
+    import csv
+
+    fieldnames = HackathonParticipant.fieldnames()
+    rows = [p.model_dump() for p in all_participants]
+
+    if output:
+        with open(output, "w", newline="", encoding="utf-8") as f:
+            writer = csv.DictWriter(f, fieldnames=fieldnames)
+            writer.writeheader()
+            writer.writerows(rows)
+        print(f"[info] Wrote → {output}", file=sys.stderr)
+    else:
+        import io
+        buf = io.StringIO()
+        writer = csv.DictWriter(buf, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(rows)
+        print(buf.getvalue())
+
+
+def participants_main() -> None:
+    load_dotenv(_ENV_FILE, override=True)
+
+    parser = argparse.ArgumentParser(
+        prog="devpost-participants",
+        description="Crawl Devpost hackathon participants page and export to CSV.",
+    )
+    parser.add_argument(
+        "hackathon_url",
+        metavar="URL",
+        help="Hackathon participants URL (e.g. https://hack-days-niet.devpost.com/participants)",
+    )
+    parser.add_argument(
+        "--jwt",
+        metavar="TOKEN",
+        default=None,
+        help="Value of the _devpost session cookie from your browser. Falls back to DEVPOST_SESSION in .env",
+    )
+    parser.add_argument(
+        "--output", "-o",
+        metavar="FILE",
+        default=None,
+        help="Output CSV file path (default: stdout)",
+    )
+    parser.add_argument(
+        "--no-email",
+        action="store_true",
+        default=False,
+        help="Skip email enrichment (faster)",
+    )
+    args = parser.parse_args()
+
+    if not args.output:
+        parsed = urlparse(args.hackathon_url)
+        slug = parsed.hostname.split(".")[0] if parsed.hostname else "hackathon"
+        args.output = f"{slug}-participants.csv"
+        print(f"[info] No -o given, defaulting to {args.output}", file=sys.stderr)
+
+    jwt_token = args.jwt or os.getenv(_PARTICIPANTS_JWT_KEY, "").strip()
+    if not jwt_token:
+        raise SystemExit(
+            "[error] No session cookie. Pass --jwt TOKEN or set DEVPOST_SESSION in .env\n"
+            "  Copy the _devpost cookie value from browser DevTools → Application → Cookies"
+        )
+
+    # Persist JWT to .env for reuse
+    if args.jwt:
+        _ENV_FILE.touch(exist_ok=True)
+        set_key(str(_ENV_FILE), _PARTICIPANTS_JWT_KEY, args.jwt)
+
+    asyncio.run(
+        _run_participants(
+            hackathon_url=args.hackathon_url,
+            jwt_token=jwt_token,
+            output=args.output,
+            no_email=args.no_email,
+        )
+    )

devpost_scraper-0.1.0/src/devpost_scraper/csv_export.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+import csv
+import sys
+from pathlib import Path
+from typing import Iterable
+
+from devpost_scraper.models import DevpostProject
+
+
+def write_projects(projects: Iterable[DevpostProject], output: str | None) -> None:
+    """Write projects to CSV. Prints to stdout if output is None."""
+    fieldnames = DevpostProject.fieldnames()
+
+    if output:
+        path = Path(output)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        fh = path.open("w", newline="", encoding="utf-8")
+        close = True
+    else:
+        fh = sys.stdout
+        close = False
+
+    writer = csv.DictWriter(fh, fieldnames=fieldnames, extrasaction="ignore")
+    writer.writeheader()
+    for project in projects:
+        writer.writerow(project.model_dump())
+
+    if close:
+        fh.close()

devpost_scraper-0.1.0/src/devpost_scraper/models.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict
+
+
+class HackathonParticipant(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    hackathon_url: str = ""
+    username: str = ""
+    name: str = ""
+    specialty: str = ""
+    profile_url: str = ""
+    github_url: str = ""
+    linkedin_url: str = ""
+    email: str = ""
+
+    @classmethod
+    def fieldnames(cls) -> list[str]:
+        return ["hackathon_url", "username", "name", "specialty", "profile_url", "github_url", "linkedin_url", "email"]
+
+
+class DevpostProject(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    search_term: str = ""
+    title: str = ""
+    tagline: str = ""
+    url: str = ""
+    hackathon_name: str = ""
+    hackathon_url: str = ""
+    summary: str = ""
+    built_with: str = ""
+    prizes: str = ""
+    team_size: str = ""
+    author_profile_url: str = ""
+    email: str = ""
+
+    @classmethod
+    def fieldnames(cls) -> list[str]:
+        return [
+            "search_term",
+            "title",
+            "tagline",
+            "url",
+            "hackathon_name",
+            "hackathon_url",
+            "summary",
+            "built_with",
+            "prizes",
+            "team_size",
+            "author_profile_url",
+            "email",
+        ]

devpost_scraper-0.1.0/src/devpost_scraper/scraper.py

@@ -0,0 +1,510 @@
+from __future__ import annotations
+
+import os
+import re
+from typing import Any
+from urllib.parse import urljoin, urlparse
+
+import httpx
+from bs4 import BeautifulSoup
+
+_EMAIL_RE = re.compile(r"[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}")
+
+# Domains we will follow when walking external links from a Devpost profile
+_WALKABLE_DOMAINS = {
+    "github.com",
+    "linktr.ee",
+    "bio.link",
+    "beacons.ai",
+    "linkin.bio",
+    "carrd.co",
+    "about.me",
+    "bento.me",
+}
+
+_SEARCH_URL = "https://devpost.com/software/search"
+_GITHUB_API_URL = "https://api.github.com/users"
+
+
+def _github_headers() -> dict[str, str]:
+    """Build GitHub API headers, including auth token if GITHUB_TOKEN is set."""
+    headers = {
+        "Accept": "application/vnd.github+json",
+        "User-Agent": "devpost-scraper/1.0",
+    }
+    token = os.environ.get("GITHUB_TOKEN", "").strip()
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+    return headers
+_JSON_HEADERS = {
+    "Accept": "application/json, text/javascript, */*; q=0.01",
+    "X-Requested-With": "XMLHttpRequest",
+    "User-Agent": (
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/122.0.0.0 Safari/537.36"
+    ),
+}
+_HTML_HEADERS = {
+    "Accept": "text/html,application/xhtml+xml",
+    "User-Agent": _JSON_HEADERS["User-Agent"],
+}
+
+
+async def search_projects(query: str, page: int = 1) -> dict[str, Any]:
+    """Search Devpost projects. Returns raw API payload with 'software' list."""
+    async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
+        resp = await client.get(
+            _SEARCH_URL,
+            params={"query": query, "page": page},
+            headers=_JSON_HEADERS,
+        )
+        resp.raise_for_status()
+        data = resp.json()
+
+    projects = []
+    for item in data.get("software", []):
+        built = item.get("built_with") or []
+        projects.append(
+            {
+                "title": item.get("name", ""),
+                "tagline": item.get("tagline", ""),
+                "url": item.get("url", ""),
+                "built_with": ", ".join(built) if isinstance(built, list) else str(built),
+                "like_count": item.get("like_count", 0),
+            }
+        )
+
+    return {
+        "projects": projects,
+        "total_count": data.get("total_count", 0),
+        "page": page,
+        "per_page": data.get("per_page", 24),
+    }
+
+
+async def get_project_details(url: str) -> dict[str, Any]:
+    """Fetch a Devpost project page and extract detail fields."""
+    async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
+        resp = await client.get(url, headers=_HTML_HEADERS)
+        resp.raise_for_status()
+        html = resp.text
+
+    soup = BeautifulSoup(html, "html.parser")
+
+    title = _text(soup.select_one("h1#app-title") or soup.select_one("h1.app_title"))
+    tagline = _text(soup.select_one("p#app-details-header-tagline") or soup.select_one("p.large"))
+
+    summary_el = soup.select_one("div#app-details") or soup.select_one("div.app-details")
+    summary = summary_el.get_text(" ", strip=True)[:500] if summary_el else ""
+
+    built_tags = [t.get_text(strip=True) for t in soup.select("span.cp-tag")]
+    built_with = ", ".join(built_tags)
+
+    hackathon_name = ""
+    hackathon_url = ""
+    challenge_link = soup.select_one("a.challenge-link") or soup.select_one("a[href*='/hackathons/']")
+    if challenge_link:
+        hackathon_name = challenge_link.get_text(strip=True)
+        hackathon_url = challenge_link.get("href", "")
+
+    prizes: list[str] = []
+    for prize_el in soup.select("div.prize, li.prize, span.prize-name"):
+        text = prize_el.get_text(strip=True)
+        if text:
+            prizes.append(text)
+
+    team_members = soup.select("ul#app-team li, div.software-team-member")
+    team_size = str(len(team_members)) if team_members else ""
+
+    return {
+        "title": title,
+        "tagline": tagline,
+        "url": url,
+        "summary": summary,
+        "built_with": built_with,
+        "hackathon_name": hackathon_name,
+        "hackathon_url": hackathon_url,
+        "prizes": "; ".join(prizes),
+        "team_size": team_size,
+    }
+
+
+def _text(el: Any) -> str:
+    if el is None:
+        return ""
+    return el.get_text(strip=True)
+
+
+def _extract_emails(html: str) -> list[str]:
+    """Find all email addresses in an HTML document (mailto: + bare text)."""
+    soup = BeautifulSoup(html, "html.parser")
+    found: set[str] = set()
+
+    for a in soup.find_all("a", href=True):
+        href: str = a["href"]
+        if href.startswith("mailto:"):
+            addr = href[7:].split("?")[0].strip()
+            if addr:
+                found.add(addr.lower())
+
+    for match in _EMAIL_RE.finditer(soup.get_text(" ")):
+        found.add(match.group().lower())
+
+    # Filter out obviously invalid / placeholder emails
+    return [e for e in found if "." in e.split("@")[-1] and len(e) < 80]
+
+
+_DEVPOST_NON_PROFILE_PATHS = {
+    "software", "hackathons", "settings", "portfolio", "search",
+    "about", "contact", "help", "careers", "login", "register",
+}
+
+
+async def get_author_profile_urls(project_url: str) -> dict[str, Any]:
+    """From a Devpost project page, return the author profile URLs."""
+    async with httpx.AsyncClient(timeout=20.0, follow_redirects=True) as client:
+        resp = await client.get(project_url, headers=_HTML_HEADERS)
+        resp.raise_for_status()
+        html = resp.text
+
+    soup = BeautifulSoup(html, "html.parser")
+    profiles: list[str] = []
+
+    for a in soup.find_all("a", href=True):
+        href: str = a["href"]
+        if href.startswith("/"):
+            href = f"https://devpost.com{href}"
+        parsed = urlparse(href)
+        if parsed.netloc not in ("devpost.com", "www.devpost.com"):
+            continue
+        path_parts = [p for p in parsed.path.strip("/").split("/") if p]
+        if len(path_parts) != 1:
+            continue
+        slug = path_parts[0]
+        if slug in _DEVPOST_NON_PROFILE_PATHS:
+            continue
+        # Devpost usernames are alphanumeric + dashes, no dots or slashes
+        if re.match(r"^[a-zA-Z0-9_\-]+$", slug):
+            profiles.append(f"https://devpost.com/{slug}")
+
+    return {"author_profile_urls": list(dict.fromkeys(profiles))}
+
+
+async def get_profile_external_links(profile_url: str) -> dict[str, Any]:
+    """From a Devpost author profile, return external links."""
+    async with httpx.AsyncClient(timeout=20.0, follow_redirects=True) as client:
+        resp = await client.get(profile_url, headers=_HTML_HEADERS)
+        resp.raise_for_status()
+        html = resp.text
+
+    soup = BeautifulSoup(html, "html.parser")
+    external: list[str] = []
+    emails = _extract_emails(html)
+
+    for a in soup.find_all("a", href=True):
+        href: str = a["href"]
+        parsed = urlparse(href)
+        if parsed.scheme not in ("http", "https"):
+            continue
+        domain = parsed.netloc.lstrip("www.")
+        if domain and domain not in ("devpost.com",):
+            external.append(href)
+
+    return {
+        "profile_url": profile_url,
+        "external_links": list(dict.fromkeys(external)),
+        "emails_on_profile": emails,
+    }
+
+
+async def extract_emails_from_url(url: str) -> dict[str, Any]:
+    """Fetch any URL and return email addresses found on the page."""
+    try:
+        async with httpx.AsyncClient(timeout=20.0, follow_redirects=True) as client:
+            resp = await client.get(url, headers=_HTML_HEADERS)
+            resp.raise_for_status()
+            html = resp.text
+    except Exception as exc:
+        return {"url": url, "emails": [], "error": str(exc)}
+
+    emails = _extract_emails(html)
+    return {"url": url, "emails": emails}
+
+
+async def get_hackathon_participants(
+    hackathon_url: str,
+    jwt_token: str,
+    page: int = 1,
+) -> dict[str, Any]:
+    """
+    Fetch one page of participants from a Devpost hackathon participants page.
+    Requires a valid Devpost session JWT (sent as _devpost_session cookie).
+    Returns {"participants": [...], "has_more": bool}.
+    """
+    base = hackathon_url.rstrip("/").removesuffix("/participants")
+    url = f"{base}/participants"
+
+    # Devpost serves participant HTML fragments via XHR; plain GET returns empty on page 2+
+    headers = {
+        **_JSON_HEADERS,
+        "Accept": "text/javascript, application/javascript",
+        "Cookie": f"_devpost={jwt_token}",
+        "Referer": url,
+    }
+
+    async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
+        resp = await client.get(url, params={"page": page}, headers=headers)
+        resp.raise_for_status()
+        html = resp.text
+
+    soup = BeautifulSoup(html, "html.parser")
+
+    # Each participant is a div.participant with data-participant-id
+    cards = soup.select("div.participant")
+
+    # CSS classes on participant card encode specialty, e.g. "participant full-stack-developer"
+    _CARD_SKIP_CLASSES = {"participant"}
+
+    participants: list[dict[str, Any]] = []
+    for card in cards:
+        link = card.select_one("a.user-profile-link")
+        if not link:
+            continue
+        profile_href: str = link.get("href", "")
+        if profile_href.startswith("/"):
+            profile_href = f"https://devpost.com{profile_href}"
+
+        parsed = urlparse(profile_href)
+        slug = parsed.path.strip("/").split("/")[0] if parsed.path else ""
+
+        # Name lives in img[alt] or h5 inside .user-name
+        img = card.select_one("img[alt]")
+        name = img["alt"].strip() if img and img.get("alt") else slug
+
+        # Specialty is encoded as extra CSS class on the card div
+        card_classes = [c for c in card.get("class", []) if c not in _CARD_SKIP_CLASSES]
+        specialty = card_classes[0].replace("-", " ").title() if card_classes else ""
+
+        participants.append({
+            "username": slug,
+            "name": name,
+            "profile_url": profile_href,
+            "specialty": specialty,
+        })
+
+    # Pagination: Devpost renders <a rel="next"> when more pages exist
+    next_link = soup.select_one('a[rel="next"]')
+    has_more = next_link is not None
+
+    return {"participants": participants, "has_more": has_more, "page": page}
+
+
+_GITHUB_ORG_PATHS = {"orgs", "repos", "topics", "collections", "explore", "marketplace", "about"}
+
+
+_NOREPLY_SUFFIXES = ("@users.noreply.github.com",)
+
+
+def _github_username_from_url(github_url: str) -> str:
+    """Extract a GitHub username from a profile URL. Returns '' for non-user URLs."""
+    parsed = urlparse(github_url)
+    path_parts = [p for p in parsed.path.strip("/").split("/") if p]
+    if len(path_parts) != 1 or path_parts[0] in _GITHUB_ORG_PATHS:
+        return ""
+    return path_parts[0]
+
+
+def _is_real_email(email: str) -> bool:
+    """Filter out GitHub noreply and placeholder addresses."""
+    if not email:
+        return False
+    email = email.lower().strip()
+    if email.endswith(_NOREPLY_SUFFIXES):
+        return False
+    if "noreply" in email or "github.com" in email:
+        return False
+    return "." in email.split("@")[-1]
+
+
+async def get_github_email(github_url: str) -> str:
+    """
+    Try three GitHub API strategies to find a user's email:
+    1. /users/{user} — public profile email field (often private)
+    2. /users/{user}/repos?sort=pushed → /repos/{owner}/{repo}/commits — mine commit author email
+    3. /users/{user}/events/public — fallback: PushEvent commit payloads
+    """
+    username = _github_username_from_url(github_url)
+    if not username:
+        return ""
+
+    try:
+        async with httpx.AsyncClient(timeout=10.0) as client:
+            # Strategy 1: profile email
+            resp = await client.get(
+                f"{_GITHUB_API_URL}/{username}",
+                headers=_github_headers(),
+            )
+            if resp.status_code == 200:
+                email = (resp.json().get("email") or "").strip()
+                if _is_real_email(email):
+                    return email
+
+            # Strategy 2: most-recently-pushed repo → commits → author email
+            resp = await client.get(
+                f"{_GITHUB_API_URL}/{username}/repos",
+                params={"sort": "pushed", "per_page": 3, "type": "owner"},
+                headers=_github_headers(),
+            )
+            if resp.status_code == 200:
+                for repo in resp.json():
+                    full_name = repo.get("full_name", "")
+                    if repo.get("fork") or not full_name:
+                        continue
+                    commit_resp = await client.get(
+                        f"https://api.github.com/repos/{full_name}/commits",
+                        params={"author": username, "per_page": 5},
+                        headers=_github_headers(),
+                    )
+                    if commit_resp.status_code != 200:
+                        continue
+                    for c in commit_resp.json():
+                        author = c.get("commit", {}).get("author", {})
+                        email = (author.get("email") or "").strip().lower()
+                        if _is_real_email(email):
+                            return email
+
+            # Strategy 3: PushEvent payloads (fallback, often has 0 commits)
+            resp = await client.get(
+                f"{_GITHUB_API_URL}/{username}/events/public",
+                params={"per_page": 20},
+                headers=_github_headers(),
+            )
+            if resp.status_code == 200:
+                for event in resp.json():
+                    if event.get("type") != "PushEvent":
+                        continue
+                    for commit in event.get("payload", {}).get("commits", []):
+                        email = (commit.get("author", {}).get("email") or "").strip().lower()
+                        if _is_real_email(email):
+                            return email
+
+    except Exception:
+        pass
+
+    return ""
+
+
+_DEVPOST_OWNED_DOMAINS = {
+    "devpost.com", "devpost.team", "info.devpost.com",
+    "secure.devpost.com", "d2dmyh35ffsxbl.cloudfront.net",
+    "d112y698adiu2z.cloudfront.net",
+}
+
+
+def _is_personal_link(url: str) -> bool:
+    """Filter out Devpost-owned links (nav, footer, CDN) from external link lists."""
+    parsed = urlparse(url)
+    domain = parsed.netloc.lstrip("www.")
+    return domain not in _DEVPOST_OWNED_DOMAINS
+
+
+async def find_participant_email(profile_url: str) -> dict[str, Any]:
+    """
+    Enrich a participant from their Devpost profile:
+    1. Extract GitHub URL, LinkedIn URL from profile social links
+    2. Try GitHub API for public email
+    3. Walk other external links for email (linktr.ee, bio.link, etc.)
+    """
+    result: dict[str, Any] = {
+        "profile_url": profile_url,
+        "external_links_walked": [],
+        "github_url": "",
+        "linkedin_url": "",
+        "email": "",
+    }
+
+    profile_data = await get_profile_external_links(profile_url)
+    all_emails: list[str] = list(profile_data.get("emails_on_profile", []))
+
+    personal_links = [l for l in profile_data.get("external_links", []) if _is_personal_link(l)]
+
+    # First pass: capture GitHub + LinkedIn URLs
+    for link in personal_links:
+        parsed = urlparse(link)
+        domain = parsed.netloc.lstrip("www.")
+        path_parts = [p for p in parsed.path.strip("/").split("/") if p]
+
+        if domain == "github.com" and path_parts and not result["github_url"]:
+            if path_parts[0] not in _GITHUB_ORG_PATHS and len(path_parts) == 1:
+                result["github_url"] = link
+
+        if domain == "linkedin.com" and "/in/" in parsed.path and not result["linkedin_url"]:
+            if "/company/" not in parsed.path:
+                result["linkedin_url"] = link
+
+    # Try GitHub API for public email
+    if result["github_url"] and not all_emails:
+        email = await get_github_email(result["github_url"])
+        if email:
+            all_emails.append(email)
+
+    # Walk remaining external links for email
+    if not all_emails:
+        for link in personal_links:
+            parsed = urlparse(link)
+            domain = parsed.netloc.lstrip("www.")
+            if domain in ("github.com", "linkedin.com"):
+                continue
+            if domain not in _WALKABLE_DOMAINS:
+                continue
+            result["external_links_walked"].append(link)
+            link_data = await extract_emails_from_url(link)
+            all_emails.extend(link_data.get("emails", []))
+            if all_emails:
+                break
+
+    result["email"] = all_emails[0] if all_emails else ""
+    return result
+
+
+async def find_author_email(project_url: str) -> dict[str, Any]:
+    """
+    Full chain: project page → author profile(s) → external links → emails.
+    Returns the first email found along with the chain of URLs walked.
+    """
+    result: dict[str, Any] = {
+        "project_url": project_url,
+        "author_profile_urls": [],
+        "external_links_walked": [],
+        "email": "",
+    }
+
+    # Step 1: get author profiles from project page
+    profiles_data = await get_author_profile_urls(project_url)
+    author_urls: list[str] = profiles_data.get("author_profile_urls", [])
+    result["author_profile_urls"] = author_urls
+
+    all_emails: list[str] = []
+
+    for profile_url in author_urls[:3]:  # cap at 3 authors
+        profile_data = await get_profile_external_links(profile_url)
+
+        # Emails directly on profile page
+        all_emails.extend(profile_data.get("emails_on_profile", []))
+
+        # Walk external links from profile
+        for link in profile_data.get("external_links", []):
+            parsed = urlparse(link)
+            domain = parsed.netloc.lstrip("www.")
+            if domain not in _WALKABLE_DOMAINS:
+                continue
+            result["external_links_walked"].append(link)
+            link_data = await extract_emails_from_url(link)
+            all_emails.extend(link_data.get("emails", []))
+
+        if all_emails:
+            break  # stop after first author with a result
+
+    result["email"] = all_emails[0] if all_emails else ""
+    return result