utopia-analyzer 1.0.0__py3-none-any.whl

utopia/github_client.py

@@ -0,0 +1,517 @@
+"""GitHub REST API client with caching, pagination, and rate-limit handling."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+from urllib.parse import urljoin
+
+import requests
+
+
+GITHUB_API_BASE = "https://api.github.com"
+CACHE_TTL_SECONDS = 3600  # 1 hour
+MAX_RETRIES = 3
+
+
+class GitHubError(Exception):
+    """Base exception for GitHub client errors."""
+
+    def __init__(self, message: str, status_code: int | None = None):
+        super().__init__(message)
+        self.message = message
+        self.status_code = status_code
+
+
+class RateLimitError(GitHubError):
+    """Raised when GitHub API rate limit is hit."""
+
+    pass
+
+
+class UserNotFoundError(GitHubError):
+    """Raised when the requested GitHub user does not exist."""
+
+    pass
+
+
+def _cache_dir() -> Path:
+    cache = Path.home() / ".utopia_cache"
+    cache.mkdir(parents=True, exist_ok=True)
+    return cache
+
+
+def _cache_path(username: str) -> Path:
+    safe_name = hashlib.sha256(username.encode()).hexdigest()[:16]
+    return _cache_dir() / f"{safe_name}.json"
+
+
+def _load_cache(username: str) -> dict[str, Any] | None:
+    path = _cache_path(username)
+    if not path.exists():
+        return None
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+        fetched_at = data.get("_fetched_at", 0)
+        if time.time() - fetched_at > CACHE_TTL_SECONDS:
+            return None
+        return data
+    except (json.JSONDecodeError, OSError):
+        return None
+
+
+def _save_cache(username: str, data: dict[str, Any]) -> None:
+    data["_fetched_at"] = time.time()
+    path = _cache_path(username)
+    try:
+        path.write_text(json.dumps(data, indent=2), encoding="utf-8")
+    except OSError:
+        pass
+
+
+def _clear_cache(username: str) -> None:
+    path = _cache_path(username)
+    if path.exists():
+        path.unlink(missing_ok=True)
+
+
+@dataclass
+class RepoData:
+    """Normalized repository data collected by the client."""
+
+    name: str
+    description: str | None
+    language: str | None
+    stars: int
+    forks: int
+    size_kb: int
+    created_at: str
+    updated_at: str
+    pushed_at: str | None
+    has_readme: bool
+    readme_word_count: int
+    open_issues_count: int
+    topics: list[str]
+    license: str | None
+    is_fork: bool
+    is_archived: bool
+    languages: dict[str, int] = field(default_factory=dict)
+    commit_activity: list[int] = field(default_factory=list)
+    has_tests: bool = False
+    has_workflows: bool = False
+    has_dockerfile: bool = False
+
+
+@dataclass
+class ProfileData:
+    """Complete data bundle for a GitHub profile analysis."""
+
+    username: str
+    name: str | None
+    bio: str | None
+    followers: int
+    following: int
+    public_repos_count: int
+    created_at: str
+    repos: list[RepoData] = field(default_factory=list)
+    merged_prs_count: int = 0
+    total_languages: dict[str, int] = field(default_factory=dict)
+
+
+def _default_headers(token: str | None) -> dict[str, str]:
+    headers = {"Accept": "application/vnd.github.v3+json"}
+    if token:
+        headers["Authorization"] = f"token {token}"
+    return headers
+
+
+def _handle_response(response: requests.Response) -> Any:
+    if response.status_code == 404:
+        raise UserNotFoundError("GitHub user not found.", status_code=404)
+    if response.status_code in (403, 429):
+        reset = response.headers.get("X-RateLimit-Reset")
+        msg = "GitHub API rate limit exceeded."
+        if reset:
+            reset_time = time.strftime("%H:%M:%S", time.localtime(int(reset)))
+            msg += f" Resets at {reset_time}."
+        msg += " Use --token <PAT> for higher limits."
+        raise RateLimitError(msg, status_code=response.status_code)
+    if response.status_code >= 400:
+        raise GitHubError(
+            f"GitHub API error: {response.status_code} {response.text[:200]}",
+            status_code=response.status_code,
+        )
+    if response.status_code == 204:
+        return None
+    if not response.content:
+        return None
+    return response.json()
+
+
+def _get(
+    session: requests.Session,
+    url: str,
+    token: str | None,
+    params: dict[str, Any] | None = None,
+) -> Any:
+    response = session.get(url, headers=_default_headers(token), params=params or {}, timeout=30)
+    return _handle_response(response)
+
+
+def _get_all_pages(
+    session: requests.Session,
+    url: str,
+    token: str | None,
+    params: dict[str, Any] | None = None,
+) -> list[dict[str, Any]]:
+    items: list[dict[str, Any]] = []
+    page = 1
+    per_page = 100
+    query_params = dict(params or {})
+    query_params["per_page"] = per_page
+
+    while True:
+        query_params["page"] = page
+        data = _get(session, url, token, query_params)
+        if not isinstance(data, list):
+            break
+        items.extend(data)
+        if len(data) < per_page:
+            break
+        page += 1
+
+    return items
+
+
+def _fetch_with_202_retry(
+    session: requests.Session,
+    url: str,
+    token: str | None,
+    retries: int = MAX_RETRIES,
+) -> Any:
+    """Fetch an endpoint that may return 202 while computing data."""
+    for attempt in range(retries):
+        response = session.get(url, headers=_default_headers(token), timeout=30)
+        if response.status_code == 202:
+            time.sleep(1.5 * (attempt + 1))
+            continue
+        return _handle_response(response)
+    return None
+
+
+def _check_file_exists(
+    session: requests.Session,
+    owner: str,
+    repo: str,
+    path: str,
+    token: str | None,
+) -> bool:
+    url = f"{GITHUB_API_BASE}/repos/{owner}/{repo}/contents/{path}"
+    try:
+        response = session.get(url, headers=_default_headers(token), timeout=30)
+        return response.status_code == 200
+    except requests.RequestException:
+        return False
+
+
+def _fetch_readme_word_count(
+    session: requests.Session,
+    owner: str,
+    repo: str,
+    token: str | None,
+) -> int:
+    url = f"{GITHUB_API_BASE}/repos/{owner}/{repo}/readme"
+    try:
+        data = _get(session, url, token)
+        if not isinstance(data, dict):
+            return 0
+        content = data.get("content", "")
+        import base64
+
+        try:
+            decoded = base64.b64decode(content).decode("utf-8", errors="ignore")
+        except Exception:
+            return 0
+        return len(decoded.split())
+    except (GitHubError, requests.RequestException):
+        return 0
+
+
+def _fetch_repo_tree_flags(
+    session: requests.Session,
+    owner: str,
+    repo: str,
+    token: str | None,
+) -> dict[str, bool]:
+    """Check for test files, workflows, and Dockerfile via repository tree."""
+    url = f"{GITHUB_API_BASE}/repos/{owner}/{repo}/git/trees/HEAD?recursive=1"
+    try:
+        data = _get(session, url, token)
+        if not isinstance(data, dict):
+            return {"has_tests": False, "has_workflows": False, "has_dockerfile": False}
+        tree = data.get("tree", [])
+        paths = [item.get("path", "").lower() for item in tree]
+
+        has_tests = any(
+            "test" in p
+            and (
+                p.endswith((".py", ".js", ".ts", ".java", ".go", ".rs", ".rb", ".cpp", ".c"))
+                or "test" in p.split("/")[-1]
+            )
+            for p in paths
+        ) or any(p.startswith("tests/") or p.startswith("test/") for p in paths)
+
+        has_workflows = any(p.startswith(".github/workflows/") for p in paths)
+        has_dockerfile = any("dockerfile" in p or "docker-compose" in p for p in paths)
+
+        return {
+            "has_tests": has_tests,
+            "has_workflows": has_workflows,
+            "has_dockerfile": has_dockerfile,
+        }
+    except (GitHubError, requests.RequestException):
+        return {"has_tests": False, "has_workflows": False, "has_dockerfile": False}
+
+
+def _fetch_repo_details(
+    session: requests.Session,
+    owner: str,
+    repo_name: str,
+    token: str | None,
+) -> RepoData:
+    repo_url = f"{GITHUB_API_BASE}/repos/{owner}/{repo_name}"
+    repo = _get(session, repo_url, token)
+    if not isinstance(repo, dict):
+        raise GitHubError(f"Unexpected response for repo {repo_name}")
+
+    has_readme = _check_file_exists(session, owner, repo_name, "README.md", token)
+    readme_word_count = _fetch_readme_word_count(session, owner, repo_name, token) if has_readme else 0
+
+    languages_url = repo.get("languages_url", "")
+    languages: dict[str, int] = {}
+    if languages_url:
+        try:
+            langs = _get(session, languages_url, token)
+            if isinstance(langs, dict):
+                languages = {k: v for k, v in langs.items() if isinstance(v, int)}
+        except GitHubError:
+            languages = {}
+
+    commit_url = f"{GITHUB_API_BASE}/repos/{owner}/{repo_name}/stats/commit_activity"
+    commit_activity_raw = _fetch_with_202_retry(session, commit_url, token)
+    commit_activity: list[int] = []
+    if isinstance(commit_activity_raw, list):
+        commit_activity = [
+            int(week.get("total", 0)) if isinstance(week, dict) else 0
+            for week in commit_activity_raw
+        ]
+
+    tree_flags = _fetch_repo_tree_flags(session, owner, repo_name, token)
+
+    license_info = repo.get("license") or {}
+    license_name = license_info.get("spdx_id") if isinstance(license_info, dict) else None
+
+    return RepoData(
+        name=repo_name,
+        description=repo.get("description"),
+        language=repo.get("language"),
+        stars=int(repo.get("stargazers_count", 0) or 0),
+        forks=int(repo.get("forks_count", 0) or 0),
+        size_kb=int(repo.get("size", 0) or 0),
+        created_at=repo.get("created_at", ""),
+        updated_at=repo.get("updated_at", ""),
+        pushed_at=repo.get("pushed_at"),
+        has_readme=has_readme,
+        readme_word_count=readme_word_count,
+        open_issues_count=int(repo.get("open_issues_count", 0) or 0),
+        topics=list(repo.get("topics", []) or []),
+        license=license_name,
+        is_fork=bool(repo.get("fork", False)),
+        is_archived=bool(repo.get("archived", False)),
+        languages=languages,
+        commit_activity=commit_activity,
+        has_tests=tree_flags["has_tests"],
+        has_workflows=tree_flags["has_workflows"],
+        has_dockerfile=tree_flags["has_dockerfile"],
+    )
+
+
+def _fetch_merged_prs_count(
+    session: requests.Session,
+    username: str,
+    token: str | None,
+) -> int:
+    url = urljoin(
+        GITHUB_API_BASE,
+        "/search/issues",
+    )
+    params = {
+        "q": f"author:{username} type:pr is:merged",
+        "per_page": 1,
+    }
+    try:
+        data = _get(session, url, token, params)
+        if isinstance(data, dict):
+            return int(data.get("total_count", 0) or 0)
+    except (GitHubError, requests.RequestException):
+        pass
+    return 0
+
+
+def fetch_profile(
+    username: str,
+    token: str | None = None,
+    no_cache: bool = False,
+    progress_callback: callable | None = None,
+) -> ProfileData:
+    """Fetch and normalize all data needed for a UTOPIA report."""
+    if no_cache:
+        _clear_cache(username)
+
+    cached = _load_cache(username)
+    if cached is not None:
+        profile = _deserialize_profile(cached)
+        if progress_callback:
+            progress_callback("Using cached data", 1.0)
+        return profile
+
+    session = requests.Session()
+    try:
+        user_url = f"{GITHUB_API_BASE}/users/{username}"
+        user = _get(session, user_url, token)
+        if not isinstance(user, dict):
+            raise GitHubError("Unexpected response from GitHub user endpoint.")
+
+        repos_url = user.get("repos_url", "")
+        repos_raw = _get_all_pages(
+            session,
+            repos_url,
+            token,
+            {"type": "owner", "sort": "pushed", "direction": "desc"},
+        )
+
+        repos: list[RepoData] = []
+        total = len(repos_raw)
+        for idx, repo in enumerate(repos_raw):
+            if progress_callback:
+                progress_callback(f"Analyzing {repo.get('name', 'repo')}", idx / max(total, 1))
+            try:
+                repos.append(_fetch_repo_details(session, username, repo["name"], token))
+            except GitHubError as exc:
+                if exc.status_code in (403, 429):
+                    raise
+                continue
+
+        if progress_callback:
+            progress_callback("Finalizing analysis", 1.0)
+
+        merged_prs = _fetch_merged_prs_count(session, username, token)
+
+        total_languages: dict[str, int] = {}
+        for repo in repos:
+            for lang, bytes_count in repo.languages.items():
+                total_languages[lang] = total_languages.get(lang, 0) + bytes_count
+
+        profile = ProfileData(
+            username=username,
+            name=user.get("name"),
+            bio=user.get("bio"),
+            followers=int(user.get("followers", 0) or 0),
+            following=int(user.get("following", 0) or 0),
+            public_repos_count=int(user.get("public_repos", 0) or 0),
+            created_at=user.get("created_at", ""),
+            repos=repos,
+            merged_prs_count=merged_prs,
+            total_languages=total_languages,
+        )
+
+        _save_cache(username, _serialize_profile(profile))
+        return profile
+    finally:
+        session.close()
+
+
+def _serialize_profile(profile: ProfileData) -> dict[str, Any]:
+    return {
+        "username": profile.username,
+        "name": profile.name,
+        "bio": profile.bio,
+        "followers": profile.followers,
+        "following": profile.following,
+        "public_repos_count": profile.public_repos_count,
+        "created_at": profile.created_at,
+        "repos": [
+            {
+                "name": r.name,
+                "description": r.description,
+                "language": r.language,
+                "stars": r.stars,
+                "forks": r.forks,
+                "size_kb": r.size_kb,
+                "created_at": r.created_at,
+                "updated_at": r.updated_at,
+                "pushed_at": r.pushed_at,
+                "has_readme": r.has_readme,
+                "readme_word_count": r.readme_word_count,
+                "open_issues_count": r.open_issues_count,
+                "topics": r.topics,
+                "license": r.license,
+                "is_fork": r.is_fork,
+                "is_archived": r.is_archived,
+                "languages": r.languages,
+                "commit_activity": r.commit_activity,
+                "has_tests": r.has_tests,
+                "has_workflows": r.has_workflows,
+                "has_dockerfile": r.has_dockerfile,
+            }
+            for r in profile.repos
+        ],
+        "merged_prs_count": profile.merged_prs_count,
+        "total_languages": profile.total_languages,
+    }
+
+
+def _deserialize_profile(data: dict[str, Any]) -> ProfileData:
+    repos = [
+        RepoData(
+            name=r["name"],
+            description=r.get("description"),
+            language=r.get("language"),
+            stars=r["stars"],
+            forks=r["forks"],
+            size_kb=r["size_kb"],
+            created_at=r["created_at"],
+            updated_at=r["updated_at"],
+            pushed_at=r.get("pushed_at"),
+            has_readme=r["has_readme"],
+            readme_word_count=r["readme_word_count"],
+            open_issues_count=r["open_issues_count"],
+            topics=r["topics"],
+            license=r.get("license"),
+            is_fork=r["is_fork"],
+            is_archived=r["is_archived"],
+            languages=r.get("languages", {}),
+            commit_activity=r.get("commit_activity", []),
+            has_tests=r.get("has_tests", False),
+            has_workflows=r.get("has_workflows", False),
+            has_dockerfile=r.get("has_dockerfile", False),
+        )
+        for r in data.get("repos", [])
+    ]
+    return ProfileData(
+        username=data["username"],
+        name=data.get("name"),
+        bio=data.get("bio"),
+        followers=data["followers"],
+        following=data["following"],
+        public_repos_count=data["public_repos_count"],
+        created_at=data["created_at"],
+        repos=repos,
+        merged_prs_count=data.get("merged_prs_count", 0),
+        total_languages=data.get("total_languages", {}),
+    )

utopia/improvement_plan.py

@@ -0,0 +1,120 @@
+"""Generate a personalized 30-day improvement plan based on weak scoring categories."""
+
+from __future__ import annotations
+
+import random
+from dataclasses import dataclass, field
+
+from utopia.github_client import ProfileData, RepoData
+from utopia.scoring import ScoreBreakdown, UtopiaScore, weakest_categories
+
+
+# ---------------------------------------------------------------------------
+# Action templates mapped to scoring categories.
+# ---------------------------------------------------------------------------
+ACTION_TEMPLATES: dict[str, list[str]] = {
+    "project_quality": [
+        "Refactor your most-starred project and add missing features",
+        "Add a CI pipeline to at least one project",
+        "Clean up empty or fork-only repos that dilute your profile",
+    ],
+    "activity": [
+        "Commit to at least 3 repos this week",
+        "Resume work on {stale_repo}",
+        "Close stale issues and merge small PRs to show momentum",
+    ],
+    "documentation": [
+        "Improve README files across your top repos",
+        "Add usage examples and screenshots to {top_repo}",
+        "Write a one-paragraph project summary for every repo missing a description",
+    ],
+    "tech_diversity": [
+        "Try a small project in {missing_language}",
+        "Add a new language or framework to an existing side project",
+        "Contribute to an open-source project outside your main stack",
+    ],
+    "community_impact": [
+        "Add topics/tags to your top repos for discoverability",
+        "Write a blog post about {top_repo}",
+        "Share your best project on social media or developer forums",
+    ],
+    "open_source": [
+        "Open a pull request to a project you use",
+        "Review and merge community PRs in your own repos",
+        "Turn a useful script into a real open-source project",
+    ],
+}
+
+
+# ---------------------------------------------------------------------------
+# Personalization helpers
+# ---------------------------------------------------------------------------
+def _most_starred_repo(repos: list[RepoData]) -> str | None:
+    original = [r for r in repos if not r.is_fork]
+    if not original:
+        return None
+    return max(original, key=lambda r: r.stars).name
+
+
+def _most_stale_repo(repos: list[RepoData]) -> str | None:
+    from datetime import datetime, timezone
+
+    def days_since(repo: RepoData) -> float:
+        if not repo.pushed_at:
+            return float("inf")
+        try:
+            dt = datetime.fromisoformat(repo.pushed_at.replace("Z", "+00:00"))
+            return (datetime.now(timezone.utc) - dt).total_seconds() / 86400
+        except (ValueError, TypeError):
+            return float("inf")
+
+    original = [r for r in repos if not r.is_fork]
+    if not original:
+        return None
+    return max(original, key=days_since).name
+
+
+def _missing_common_language(profile: ProfileData) -> str:
+    """Suggest a common language not already prominent in the profile."""
+    common = ["Python", "JavaScript", "TypeScript", "Go", "Rust", "Java"]
+    existing = {lang.lower() for lang in profile.total_languages}
+    for lang in common:
+        if lang.lower() not in existing:
+            return lang
+    return random.choice(common)
+
+
+@dataclass
+class ImprovementPlan:
+    weeks: list[list[str]] = field(default_factory=list)
+
+
+def generate_improvement_plan(
+    profile: ProfileData,
+    score: UtopiaScore,
+) -> ImprovementPlan:
+    """Build a 4-week plan focusing on the weakest scoring categories."""
+    repos = profile.repos
+    top_repo = _most_starred_repo(repos) or "your best project"
+    stale_repo = _most_stale_repo(repos) or "an old project"
+    missing_language = _missing_common_language(profile)
+
+    weak = weakest_categories(score, n=4)
+    if len(weak) < 4:
+        # Pad with remaining categories if fewer than 4 are weak.
+        used = {b.category for b in weak}
+        remaining = [b for b in score.breakdown if b.category not in used]
+        weak.extend(remaining[: 4 - len(weak)])
+
+    weeks: list[list[str]] = []
+    for breakdown in weak[:4]:
+        templates = list(ACTION_TEMPLATES.get(breakdown.category, []))
+        random.shuffle(templates)
+        action = templates[0].format(
+            top_repo=top_repo,
+            stale_repo=stale_repo,
+            missing_language=missing_language,
+        )
+        weeks.append([action])
+
+    return ImprovementPlan(weeks=weeks)