skip-trace 0.1.0__py3-none-any.whl

skip_trace/collectors/github.py

@@ -0,0 +1,241 @@
+# skip_trace/collectors/github.py
+from __future__ import annotations
+
+import datetime
+import logging
+from typing import Dict, List, Optional, Set
+from urllib.parse import urlparse
+
+from bs4 import BeautifulSoup
+from github import Github, GithubException
+from github.NamedUser import NamedUser
+
+from ..analysis.evidence import generate_evidence_id
+from ..config import CONFIG
+from ..exceptions import CollectorError, NetworkError
+from ..schemas import EvidenceKind, EvidenceRecord, EvidenceSource
+from ..utils import http_client
+
+logger = logging.getLogger(__name__)
+
+_github_client: Optional[Github] = None
+
+
+def get_github_client() -> Optional[Github]:
+    """
+    Initializes and returns a singleton PyGithub client instance.
+
+    Uses the GITHUB_TOKEN from the config if available.
+
+    Returns:
+        An authenticated Github client instance, or None if the token is missing.
+    """
+    global _github_client
+    if _github_client:
+        return _github_client
+
+    github_config = CONFIG.get("github", {})
+    api_key = github_config.get("api_key")
+
+    if not api_key:
+        logger.warning(
+            "GITHUB_TOKEN not found in environment. GitHub API requests will be unauthenticated and rate-limited."
+        )
+        _github_client = Github()
+    else:
+        logger.debug("Authenticating to GitHub API with token.")
+        _github_client = Github(api_key)
+
+    return _github_client
+
+
+def _parse_repo_url(url: str) -> Optional[str]:
+    """Parses a GitHub URL to extract the 'owner/repo' string."""
+    try:
+        parsed = urlparse(url)
+        if parsed.hostname and "github.com" in parsed.hostname:
+            path = parsed.path.strip("/")
+            if ".git" in path:
+                path = path.replace(".git", "")
+            if len(path.split("/")) >= 2:
+                return "/".join(path.split("/")[:2])
+    except Exception:
+        pass  # nosec # noqa
+    logger.debug(f"Could not parse a valid GitHub repository from URL: {url}")
+    return None
+
+
+def _scrape_socials_from_html(html_url: str) -> Dict[str, str]:
+    """Scrapes a user's GitHub profile page for social media and blog links."""
+    contacts: Dict[str, str] = {}
+    try:
+        logger.debug(f"Scraping GitHub profile HTML page for social links: {html_url}")
+        response = http_client.make_request(html_url)
+        soup = BeautifulSoup(response.text, "html.parser")
+
+        # Find all links within the user profile section
+        # profile_links = soup.select('div[data-bio-টারের] a[href], ul.vcard-details a[href]')
+        # brittle!
+        profile_links = soup.select(
+            "div.user-profile-bio a[href], ul.vcard-details a[href]"
+        )
+        for link in profile_links:
+            href = link.get("href")
+            if not href:
+                continue
+
+            # Simple heuristic mapping of domain to platform name
+            if "linkedin.com/in" in href and "linkedin" not in contacts:
+                contacts["linkedin"] = href  # type: ignore[assignment]
+            elif (
+                "mastodon.social" in href or "fosstodon.org" in href
+            ) and "mastodon" not in contacts:
+                contacts["mastodon"] = href  # type: ignore[assignment]
+            elif "twitter.com" in href and "twitter" not in contacts:
+                # Prefer the twitter_username from API, but take this if needed
+                contacts["twitter"] = href  # type: ignore[assignment]
+            elif "blog." in href or "medium.com" in href and "blog" not in contacts:
+                contacts["blog"] = href  # type: ignore[assignment]
+
+    except NetworkError as e:
+        logger.warning(f"Could not scrape GitHub profile page {html_url}: {e}")
+    except Exception as e:
+        logger.error(f"Error during social scraping for {html_url}: {e}")
+
+    return contacts
+
+
+def _create_records_from_user_profile(user: NamedUser) -> List[EvidenceRecord]:
+    """Creates evidence records from a full GitHub user profile."""
+    records = []
+    name = user.name or user.login
+    now = datetime.datetime.now(datetime.timezone.utc)
+
+    # Evidence for company affiliation
+    if user.company:
+        value: dict[str, str | None] = {"user_name": name, "company_name": user.company}
+        records.append(
+            EvidenceRecord(
+                id=generate_evidence_id(
+                    EvidenceSource.REPO,
+                    EvidenceKind.USER_COMPANY,
+                    user.html_url,
+                    str(value),
+                    name,
+                    hint="company",
+                ),
+                source=EvidenceSource.REPO,
+                locator=user.html_url,
+                kind=EvidenceKind.USER_COMPANY,
+                value=value,
+                observed_at=now,
+                confidence=0.8,
+                notes=f"User '{name}' lists company affiliation as '{user.company}'.",
+            )
+        )
+
+    # Evidence for other profile contacts
+    profile_contacts = {
+        "email": user.email,
+        "twitter": (
+            f"https://twitter.com/{user.twitter_username}"
+            if user.twitter_username
+            else None
+        ),
+        "blog": user.blog,
+    }
+
+    # Scrape HTML for links not available in the API
+    scraped_contacts = _scrape_socials_from_html(user.html_url)
+    # The scraped contacts take precedence if they exist
+    profile_contacts.update(scraped_contacts)
+
+    # Filter out empty values
+    profile_contacts = {k: v for k, v in profile_contacts.items() if v}
+    if profile_contacts:
+        profile_info = {"user_name": name, "contacts": profile_contacts}
+        records.append(
+            EvidenceRecord(
+                id=generate_evidence_id(
+                    EvidenceSource.REPO,
+                    EvidenceKind.USER_PROFILE,
+                    user.html_url,
+                    str(profile_info),  # TODO: stringify this better?
+                    name,
+                    hint="profile",
+                ),
+                source=EvidenceSource.REPO,
+                locator=user.html_url,
+                kind=EvidenceKind.USER_PROFILE,
+                value=profile_info,
+                observed_at=now,
+                confidence=0.9,
+                notes=f"Found contact details on GitHub user profile for '{name}'.",
+            )
+        )
+
+    return records
+
+
+def extract_from_repo_url(repo_url: str) -> List[EvidenceRecord]:
+    """
+    Extracts ownership evidence from a GitHub repository URL.
+
+    Args:
+        repo_url: The full URL of the GitHub repository.
+
+    Returns:
+        A list of EvidenceRecord objects.
+    """
+    evidence = []
+    processed_users: Set[str] = set()
+    repo_full_name = _parse_repo_url(repo_url)
+    if not repo_full_name:
+        return []
+
+    client = get_github_client()
+    if not client:
+        return []
+
+    try:
+        logger.debug(f"Fetching repository details for '{repo_full_name}'")
+        repo = client.get_repo(repo_full_name)
+
+        # 1. Process the repository owner's full profile
+        owner = repo.owner
+        if owner.login not in processed_users:
+            evidence.extend(_create_records_from_user_profile(owner))
+            processed_users.add(owner.login)
+
+        # 2. Process recent commit authors' full profiles
+        logger.debug(f"Fetching recent commits for '{repo_full_name}'")
+        commits = repo.get_commits()
+        # Limit to the most recent 25 commits to avoid excessive API usage
+        for i, commit in enumerate(commits):
+            if i >= 10:  # Limit to recent 10 to reduce API calls
+                break
+            # commit.author is a full NamedUser if available
+            if (
+                isinstance(commit.author, NamedUser)
+                and commit.author.login not in processed_users
+            ):
+                evidence.extend(_create_records_from_user_profile(commit.author))
+                processed_users.add(commit.author.login)
+
+    except GithubException as e:
+        logger.error(f"GitHub API error for '{repo_full_name}': {e.status} {e.data}")
+        raise CollectorError(
+            f"Could not access GitHub repository '{repo_full_name}'"
+        ) from e
+    except Exception as e:
+        logger.error(
+            f"An unexpected error occurred while processing GitHub repo '{repo_full_name}': {e}"
+        )
+        raise CollectorError(
+            f"Unexpected error for GitHub repo '{repo_full_name}'"
+        ) from e
+
+    logger.info(
+        f"Extracted {len(evidence)} evidence records from GitHub user profiles for repo '{repo_full_name}'."
+    )
+    return evidence

skip_trace/collectors/package_files.py

@@ -0,0 +1,150 @@
+# skip_trace/collectors/package_files.py
+from __future__ import annotations
+
+import logging
+import os
+import shutil
+import tarfile
+import zipfile
+from typing import Any, Dict, List, Optional
+
+from ..analysis import source_scanner
+from ..exceptions import CollectorError, NetworkError
+from ..schemas import EvidenceRecord
+from ..utils import http_client
+from ..utils.safe_targz import safe_extract_auto
+
+logger = logging.getLogger(__name__)
+PACKAGE_DOWNLOAD_DIR = ".packages"
+
+
+def _ensure_download_dir():
+    """Ensures the package download directory and .gitignore exist."""
+    os.makedirs(PACKAGE_DOWNLOAD_DIR, exist_ok=True)
+    gitignore_path = os.path.join(PACKAGE_DOWNLOAD_DIR, ".gitignore")
+    if not os.path.exists(gitignore_path):
+        with open(gitignore_path, "w", encoding="utf-8") as f:
+            f.write("*\n")
+
+
+def _find_download_url(metadata: Dict[str, Any]) -> Optional[str]:
+    """Finds the best distribution URL from PyPI metadata."""
+    urls = metadata.get("urls", [])
+    if not urls:
+        return None
+
+    # Prioritize wheels, then sdist, then anything else
+    wheel_url = None
+    sdist_url = None
+    for url_info in urls:
+        packagetype = url_info.get("packagetype")
+        if packagetype == "bdist_wheel":
+            wheel_url = url_info.get("url")
+        elif packagetype == "sdist":
+            sdist_url = url_info.get("url")
+
+    # Return in order of preference: wheel, then sdist, then fallback
+    return wheel_url or sdist_url or (urls[0].get("url") if urls else None)
+
+
+def collect_from_package_files(metadata: Dict[str, Any]) -> List[EvidenceRecord]:
+    """
+    Downloads, extracts, and scans a package's files for evidence.
+
+    Args:
+        metadata: The PyPI JSON metadata for the package.
+
+    Returns:
+        A list of EvidenceRecord objects found within the package files.
+    """
+    info = metadata.get("info", {})
+    package_name = info.get("name", "unknown")
+    package_version = info.get("version", "latest")
+    logger.info(f"Starting file analysis for {package_name} v{package_version}")
+
+    download_url = _find_download_url(metadata)
+    if not download_url:
+        logger.warning(
+            f"No download URL found for {package_name}. Skipping file analysis."
+        )
+        return []
+
+    _ensure_download_dir()
+    filename = os.path.basename(download_url)
+    download_path = os.path.join(PACKAGE_DOWNLOAD_DIR, filename)
+
+    # Download the file if it doesn't already exist
+    if not os.path.exists(download_path):
+        logger.info(f"Downloading {filename} from {download_url}")
+        try:
+            with http_client.get_client().stream("GET", download_url) as response:
+                response.raise_for_status()
+                with open(download_path, "wb") as f:
+                    for chunk in response.iter_bytes():
+                        f.write(chunk)
+        except (
+            NetworkError,
+            http_client.httpx.RequestError,
+            http_client.httpx.HTTPStatusError,
+        ) as e:
+            raise CollectorError(f"Failed to download package {filename}: {e}") from e
+
+    # Determine the persistent extraction directory path from the filename
+    base_filename = filename
+    for ext in [".whl", ".zip", ".tar.gz", ".tgz", ".tar.bz2"]:
+        if base_filename.endswith(ext):
+            base_filename = base_filename[: -len(ext)]
+            break
+    extract_dir = os.path.join(PACKAGE_DOWNLOAD_DIR, base_filename)
+
+    # Extract the archive ONLY if the destination directory doesn't already exist
+    if not os.path.exists(extract_dir):
+        logger.info(f"Extracting {download_path} to {extract_dir}")
+        os.makedirs(extract_dir, exist_ok=True)
+        try:
+            if download_path.endswith((".whl", ".zip")):
+                with zipfile.ZipFile(download_path, "r") as zf:  # nosec # noqa
+                    zf.extractall(extract_dir)  # nosec # noqa
+            elif download_path.endswith(
+                (".tar.gz", ".tgz", ".tar.bz2", ".tar.xz", ".tar")
+            ):
+                safe_extract_auto(download_path, extract_dir)
+            # elif download_path.endswith((".tar.gz", ".tgz")):
+            #     with tarfile.open(download_path, "r:gz") as tf:  # nosec # noqa
+            #         tf.extractall(extract_dir)  # nosec # noqa
+            # elif download_path.endswith(".tar.bz2"):
+            #     with tarfile.open(download_path, "r:bz2") as tf:  # nosec # noqa
+            #         tf.extractall(extract_dir)  # nosec # noqa
+            else:
+                logger.warning(
+                    f"Unsupported archive format for {filename}. Skipping file scan."
+                )
+                shutil.rmtree(extract_dir)  # Clean up the empty dir
+                return []
+        except (zipfile.BadZipFile, tarfile.TarError, PermissionError) as e:
+            logger.error(f"Failed to extract archive {download_path}: {e}")
+            # Clean up potentially corrupted extraction on error
+            shutil.rmtree(extract_dir, ignore_errors=True)
+            return []
+    else:
+        logger.info(f"Using cached package files from {extract_dir}")
+
+    # Determine the actual directory to scan (handles sdists with a single top-level folder)
+    scan_target_dir = extract_dir
+    # This logic applies to sdists, which often have a root folder. Wheels do not.
+    if filename.endswith((".tar.gz", ".tgz", ".tar.bz2")):
+        try:
+            dir_contents = os.listdir(extract_dir)
+            if len(dir_contents) == 1 and os.path.isdir(
+                os.path.join(extract_dir, dir_contents[0])
+            ):
+                scan_target_dir = os.path.join(extract_dir, dir_contents[0])
+        except FileNotFoundError:
+            logger.error(
+                f"Extraction directory {extract_dir} not found after apparent success. Check permissions."
+            )
+            return []
+
+    locator_prefix = f"{package_name}-{package_version}"
+    evidence = source_scanner.scan_directory(scan_target_dir, locator_prefix)
+    return evidence

skip_trace/collectors/pypi.py

@@ -0,0 +1,158 @@
+# skip_trace/collectors/pypi.py
+from __future__ import annotations
+
+import datetime
+import logging
+from typing import Any, Dict, List, Optional, Set
+
+from bs4 import BeautifulSoup
+
+from ..analysis.evidence import extract_from_pypi as analyze_pypi_metadata
+from ..analysis.evidence import generate_evidence_id
+from ..exceptions import NetworkError, NoEvidenceError
+from ..schemas import EvidenceKind, EvidenceRecord, EvidenceSource
+from ..utils import http_client
+
+logger = logging.getLogger(__name__)
+PYPI_JSON_API_URL = "https://pypi.org/pypi"
+PYPI_PROJECT_URL = "https://pypi.org/project"
+
+
+def fetch_package_metadata(
+    package_name: str, version: Optional[str] = None
+) -> Dict[str, Any]:
+    """
+    Fetches package metadata from the PyPI JSON API.
+
+    :param package_name: The name of the package.
+    :param version: The optional specific version of the package.
+    :raises NoEvidenceError: If the package is not found (404).
+    :raises NetworkError: For other network or HTTP errors.
+    :return: A dictionary containing the package's JSON metadata.
+    """
+    if version:
+        url = f"{PYPI_JSON_API_URL}/{package_name}/{version}/json"
+    else:
+        url = f"{PYPI_JSON_API_URL}/{package_name}/json"
+
+    try:
+        response = http_client.make_request(url)
+        return response.json()
+    except NetworkError as e:
+        if "404" in str(e):
+            raise NoEvidenceError(
+                f"Package '{package_name}'"
+                f"{f' version {version}' if version else ''} not found on PyPI."
+            ) from e
+        raise
+
+
+def _scrape_user_profile_url(package_name: str) -> Optional[str]:
+    """Scrapes the PyPI project page to find the user profile URL."""
+    try:
+        url = f"{PYPI_PROJECT_URL}/{package_name}/"
+        logger.debug(f"Scraping project page for user link: {url}")
+        response = http_client.make_request(url)
+        soup = BeautifulSoup(response.text, "html.parser")
+
+        # The user link is typically in a `p` tag with the class 'sidebar-section__user-gravatar-text'
+        user_link = soup.find("a", href=lambda href: href and href.startswith("/user/"))
+        if user_link and user_link.has_attr("href"):
+            profile_url = f"https://pypi.org{user_link['href']}"
+            logger.debug(f"Found user profile URL: {profile_url}")
+            return profile_url
+    except NetworkError as e:
+        logger.warning(f"Could not scrape project page for '{package_name}': {e}")
+    return None
+
+
+def _fetch_other_package_urls(user_profile_url: str) -> Set[str]:
+    """Scrapes a user's profile page to find their other packages."""
+    packages = set()
+    try:
+        logger.debug(f"Scraping user profile for other packages: {user_profile_url}")
+        response = http_client.make_request(user_profile_url)
+        soup = BeautifulSoup(response.text, "html.parser")
+
+        # Links to packages are in a 'package-snippet' class
+        for link in soup.find_all("a", class_="package-snippet"):
+            if link.has_attr("href") and link["href"].startswith("/project/"):  # type: ignore[union-attr]
+                packages.add(link["href"].split("/")[2])  # type: ignore[union-attr]
+        logger.debug(f"Found {len(packages)} other packages by user.")
+        return packages
+    except NetworkError as e:
+        logger.warning(f"Could not scrape user profile page '{user_profile_url}': {e}")
+    return packages
+
+
+def cross_reference_by_user(package_name: str) -> List[EvidenceRecord]:
+    """
+    Finds other packages by the same user to uncover more evidence.
+    Also creates an evidence record for the PyPI user itself.
+
+    Args:
+        package_name: The name of the starting package.
+
+    Returns:
+        A list of new EvidenceRecord objects found from related packages.
+    """
+    new_evidence: List[EvidenceRecord] = []
+    profile_url = _scrape_user_profile_url(package_name)
+
+    # --- NEW: Always create evidence for the PyPI user if found ---
+    if profile_url:
+        try:
+            username = profile_url.strip("/").rsplit("/", maxsplit=1)[-1]
+            value = {"name": username, "url": profile_url}
+            record = EvidenceRecord(
+                id=generate_evidence_id(
+                    EvidenceSource.PYPI,
+                    EvidenceKind.PYPI_USER,
+                    profile_url,
+                    str(value),
+                    username,
+                ),
+                source=EvidenceSource.PYPI,
+                locator=profile_url,
+                kind=EvidenceKind.PYPI_USER,
+                value=value,
+                observed_at=datetime.datetime.now(datetime.timezone.utc),
+                confidence=0.50,  # This is a strong signal
+                notes=f"Package is published by PyPI user '{username}'.",
+            )
+            new_evidence.append(record)
+            logger.debug(f"Created evidence record for PyPI user '{username}'.")
+        except (IndexError, TypeError) as e:
+            logger.warning(
+                f"Could not parse username from profile URL '{profile_url}': {e}"
+            )
+
+    # --- Continue with existing cross-referencing logic ---
+    if not profile_url:
+        return []
+
+    other_packages = _fetch_other_package_urls(profile_url)
+    if not other_packages:
+        return new_evidence
+
+    # new_evidence: List[EvidenceRecord] = []
+    # Limit to analyzing a few other packages to avoid excessive requests
+    for other_pkg in list(other_packages)[:3]:
+        if other_pkg == package_name:
+            continue
+        try:
+            logger.info(f"Cross-referencing with related package: '{other_pkg}'")
+            metadata = fetch_package_metadata(other_pkg)
+            # We only care about strong signals (like repo URLs) from other packages
+            evidence, _ = analyze_pypi_metadata(metadata)
+            for record in evidence:
+                if "repository URL" in record.notes:
+                    new_evidence.append(record)
+        except NoEvidenceError:
+            logger.debug(f"Skipping related package '{other_pkg}', not found.")
+            continue
+
+    logger.info(
+        f"Found {len(new_evidence)} new evidence records via user cross-reference."
+    )
+    return new_evidence