skip-trace 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

skip_trace/collectors/urls.py

@@ -0,0 +1,96 @@
+# skip_trace/collectors/urls.py
+from __future__ import annotations
+
+import datetime
+import logging
+import os
+from typing import List, Set
+
+from bs4 import BeautifulSoup
+
+from ..analysis.content_scanner import scan_text
+from ..analysis.evidence import generate_evidence_id
+from ..schemas import EvidenceKind, EvidenceRecord, EvidenceSource
+from ..utils import http_client
+from ..utils.cache import get_cached_data, set_cached_data
+
+logger = logging.getLogger(__name__)
+URL_CACHE_DIR = ".urls"
+
+
+def _ensure_download_dir():
+    """Ensures the URL cache directory and .gitignore exist."""
+    os.makedirs(URL_CACHE_DIR, exist_ok=True)
+    gitignore_path = os.path.join(URL_CACHE_DIR, ".gitignore")
+    if not os.path.exists(gitignore_path):
+        with open(gitignore_path, "w", encoding="utf-8") as f:
+            f.write("*\n")
+
+
+def collect_from_urls(urls: Set[str]) -> List[EvidenceRecord]:
+    """
+    Downloads, caches, and scans a list of URLs for evidence.
+
+    Args:
+        urls: A set of unique URLs to scan.
+
+    Returns:
+        A list of EvidenceRecord objects from the URLs.
+    """
+    _ensure_download_dir()
+    all_evidence: List[EvidenceRecord] = []
+    now = datetime.datetime.now(datetime.timezone.utc)
+
+    for url in urls:
+        logger.info(f"Analyzing URL: {url}")
+        cached_data = get_cached_data("url", url)
+
+        status_code = -1
+        content = ""
+
+        if cached_data:
+            logger.debug(f"Using cached content for {url}")
+            status_code = cached_data.get("status_code", -1)
+            content = cached_data.get("content", "")
+        else:
+            response = http_client.make_request_safe(url)
+            if response:
+                status_code = response.status_code
+                if status_code == 200:
+                    content = response.text
+                set_cached_data(
+                    "url", url, {"status_code": status_code, "content": content}
+                )
+            else:
+                set_cached_data(
+                    "url", url, {"status_code": -1, "content": ""}
+                )  # Cache connection failure
+
+        # Create an evidence record for the URL status itself
+        status_value = {"status_code": status_code}
+        status_record = EvidenceRecord(
+            id=generate_evidence_id(
+                EvidenceSource.URL, EvidenceKind.URL_STATUS, url, str(status_value), url
+            ),
+            source=EvidenceSource.URL,
+            locator=url,
+            kind=EvidenceKind.URL_STATUS,
+            value=status_value,
+            observed_at=now,
+            confidence=0.0,  # This is informational, not for scoring
+            notes=f"HTTP status for {url} was {status_code}.",
+        )
+        all_evidence.append(status_record)
+
+        if content:
+            try:
+                soup = BeautifulSoup(content, "html.parser")
+                text_content = soup.get_text(separator=" ", strip=True)
+                url_evidence = scan_text(text_content, url, EvidenceSource.URL)
+                if url_evidence:
+                    logger.info(f"Found {len(url_evidence)} evidence records on {url}")
+                    all_evidence.extend(url_evidence)
+            except Exception as e:
+                logger.warning(f"Could not parse or scan HTML from {url}: {e}")
+
+    return all_evidence

skip_trace/m.py

@@ -0,0 +1,287 @@
+# skip_trace/main.py
+from __future__ import annotations
+
+import argparse
+import dataclasses
+import json
+import logging
+import sys
+from typing import Set
+from urllib.parse import urlparse
+
+import tldextract
+from rich.logging import RichHandler
+
+from . import schemas
+from .analysis import evidence as evidence_analyzer
+from .analysis import scoring
+from .collectors import (
+    github,
+    github_files,
+    package_files,
+    pypi,
+    pypi_attestations,
+    urls,
+    whois,
+)
+from .config import CONFIG
+from .exceptions import CollectorError, NetworkError, NoEvidenceError
+from .reporting import json_reporter, md_reporter
+
+# Create a logger instance for this module
+logger = logging.getLogger(__name__)
+
+
+def setup_logging(level: str = "INFO"):
+    """Configures the application's logger.
+
+    Args:
+        level: The minimum logging level to display (e.g., "INFO", "DEBUG").
+    """
+    logging.basicConfig(
+        level=level,
+        format="%(message)s",
+        datefmt="[%X]",
+        handlers=[RichHandler(rich_tracebacks=True, show_path=False)],
+    )
+
+
+def run_who_owns(args: argparse.Namespace) -> int:
+    """Handler for the 'who-owns' command."""
+    logger.info(f"Executing 'who-owns' for package: {args.package}")
+
+    try:
+        # Collect initial data from PyPI
+        metadata = pypi.fetch_package_metadata(args.package, args.version)
+        package_name = metadata.get("info", {}).get("name", args.package)
+        package_version = metadata.get("info", {}).get("version")
+        logger.info(
+            f"Successfully fetched metadata for {package_name} v{package_version}"
+        )
+
+        # Analyze primary package metadata
+        evidence_records, pypi_maintainers = evidence_analyzer.extract_from_pypi(
+            metadata
+        )
+
+        logger.info(f"Evidence records so far {len(evidence_records)} -- pypi metadata")
+
+        # Check for PyPI attestations
+        attestation_evidence = pypi_attestations.collect(metadata)
+        evidence_records.extend(attestation_evidence)
+        logger.info(
+            f"Evidence records so far {len(evidence_records)} -- collected from PyPI attestations"
+        )
+
+        # Analyze package contents for deep evidence
+        try:
+            package_files_evidence = package_files.collect_from_package_files(metadata)
+            evidence_records.extend(package_files_evidence)
+            logger.info(
+                f"Evidence records so far {len(evidence_records)} -- collected from source code in package"
+            )
+        except CollectorError as e:
+            logger.warning(f"Could not analyze package files for {package_name}: {e}")
+
+        # Cross-Reference for more PyPI evidence
+        cross_ref_evidence = pypi.cross_reference_by_user(package_name)
+        evidence_records.extend(cross_ref_evidence)
+        logger.info(
+            f"Evidence records so far {len(evidence_records)} -- user cross ref"
+        )
+
+        # Fetch evidence from code repositories found in PyPI evidence
+        repo_urls = set()
+        for record in evidence_records:
+            if (
+                record.source == schemas.EvidenceSource.PYPI
+                and record.kind == schemas.EvidenceKind.ORGANIZATION
+            ):
+                url = record.value.get("url")
+                if url and "github.com" in url:
+                    repo_urls.add(url)
+
+        for url in repo_urls:
+            logger.info(f"Analyzing GitHub repository: {url}")
+            try:
+                github_evidence = github.extract_from_repo_url(url)
+                evidence_records.extend(github_evidence)
+                logger.info(
+                    f"Evidence records so far {len(evidence_records)} -- collected from github"
+                )
+            except CollectorError as e:
+                logger.warning(f"Could not fully analyze GitHub repo {url}: {e}")
+
+            # NEW: Collect evidence from GitHub files (SECURITY.md, FUNDING.yml, contributors)
+            try:
+                github_files_evidence = github_files.collect_from_repo_url(url)
+                evidence_records.extend(github_files_evidence)
+                logger.info(
+                    f"Evidence records so far {len(evidence_records)} -- collected from github files"
+                )
+            except CollectorError as e:
+                logger.warning(f"Could not collect GitHub files for {url}: {e}")
+
+        # Extract domains and perform WHOIS lookups
+        domains_to_check: Set[str] = set()
+        urls_to_scan: Set[str] = set()
+        ignored_domains = set(CONFIG.get("whois_ignored_domains", []))
+
+        for record in evidence_records:
+            # Extract domains for WHOIS
+            if email := record.value.get("email"):
+                if "@" in email:
+                    domain = email.split("@")[1]
+                    if domain not in ignored_domains:
+                        domains_to_check.add(domain)
+            # Extract domains and full URLs
+            if url := record.value.get("url"):
+                urls_to_scan.add(url)
+
+                # If it's a GitHub repo URL, also scan the user/org URL.
+                try:
+                    parsed_url = urlparse(url)
+                    if "github.com" in parsed_url.netloc:
+                        path_parts = [p for p in parsed_url.path.split("/") if p]
+                        if len(path_parts) >= 2:  # e.g., /owner/repo
+                            user_url = f"{parsed_url.scheme}://{parsed_url.netloc}/{path_parts[0]}"
+                            urls_to_scan.add(user_url)
+                except Exception as e:
+                    logger.debug(f"Could not parse user URL from {url}: {e}")
+
+                # Gather domains from URLs for WHOIS, respecting the ignore list
+                extracted = tldextract.extract(url)
+                if extracted.registered_domain:
+                    if extracted.registered_domain not in ignored_domains:
+                        domains_to_check.add(extracted.registered_domain)
+                        urls_to_scan.add(url)
+
+        # Perform WHOIS lookups
+        logger.info(f"Domains for WHOIS: {', '.join(sorted(list(domains_to_check)))}")
+        if domains_to_check:
+            for domain in domains_to_check:
+                try:
+                    evidence_records.extend(whois.collect_from_domain(domain))
+                    logger.info(
+                        f"Evidence records so far {len(evidence_records)} -- collected from domains/whois"
+                    )
+                except CollectorError as e:
+                    logger.warning(f"WHOIS failed for {domain}: {e}")
+
+        # Scan homepage URLs
+        logger.info(f"URLs to scan: {', '.join(sorted(list(urls_to_scan)))}")
+        if urls_to_scan:
+            try:
+                evidence_records.extend(urls.collect_from_urls(urls_to_scan))
+                logger.info(
+                    f"Evidence records so far {len(evidence_records)} -- collected from urls"
+                )
+            except CollectorError as e:
+                logger.warning(f"URL scanning failed: {e}")
+
+        # Score all collected evidence
+        owner_candidates = scoring.score_owners(evidence_records)
+
+        # Assemble final result object
+        package_result = schemas.PackageResult(
+            package=package_name,
+            version=package_version,
+            owners=owner_candidates,
+            maintainers=pypi_maintainers,
+            evidence=evidence_records,
+        )
+
+        # 10. Report
+        if args.output_format == "json":
+            json_reporter.render(package_result)
+        else:
+            md_reporter.render(package_result)
+
+        # Exit codes
+        top_score = owner_candidates[0].score if owner_candidates else 0
+        return 0 if top_score >= 0.5 else 101
+    except NoEvidenceError as e:
+        logger.error(f"{type(e).__name__}: {e}")
+        return 101  # As per the PEP for "No usable evidence"
+    except NetworkError as e:
+        print(f"Error: A network problem occurred: {e}", file=sys.stderr)
+        return 101
+
+
+# --- Handler for the `explain` command ---
+def run_explain(args: argparse.Namespace) -> int:
+    """Handler for the 'explain' command."""
+    logger.info(f"Explaining evidence for package: {args.package}")
+    try:
+        metadata = pypi.fetch_package_metadata(args.package)
+        evidence_records, _ = evidence_analyzer.extract_from_pypi(metadata)
+
+        if args.id:
+            record = next(
+                (r for r in evidence_records if r.id.startswith(args.id)), None
+            )
+            if record:
+                print(json.dumps(dataclasses.asdict(record), indent=2, default=str))
+                return 0
+            logger.error(f"Evidence ID matching '{args.id}' not found.")
+            return 1
+        # Show all evidence
+        output: list[dict[str, str | None]] = [
+            dataclasses.asdict(r) for r in evidence_records
+        ]
+        print(json.dumps(output, indent=2, default=str))
+        return 0
+
+    except (NoEvidenceError, NetworkError) as e:
+        logger.error(f"{type(e).__name__}: {e}")
+        return 101
+
+
+def run_venv(args: argparse.Namespace) -> int:
+    """Handler for the 'venv' command."""
+    print("Executing 'venv' command...")
+    print(f"  Path: {args.path or 'current environment'}")
+    # TODO: Implement the actual logic
+    return 200  # Placeholder for "No anonymous"
+
+
+def run_reqs(args: argparse.Namespace) -> int:
+    """Handler for the 'reqs' command."""
+    print("Executing 'reqs' command...")
+    print(f"  Requirements File: {args.requirements_file}")
+    # TODO: Implement the actual logic
+    return 200  # Placeholder for "No anonymous"
+
+
+# ... Add placeholder functions for other commands ...
+
+
+def run_command(args: argparse.Namespace) -> int:
+    """
+    Dispatches the parsed arguments to the appropriate handler function.
+
+    Args:
+        args: The parsed arguments from argparse.
+
+    Returns:
+        An exit code.
+    """
+    # Prefer --verbose if set
+    log_level = "DEBUG" if args.log_level == "DEBUG" else args.log_level
+    setup_logging(log_level)
+    command_handlers = {
+        "who-owns": run_who_owns,
+        "explain": run_explain,
+        "venv": run_venv,
+        "reqs": run_reqs,
+        # "graph": run_graph,
+        # "cache": run_cache,
+        # "policy": run_policy,
+    }
+
+    handler = command_handlers.get(args.command)
+
+    if handler:
+        return handler(args)
+    print(f"Error: Command '{args.command}' is not yet implemented.", file=sys.stderr)
+    return 2

skip_trace/main.py

@@ -7,6 +7,7 @@ import json
 import logging
 import sys
 from typing import Set
+from urllib.parse import urlparse
 
 import tldextract
 from rich.logging import RichHandler
@@ -14,11 +15,18 @@ from rich.logging import RichHandler
 from . import schemas
 from .analysis import evidence as evidence_analyzer
 from .analysis import scoring
-from .collectors import github, package_files, pypi, whois
+from .collectors import (
+    github,
+    github_files,
+    package_files,
+    pypi,
+    pypi_attestations,
+    urls,
+    whois,
+)
 from .config import CONFIG
 from .exceptions import CollectorError, NetworkError, NoEvidenceError
 from .reporting import json_reporter, md_reporter
-from .utils.validation import is_valid_email
 
 # Create a logger instance for this module
 logger = logging.getLogger(__name__)
@@ -43,24 +51,46 @@ def run_who_owns(args: argparse.Namespace) -> int:
     logger.info(f"Executing 'who-owns' for package: {args.package}")
 
     try:
-        # 1. Collect initial data from PyPI
+        # Collect initial data from PyPI
         metadata = pypi.fetch_package_metadata(args.package, args.version)
         package_name = metadata.get("info", {}).get("name", args.package)
         package_version = metadata.get("info", {}).get("version")
-        logger.debug(
+        logger.info(
             f"Successfully fetched metadata for {package_name} v{package_version}"
         )
 
-        # 2. Analyze primary package metadata
+        # Analyze primary package metadata
         evidence_records, pypi_maintainers = evidence_analyzer.extract_from_pypi(
             metadata
         )
 
-        # 3. Cross-Reference for more PyPI evidence
+        logger.info(f"Evidence records so far {len(evidence_records)} -- pypi metadata")
+
+        # Check for PyPI attestations
+        attestation_evidence = pypi_attestations.collect(metadata)
+        evidence_records.extend(attestation_evidence)
+        logger.info(
+            f"Evidence records so far {len(evidence_records)} -- collected from PyPI attestations"
+        )
+
+        # Analyze package contents for deep evidence
+        try:
+            package_files_evidence = package_files.collect_from_package_files(metadata)
+            evidence_records.extend(package_files_evidence)
+            logger.info(
+                f"Evidence records so far {len(evidence_records)} -- collected from source code in package"
+            )
+        except CollectorError as e:
+            logger.warning(f"Could not analyze package files for {package_name}: {e}")
+
+        # Cross-Reference for more PyPI evidence
         cross_ref_evidence = pypi.cross_reference_by_user(package_name)
         evidence_records.extend(cross_ref_evidence)
+        logger.info(
+            f"Evidence records so far {len(evidence_records)} -- user cross ref"
+        )
 
-        # 4. Fetch evidence from code repositories found in PyPI evidence
+        # Fetch evidence from code repositories found in PyPI evidence
         repo_urls = set()
         for record in evidence_records:
             if (
@@ -76,76 +106,83 @@ def run_who_owns(args: argparse.Namespace) -> int:
             try:
                 github_evidence = github.extract_from_repo_url(url)
                 evidence_records.extend(github_evidence)
+                logger.info(
+                    f"Evidence records so far {len(evidence_records)} -- collected from github"
+                )
             except CollectorError as e:
                 logger.warning(f"Could not fully analyze GitHub repo {url}: {e}")
 
-        # 5. Extract domains and perform WHOIS lookups
+            # NEW: Collect evidence from GitHub files (SECURITY.md, FUNDING.yml, contributors)
+            try:
+                github_files_evidence = github_files.collect_from_repo_url(url)
+                evidence_records.extend(github_files_evidence)
+                logger.info(
+                    f"Evidence records so far {len(evidence_records)} -- collected from github files"
+                )
+            except CollectorError as e:
+                logger.warning(f"Could not collect GitHub files for {url}: {e}")
+
+        # Extract domains and perform WHOIS lookups
         domains_to_check: Set[str] = set()
+        urls_to_scan: Set[str] = set()
         ignored_domains = set(CONFIG.get("whois_ignored_domains", []))
 
         for record in evidence_records:
-            potential_domains: Set[str] = set()
-
-            # Case 1: Maintainer/Author email
-            if record.kind in (
-                schemas.EvidenceKind.EMAIL,
-                schemas.EvidenceKind.MAINTAINER,
-                schemas.EvidenceKind.AUTHOR_TAG,
-            ):
-                if email := record.value.get("email"):
-                    if "@" in email:
-                        potential_domains.add(email.split("@")[1])
-
-            # Case 2: URL from project_urls or org links
-            elif record.kind in (
-                schemas.EvidenceKind.ORGANIZATION,
-                schemas.EvidenceKind.PROJECT_URL,
-            ):
-                if url := record.value.get("url"):
-                    extracted = tldextract.extract(url)
-                    if extracted.registered_domain:
-                        potential_domains.add(extracted.registered_domain)
-
-            # Case 3: Contacts from a user profile (email, blog, etc.)
-            elif record.kind == schemas.EvidenceKind.USER_PROFILE:
-                if contacts := record.value.get("contacts"):
-                    for contact_value in contacts.values():
-                        if not contact_value:
-                            continue
-                        if valid_email := is_valid_email(contact_value):
-                            potential_domains.add(valid_email.split("@")[1])
-                        elif contact_value and "://" in contact_value:
-                            extracted = tldextract.extract(contact_value)
-                            if extracted.registered_domain:
-                                potential_domains.add(extracted.registered_domain)
-
-            # Add valid domains to the main set to be checked
-            for domain in potential_domains:
-                if domain not in ignored_domains:
-                    domains_to_check.add(domain)
-
+            # Extract domains for WHOIS
+            if email := record.value.get("email"):
+                if "@" in email:
+                    domain = email.split("@")[1]
+                    if domain not in ignored_domains:
+                        domains_to_check.add(domain)
+            # Extract domains and full URLs
+            if url := record.value.get("url"):
+                urls_to_scan.add(url)
+
+                # If it's a GitHub repo URL, also scan the user/org URL.
+                try:
+                    parsed_url = urlparse(url)
+                    if "github.com" in parsed_url.netloc:
+                        path_parts = [p for p in parsed_url.path.split("/") if p]
+                        if len(path_parts) >= 2:  # e.g., /owner/repo
+                            user_url = f"{parsed_url.scheme}://{parsed_url.netloc}/{path_parts[0]}"
+                            urls_to_scan.add(user_url)
+                except Exception as e:
+                    logger.debug(f"Could not parse user URL from {url}: {e}")
+
+                # Gather domains from URLs for WHOIS, respecting the ignore list
+                extracted = tldextract.extract(url)
+                if extracted.registered_domain:
+                    if extracted.registered_domain not in ignored_domains:
+                        domains_to_check.add(extracted.registered_domain)
+                        urls_to_scan.add(url)
+
+        # Perform WHOIS lookups
+        logger.info(f"Domains for WHOIS: {', '.join(sorted(list(domains_to_check)))}")
         if domains_to_check:
-            logger.info(
-                f"Found domains for WHOIS lookup: {', '.join(sorted(list(domains_to_check)))}"
-            )
             for domain in domains_to_check:
                 try:
-                    whois_evidence = whois.collect_from_domain(domain)
-                    evidence_records.extend(whois_evidence)
+                    evidence_records.extend(whois.collect_from_domain(domain))
+                    logger.info(
+                        f"Evidence records so far {len(evidence_records)} -- collected from domains/whois"
+                    )
                 except CollectorError as e:
-                    logger.warning(f"Could not get WHOIS evidence for {domain}: {e}")
+                    logger.warning(f"WHOIS failed for {domain}: {e}")
 
-        # 6. Analyze package contents for deep evidence
-        try:
-            package_files_evidence = package_files.collect_from_package_files(metadata)
-            evidence_records.extend(package_files_evidence)
-        except CollectorError as e:
-            logger.warning(f"Could not analyze package files for {package_name}: {e}")
+        # Scan homepage URLs
+        logger.info(f"URLs to scan: {', '.join(sorted(list(urls_to_scan)))}")
+        if urls_to_scan:
+            try:
+                evidence_records.extend(urls.collect_from_urls(urls_to_scan))
+                logger.info(
+                    f"Evidence records so far {len(evidence_records)} -- collected from urls"
+                )
+            except CollectorError as e:
+                logger.warning(f"URL scanning failed: {e}")
 
-        # 7. Score all collected evidence
+        # Score all collected evidence
         owner_candidates = scoring.score_owners(evidence_records)
 
-        # 8. Assemble final result object
+        # Assemble final result object
         package_result = schemas.PackageResult(
             package=package_name,
             version=package_version,
@@ -154,30 +191,15 @@ def run_who_owns(args: argparse.Namespace) -> int:
             evidence=evidence_records,
         )
 
-        # 9. Report
+        # 10. Report
         if args.output_format == "json":
             json_reporter.render(package_result)
         else:
             md_reporter.render(package_result)
 
-        # PEP specified exit codes based on score
-        # Using placeholder thresholds for now
+        # Exit codes
         top_score = owner_candidates[0].score if owner_candidates else 0
-        if top_score >= 0.7:
-            return 0  # Success
-        if top_score >= 0.5:
-            return 0  # Indeterminate # The tool didn't fail
-        return 101  # No usable evidence
-
-        # TODO: Pass evidence_records to the scoring engine
-        # Later, this will be replaced by a call to the analysis and reporting modules.
-        # For example:
-        #
-        # evidence = analysis.evidence.extract_from_pypi(metadata)
-        # owners = analysis.scoring.score_owners(evidence)
-        # package_result = schemas.PackageResult(package=args.package, owners=owners, evidence=evidence)
-        # reporting.json_reporter.render(package_result)
-        # return 0
+        return 0 if top_score >= 0.5 else 101
     except NoEvidenceError as e:
         logger.error(f"{type(e).__name__}: {e}")
         return 101  # As per the PEP for "No usable evidence"
@@ -195,13 +217,11 @@ def run_explain(args: argparse.Namespace) -> int:
         evidence_records, _ = evidence_analyzer.extract_from_pypi(metadata)
 
         if args.id:
-            # Filter for a specific evidence ID
             record = next(
                 (r for r in evidence_records if r.id.startswith(args.id)), None
             )
             if record:
-                output_record = dataclasses.asdict(record)
-                print(json.dumps(output_record, indent=2, default=str))
+                print(json.dumps(dataclasses.asdict(record), indent=2, default=str))
                 return 0
             logger.error(f"Evidence ID matching '{args.id}' not found.")
             return 1
@@ -209,7 +229,6 @@ def run_explain(args: argparse.Namespace) -> int:
         output: list[dict[str, str | None]] = [
             dataclasses.asdict(r) for r in evidence_records
         ]
-
         print(json.dumps(output, indent=2, default=str))
         return 0
 
@@ -255,7 +274,6 @@ def run_command(args: argparse.Namespace) -> int:
         "explain": run_explain,
         "venv": run_venv,
         "reqs": run_reqs,
-        # "explain": run_explain,
         # "graph": run_graph,
         # "cache": run_cache,
         # "policy": run_policy,