skip-trace 0.1.0__py3-none-any.whl

skip_trace/__about__.py

@@ -0,0 +1,19 @@
+"""Metadata for skip_trace."""
+
+__all__ = [
+    "__title__",
+    "__version__",
+    "__description__",
+    "__readme__",
+    "__credits__",
+    "__requires_python__",
+    "__status__",
+]
+
+__title__ = "skip-trace"
+__version__ = "0.1.0"
+__description__ = "Ownership Attribution for Python Packages"
+__readme__ = "README.md"
+__credits__ = [{"name": "Matthew Dean Martin", "email": "matthewdeanmartin@gmail.com"}]
+__requires_python__ = ">=3.8"
+__status__ = "1 - Planning"

skip_trace/__init__.py

@@ -0,0 +1,6 @@
+# skip_trace/__init__.py
+
+__version__ = "0.1.0"
+
+# __all__ will be populated as public functions/classes are added.
+__all__ = []

skip_trace/__main__.py

@@ -0,0 +1,9 @@
+# skip_trace/__main__.py
+from __future__ import annotations
+
+import sys
+
+from .cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

skip_trace/analysis/__init__.py

@@ -0,0 +1,4 @@
+# skip_trace/analysis/__init__.py
+from . import evidence, scoring, source_scanner
+
+__all__ = ["evidence", "scoring", "source_scanner"]

skip_trace/analysis/evidence.py

@@ -0,0 +1,312 @@
+# skip_trace/analysis/evidence.py
+from __future__ import annotations
+
+import datetime
+import hashlib
+import logging
+import re
+from typing import Any, Dict, List, Optional, Tuple
+
+import tldextract
+
+from ..schemas import EvidenceKind, EvidenceRecord, EvidenceSource, Maintainer
+from ..utils.validation import is_valid_email
+from . import ner  # Import the NER module
+
+logger = logging.getLogger(__name__)
+
+
+def _slugify(text: str) -> str:
+    """
+    Creates a URL-friendly slug from a string.
+
+    Converts to lowercase, folds to ASCII, replaces non-alphanumeric
+    characters with hyphens, and removes duplicate hyphens.
+
+    Args:
+        text: The string to slugify.
+
+    Returns:
+        A slugified string.
+    """
+    if not text:
+        return ""
+    # Simple ASCII folding
+    text = text.encode("ascii", "ignore").decode("ascii")
+    text = text.lower()
+    # Replace non-alphanumeric with hyphen
+    text = re.sub(r"[^a-z0-9]+", "-", text).strip("-")
+    return text
+
+
+def generate_evidence_id(
+    source: EvidenceSource,
+    kind: EvidenceKind,
+    locator: str,
+    value: Any,  # Changed to Any for dataclasses
+    slug_subject: str,
+    hint: Optional[str] = None,
+) -> str:
+    """
+    Generates a human-readable and deterministic Evidence ID.
+
+    Format: e-<source>-<kind>-<slug>[--<hint>]~<hash8>
+
+    Args:
+        source: The source of the evidence.
+        kind: The kind of evidence.
+        locator: The URL or path where the evidence was found.
+        value: The value of the evidence itself.
+        slug_subject: The primary entity to use for the slug (e.g., person, org).
+        hint: An optional hint to add to the slug for disambiguation.
+
+    Returns:
+        A formatted, unique evidence ID string.
+    """
+    slug = _slugify(slug_subject)
+    if hint:
+        slug = f"{slug}--{_slugify(hint)}"
+
+    # Truncate slug to a reasonable length to keep ID under 96 chars
+    max_slug_len = 60
+    slug = slug[:max_slug_len]
+
+    # Create the hash
+    hasher = hashlib.sha256()
+    hasher.update(f"{source.value}|{kind.value}|{locator}|{value}".encode("utf-8"))
+    hash8 = hasher.hexdigest()[:8]
+
+    return f"e-{source.value}-{kind.value}-{slug}~{hash8}"
+
+
+def _parse_contact_string(contact_str: str) -> Dict[str, Optional[str]]:
+    """
+    Parses a contact string into its name and email components.
+
+    Handles formats like "Name <email>", "email", and "Name".
+
+    Args:
+        contact_str: The string to parse.
+
+    Returns:
+        A dictionary with "name" and "email" keys.
+    """
+    if not contact_str or not contact_str.strip():
+        return {"name": None, "email": None}
+
+    # Pattern for "Name <email@domain.com>"
+    match = re.search(r"(.+)<(.+)>", contact_str)
+    if match:
+        name = match.group(1).strip()
+        # Validate the email part using the robust validator
+        email = is_valid_email(match.group(2).strip())
+        return {"name": name, "email": email}
+
+    # If the whole string is a valid email, use it.
+    if email := is_valid_email(contact_str):
+        return {"name": None, "email": email}
+
+    # Fallback to NER if it's not a clear email format
+    list_of_entities = ner.extract_entities(contact_str.strip())
+    for entity, kind in list_of_entities:
+        if kind == "PERSON":
+            return {"name": entity, "email": None}
+
+    # If no email and no person from NER, we got nothing
+    return {"name": None, "email": None}
+
+
+# Helper to sanitize fields that might contain the literal string "None"
+def _clean_pypi_field(field_value: Any) -> str:
+    """Returns an empty string if the field is None or the literal string 'None'."""
+    if field_value is None or str(field_value).strip().lower() == "none":
+        return ""
+    return str(field_value).strip()
+
+
+def extract_from_pypi(
+    metadata: Dict[str, Any],
+) -> Tuple[List[EvidenceRecord], List[Maintainer]]:
+    """
+    Extracts evidence from raw PyPI package metadata.
+
+    Args:
+        metadata: The dictionary of package metadata from the PyPI JSON API.
+
+    Returns:
+        A tuple containing:
+         - A list of EvidenceRecord objects.
+         - A list of Maintainer objects from direct PyPI fields.
+    """
+    evidence_list: List[EvidenceRecord] = []
+    maintainer_list: List[Maintainer] = []
+    seen_maintainers = set()
+
+    info = metadata.get("info", {})
+    if not info:
+        logger.warning("PyPI metadata is missing the 'info' dictionary.")
+        return [], []
+
+    package_name = info.get("name", "unknown")
+    package_version = info.get("version", "latest")
+    locator = f"https://pypi.org/pypi/{package_name}/{package_version}/json"
+    now = datetime.datetime.now(datetime.timezone.utc)
+
+    # --- Create separate evidence for names and emails ---
+    def process_contact_string(
+        raw_string: str, role_kind: EvidenceKind, field_name: str
+    ) -> None:
+        """
+        Parses a string for contacts and creates separate evidence records
+        for each piece of information (name, email).
+        """
+        if not raw_string:
+            return
+
+        def add_separate_evidence(
+            parsed_contact: Dict[str, Optional[str]],
+            source_note: str,
+            confidence: float,
+        ) -> None:
+            """Creates and appends separate evidence for name and email."""
+            name = parsed_contact.get("name")
+            email = parsed_contact.get("email")
+
+            # Also create a simple Maintainer object for direct reporting
+            if name or email:
+                key = (name, email)
+                if key not in seen_maintainers:
+                    maintainer_list.append(
+                        Maintainer(
+                            name=name or "Unknown", email=email, confidence=confidence
+                        )
+                    )
+                    seen_maintainers.add(key)
+
+            # Create evidence for the name, if it exists
+            if name:
+                # NOTE: Assumes EvidenceKind.PERSON exists in your schema
+                kind = EvidenceKind.PERSON
+                value = {"name": name}
+                record = EvidenceRecord(
+                    id=generate_evidence_id(
+                        EvidenceSource.PYPI, kind, locator, str(value), name
+                    ),
+                    source=EvidenceSource.PYPI,
+                    locator=locator,
+                    kind=kind,
+                    value=value,
+                    observed_at=now,
+                    confidence=confidence,
+                    notes=f"Found person '{name}' from PyPI '{field_name}' field ({source_note}). Designated as {role_kind.value}.",
+                )
+                evidence_list.append(record)
+                logger.debug(
+                    f"Created {kind.value} evidence for '{name}' from '{field_name}'."
+                )
+
+            # Create evidence for the email, if it exists
+            if email:
+                # NOTE: Assumes EvidenceKind.EMAIL exists in your schema
+                kind = EvidenceKind.EMAIL
+                value = {"email": email}
+                # Use the name for the slug if available, otherwise email's local part
+                slug_subject = name or email.split("@")[0]
+                record = EvidenceRecord(
+                    id=generate_evidence_id(
+                        EvidenceSource.PYPI, kind, locator, str(value), slug_subject
+                    ),
+                    source=EvidenceSource.PYPI,
+                    locator=locator,
+                    kind=kind,
+                    value=value,
+                    observed_at=now,
+                    confidence=confidence + 0.1,  # Emails are slightly more reliable
+                    notes=f"Found email for '{slug_subject}' from PyPI '{field_name}' field ({source_note}). Designated as {role_kind.value}.",
+                )
+                evidence_list.append(record)
+                logger.debug(
+                    f"Created {kind.value} evidence for '{email}' from '{field_name}'."
+                )
+
+        # Attempt to use NER to find multiple entities
+        entities = ner.extract_entities(raw_string)
+        if entities:
+            logger.debug(
+                f"NER found {len(entities)} entities in PyPI field '{field_name}': {entities}"
+            )
+            for entity_name, _entity_label in entities:
+                parsed = _parse_contact_string(entity_name)
+                add_separate_evidence(parsed, "NER", confidence=0.45)
+        # else:
+        #     # Fallback to simple parsing if NER finds nothing
+        #     parsed = _parse_contact_string(raw_string)
+        #     add_separate_evidence(parsed, "regex fallback", confidence=0.30)
+
+    # Process author and maintainer fields
+    author_name = _clean_pypi_field(info.get("author"))
+    author_email = _clean_pypi_field(info.get("author_email"))
+    # Prefer email string as it's more likely to contain both name and email
+    author_string = author_email or author_name
+    if author_string:
+        process_contact_string(
+            author_string, EvidenceKind.AUTHOR_TAG, "author/author_email"
+        )
+
+    maintainer_name = _clean_pypi_field(info.get("maintainer"))
+    maintainer_email = _clean_pypi_field(info.get("maintainer_email"))
+    maintainer_string = maintainer_email or maintainer_name
+
+    # Only process maintainer if it's different from the author string
+    if maintainer_string and maintainer_string != author_string:
+        process_contact_string(
+            maintainer_string, EvidenceKind.MAINTAINER, "maintainer/maintainer_email"
+        )
+
+    # --- Project URL parsing ---
+    project_urls = info.get("project_urls")
+    if isinstance(project_urls, dict):
+        logger.debug(f"Found {len(project_urls)} project URLs to analyze.")
+        for label, url in project_urls.items():
+            if not url or not isinstance(url, str):
+                continue
+
+            domain_info = tldextract.extract(url)
+            repo_host = domain_info.domain
+            logger.debug(f"Parsing project URL ({label}): {url}")
+
+            if repo_host in ("github", "gitlab", "codeberg"):
+                path_parts = url.strip("/").split("/")
+                if len(path_parts) >= 4:
+                    org_or_user = path_parts[3]
+                    logger.debug(f"Extracted user/org '{org_or_user}' from URL.")
+                    value = {"name": org_or_user, "url": url}
+                    notes = f"Found user/org '{org_or_user}' from repository URL in project_urls."
+                    record = EvidenceRecord(
+                        id=generate_evidence_id(
+                            EvidenceSource.PYPI,
+                            EvidenceKind.ORGANIZATION,
+                            locator,
+                            str(value),
+                            org_or_user,
+                            hint=f"{repo_host}-user",
+                        ),
+                        source=EvidenceSource.PYPI,
+                        locator=locator,
+                        kind=EvidenceKind.ORGANIZATION,
+                        value=value,
+                        observed_at=now,
+                        confidence=0.35,
+                        notes=notes,
+                    )
+                    already_in = False
+                    for already in evidence_list:
+                        if already.notes == notes:
+                            already_in = True
+                    if not already_in:
+                        evidence_list.append(record)
+
+    logger.info(
+        f"Extracted {len(evidence_list)} evidence records and {len(maintainer_list)} maintainers from PyPI."
+    )
+    return evidence_list, maintainer_list

skip_trace/analysis/ner.py

@@ -0,0 +1,58 @@
+# skip_trace/analysis/ner.py
+from __future__ import annotations
+
+import logging
+from typing import List, Optional, Tuple
+
+import spacy
+from spacy.language import Language
+
+SPACY_AVAILABLE = True
+
+
+logger = logging.getLogger(__name__)
+
+_nlp: Optional[Language] = None
+
+
+def _get_nlp_model() -> Optional[Language]:
+    """Loads and caches the spaCy model. Returns None if unavailable."""
+    global _nlp
+    if not SPACY_AVAILABLE:
+        return None
+    if _nlp is None:
+        try:
+            logger.debug("Loading spaCy model 'en_core_web_sm'...")
+            _nlp = spacy.load("en_core_web_sm")
+            logger.info("Successfully loaded spaCy NER model.")
+        except IOError:
+            logger.warning(
+                "spaCy is installed, but model 'en_core_web_sm' not found. "
+                "Run 'python -m spacy download en_core_web_sm' to install it."
+            )
+            return None
+    return _nlp
+
+
+def extract_entities(text: str) -> List[Tuple[str, str]]:
+    """
+    Extracts person and organization entities from a string using spaCy.
+
+    Args:
+        text: The text to process.
+
+    Returns:
+        A list of tuples, where each tuple is (entity_text, entity_label).
+        Returns an empty list if spaCy is not available or fails.
+    """
+    nlp = _get_nlp_model()
+    if not nlp:
+        return []
+
+    doc = nlp(text)
+    entities = []
+    for ent in doc.ents:
+        if ent.label_ in ["PERSON", "ORG"]:
+            entities.append((ent.text.strip(), ent.label_))
+            logger.debug(f"NER found entity: '{ent.text}' (Label: {ent.label_})")
+    return entities

skip_trace/analysis/scoring.py

@@ -0,0 +1,282 @@
+# skip_trace/analysis/scoring.py
+from __future__ import annotations
+
+import collections
+import logging
+from typing import Dict, List, Optional, Tuple
+
+import tldextract
+
+from ..analysis.evidence import _parse_contact_string  # Import the parser for reuse
+from ..config import CONFIG
+from ..schemas import (
+    Contact,
+    ContactType,
+    EvidenceKind,
+    EvidenceRecord,
+    OwnerCandidate,
+    OwnerKind,
+)
+
+# Words that indicate a regex grabbed junk from a license instead of a name.
+JUNK_WORDS = {
+    "copyright",
+    "holders",
+    "license",
+    "document",
+    "accompanies",
+    "identifies",
+    "endorse",
+    "promote",
+    "software",
+    "permission",
+    "danger",
+    "warranty",
+    "bsd",
+    "liability",
+    # duped
+    "notice",
+    "authors",
+    "conditions",
+    # stop words
+    "and",
+    "other",
+    "the",
+    "for",
+    "with",
+    "this",
+    "list",
+    "following",
+    "txt",
+    "damages",
+    "owner",
+    # legalese
+    "incidental",
+    "holder",
+    # license names
+    "MIT",
+    "BSD",
+}
+
+logger = logging.getLogger(__name__)
+
+
+def _normalize_name(name: str) -> str:
+    """Normalizes a name for entity grouping."""
+    # Also parse out emails that might be part of the name
+    parsed = _parse_contact_string(name)
+    raw_name = parsed.get("name") or parsed.get("email") or name
+    # Strip common trailing punctuation for better grouping
+    return raw_name.strip().rstrip(",.'").lower()
+
+
+def _get_entity_from_record(record: EvidenceRecord) -> Tuple[Optional[str], OwnerKind]:
+    """Extracts a primary entity name and kind from an evidence record."""
+    kind = OwnerKind.INDIVIDUAL  # Default
+    name = None
+
+    if record.kind in (
+        EvidenceKind.MAINTAINER,
+        EvidenceKind.AUTHOR_TAG,
+        EvidenceKind.COMMIT_AUTHOR,
+        EvidenceKind.PYPI_USER,
+        EvidenceKind.USER_PROFILE,
+        EvidenceKind.CONTACT,  # Handle generic contacts
+    ):
+        raw_name = record.value.get("name") or record.value.get("email")
+        if raw_name:
+            # The name might be "Name <email>", so parse it
+            parsed = _parse_contact_string(raw_name)
+            name = parsed.get("name") or parsed.get("email")
+        kind = OwnerKind.INDIVIDUAL
+    elif record.kind in (EvidenceKind.ORGANIZATION, EvidenceKind.REPO_OWNER):
+        name = record.value.get("name")
+        # Check if the name looks like a user or an org
+        # A simple heuristic: if it contains spaces, it's likely a person's name
+        if name and " " in name:
+            kind = OwnerKind.INDIVIDUAL
+        else:
+            kind = OwnerKind.PROJECT
+    # Handle user profile and company evidence
+    elif record.kind == EvidenceKind.USER_PROFILE:
+        name = record.value.get("user_name")
+        kind = OwnerKind.INDIVIDUAL
+    elif record.kind == EvidenceKind.USER_COMPANY:
+        # The primary entity is the user, but this record also implies a company
+        name = record.value.get("user_name")
+        kind = OwnerKind.INDIVIDUAL
+    elif record.kind == EvidenceKind.PROJECT_URL:
+        url = record.value.get("url", "")
+        domain_info = tldextract.extract(url)
+        if domain_info.domain and domain_info.suffix:
+            name = domain_info.domain.capitalize()
+            kind = OwnerKind.COMPANY
+    # Handle WHOIS domain evidence
+    elif record.kind == EvidenceKind.DOMAIN:
+        name = record.value.get("name")
+        kind = OwnerKind.COMPANY
+    # Handle COPYRIGHT evidence from file scans
+    elif record.kind == EvidenceKind.COPYRIGHT:
+        # The scanner is now responsible for pre-filtering junk.
+        # This logic can now trust its input more.
+        raw_holder = record.value.get("holder")
+        if not raw_holder:
+            return None, kind
+
+        # --- NEW: Sanitize the raw string before accepting it as a name ---
+        # 1. Reject if it's too long to be a name.
+        if len(raw_holder) > 50:
+            return None, kind
+        # 2. Reject if it contains common license garbage words.
+        if any(word in raw_holder.lower() for word in JUNK_WORDS):
+            return None, kind
+
+        parsed = _parse_contact_string(raw_holder)
+        name = parsed.get("name") or parsed.get("email") or raw_holder
+        if parsed.get("email") or " " in name or "," in name:
+            kind = OwnerKind.INDIVIDUAL
+        else:
+            kind = OwnerKind.COMPANY
+
+    return name, kind
+
+
+def score_owners(evidence_records: List[EvidenceRecord]) -> List[OwnerCandidate]:
+    """
+    Scores and ranks potential owners from a list of evidence.
+
+    This function performs entity resolution by normalizing names, aggregates
+    evidence for each unique entity, and calculates a score based on the
+    weights defined in the application configuration.
+
+    Args:
+        evidence_records: A list of EvidenceRecord objects to analyze.
+
+    Returns:
+        A list of OwnerCandidate objects, sorted by score in descending order.
+    """
+    # Get suppression settings from config
+    suppressed_orgs = CONFIG.get("suppressed_tool_orgs", [])
+    lenient_mode = CONFIG.get("lenient_mode_enabled", False)
+
+    # --- 1. Initial Entity Extraction & Alias Mapping ---
+    entities: Dict[str, OwnerCandidate] = {}
+    evidence_by_entity: Dict[str, List[EvidenceRecord]] = collections.defaultdict(list)
+
+    # 1. First pass: extract all entities and map evidence
+    for record in evidence_records:
+        name, kind = _get_entity_from_record(record)
+        # if not name:
+        #     logging.warning(f"Skipping {record.kind}")
+        #     continue
+
+        if not name:
+            name = ""
+
+        # Suppress tool orgs like 'github' unless in lenient mode
+        if name and (name.lower() in suppressed_orgs) and not lenient_mode:
+            continue
+
+        norm_name = _normalize_name(name)
+        evidence_by_entity[norm_name].append(record)
+        if norm_name not in entities:
+            # Use the raw name for display, but the normalized name for grouping
+            entities[norm_name] = OwnerCandidate(
+                name=name.strip().rstrip(",.'"), kind=kind
+            )
+
+        # Also create entities for companies mentioned in user profiles
+        if record.kind == EvidenceKind.USER_COMPANY:
+            company_name = record.value.get("company_name")
+            if company_name:
+                norm_co_name = _normalize_name(company_name)
+                if norm_co_name not in entities:
+                    entities[norm_co_name] = OwnerCandidate(
+                        name=company_name, kind=OwnerKind.COMPANY
+                    )
+                evidence_by_entity[norm_co_name].append(
+                    record
+                )  # Associate this evidence with the company too
+
+    # --- 2. Score each candidate and collect contacts ---
+    contact_map = {
+        "email": ContactType.EMAIL,
+        "twitter": ContactType.TWITTER,
+        "linkedin": ContactType.LINKEDIN,
+        "mastodon": ContactType.MASTODON,
+        "facebook": ContactType.FACEBOOK,
+        "instagram": ContactType.INSTAGRAM,
+        "youtube": ContactType.YOUTUBE,
+        "tiktok": ContactType.TIKTOK,
+    }
+    for norm_name, owner in entities.items():
+        score = 0.0
+        seen_rationale_keys = set()
+        contacts: Dict[Tuple[ContactType, str], Contact] = {}
+
+        for record in evidence_by_entity[norm_name]:
+            owner.evidence.append(record.id)
+            rationale_key = f"{record.source.value}-{record.kind.value}"
+            weight = record.confidence  # Use confidence from collector
+
+            if rationale_key not in seen_rationale_keys:
+                score += weight
+                seen_rationale_keys.add(rationale_key)
+            else:
+                score += weight * 0.1  # Diminishing return
+
+            # --- UPDATED: Collect contact info from all relevant evidence kinds ---
+            contact_source_string = None
+            if record.kind in (
+                EvidenceKind.MAINTAINER,
+                EvidenceKind.AUTHOR_TAG,
+                EvidenceKind.COMMIT_AUTHOR,
+                EvidenceKind.CONTACT,
+            ):
+                contact_source_string = record.value.get("email") or record.value.get(
+                    "name"
+                )
+            elif record.kind in (EvidenceKind.ORGANIZATION, EvidenceKind.REPO_OWNER):
+                if url := record.value.get("url", record.locator):
+                    contacts[(ContactType.REPO, url)] = Contact(
+                        type=ContactType.REPO, value=url
+                    )
+            elif record.kind == EvidenceKind.PYPI_USER:
+                if url := record.value.get("url"):
+                    contacts[(ContactType.URL, url)] = Contact(
+                        type=ContactType.URL, value=url
+                    )
+            # Collect contacts from USER_PROFILE evidence
+            elif record.kind == EvidenceKind.USER_PROFILE:
+                for key, value in record.value.get("contacts", {}).items():
+                    contact_type = contact_map.get(
+                        key, ContactType.URL
+                    )  # Default to generic URL
+                    contacts[(contact_type, value)] = Contact(
+                        type=contact_type, value=value
+                    )
+            elif record.kind == EvidenceKind.COPYRIGHT:
+                contact_source_string = record.value.get("holder")
+
+            # Parse any found string for an email
+            if contact_source_string:
+                parsed_contact = _parse_contact_string(contact_source_string)
+                if email := parsed_contact.get("email"):
+                    contacts[(ContactType.EMAIL, email)] = Contact(
+                        type=ContactType.EMAIL, value=email
+                    )
+
+        owner.score = min(round(score, 2), 1.0)
+        owner.evidence = sorted(list(set(owner.evidence)))
+        owner.rationale = " + ".join(sorted(list(seen_rationale_keys)))
+        owner.contacts = sorted(
+            list(contacts.values()), key=lambda c: (c.type.value, c.value)
+        )
+
+    # 4. Filter and Sort
+    # filtered_candidates = [
+    #     c for c in entities.values()
+    #     if not (c.name.lower() in ["nobody", "nobody in particular", "example"] and c.score < 0.3)
+    # ]
+
+    return sorted(entities.values(), key=lambda c: c.score, reverse=True)