osintkit 0.1.1__py3-none-any.whl

osintkit/collectors/github.py

@@ -0,0 +1,190 @@
+"""GitHub public API collectors."""
+
+from __future__ import annotations
+
+import re
+from urllib.parse import quote
+
+from .. import errors
+from .base import Context, collector
+
+API = "https://api.github.com"
+
+
+def _headers(ctx: Context, extra: dict[str, str] | None = None) -> dict[str, str]:
+    headers = {
+        "Accept": "application/vnd.github+json",
+        "X-GitHub-Api-Version": "2022-11-28",
+    }
+    if ctx.github_token:
+        headers["Authorization"] = f"Bearer {ctx.github_token}"
+    if extra:
+        headers.update(extra)
+    return headers
+
+
+def _trim_user(user: dict) -> dict:
+    return {
+        "login": user.get("login"),
+        "name": user.get("name"),
+        "company": user.get("company"),
+        "blog": user.get("blog") or None,
+        "location": user.get("location"),
+        "bio": user.get("bio"),
+        "twitter_username": user.get("twitter_username"),
+        "public_repos": user.get("public_repos"),
+        "public_gists": user.get("public_gists"),
+        "followers": user.get("followers"),
+        "following": user.get("following"),
+        "created_at": user.get("created_at"),
+        "updated_at": user.get("updated_at"),
+        "url": user.get("html_url"),
+    }
+
+
+@collector("github_profile", ("username",), "GitHub profile and recent repositories", order=50)
+async def github_profile(ctx: Context) -> dict | None:
+    login = quote(ctx.target)
+    response = await ctx.http.get(
+        f"{API}/users/{login}",
+        namespace="github",
+        headers=_headers(ctx),
+        empty_on=(404,),
+    )
+    if response.status_code == 404:
+        return None
+    if response.status_code == 403:
+        raise errors.RateLimited("GitHub API rate limit reached")
+    user = response.json()
+    if not isinstance(user, dict):
+        return None
+
+    repos: list[dict] = []
+    repos_response = await ctx.http.get(
+        f"{API}/users/{login}/repos",
+        namespace="github",
+        params={"per_page": "50", "sort": "updated"},
+        headers=_headers(ctx),
+        empty_on=(404,),
+    )
+    if repos_response.status_code == 200:
+        payload = repos_response.json()
+        if isinstance(payload, list):
+            repos = [
+                {
+                    "name": r.get("name"),
+                    "description": (r.get("description") or "")[:200],
+                    "language": r.get("language"),
+                    "stars": r.get("stargazers_count"),
+                    "fork": r.get("fork"),
+                    "updated_at": r.get("updated_at"),
+                    "url": r.get("html_url"),
+                }
+                for r in payload
+                if isinstance(r, dict)
+            ]
+
+    return {"profile": _trim_user(user), "repositories": repos}
+
+
+@collector("github_commit_search", ("email",), "Commits authored with this email", order=50)
+async def github_commit_search(ctx: Context) -> dict:
+    response = await ctx.http.get(
+        f"{API}/search/commits",
+        namespace="github",
+        params={"q": f'author-email:"{ctx.target}"', "per_page": "50"},
+        headers=_headers(ctx, {"Accept": "application/vnd.github.cloak-preview+json"}),
+    )
+    body = response.json()
+    items = body.get("items") if isinstance(body, dict) else None
+    if not isinstance(items, list):
+        return {"total": 0, "commits": []}
+    commits = []
+    for item in items:
+        commit = item.get("commit") or {}
+        author = commit.get("author") or {}
+        commits.append(
+            {
+                "repo": (item.get("repository") or {}).get("full_name"),
+                "message": (commit.get("message") or "").splitlines()[0][:160],
+                "author_name": author.get("name"),
+                "author_date": author.get("date"),
+                "url": item.get("html_url"),
+            }
+        )
+    return {"total": body.get("total_count", len(commits)), "commits": commits}
+
+
+@collector("github_user_search", ("email", "company"), "GitHub user search", order=51)
+async def github_user_search(ctx: Context) -> dict:
+    query = f'"{ctx.target}"' if ctx.target_type == "email" else f'company:"{ctx.target}"'
+    response = await ctx.http.get(
+        f"{API}/search/users",
+        namespace="github",
+        params={"q": query, "per_page": "30"},
+        headers=_headers(ctx),
+    )
+    body = response.json()
+    items = body.get("items") if isinstance(body, dict) else None
+    if not isinstance(items, list):
+        return {"total": 0, "users": []}
+    return {
+        "total": body.get("total_count", len(items)),
+        "users": [
+            {"login": u.get("login"), "url": u.get("html_url"), "type": u.get("type")}
+            for u in items
+            if isinstance(u, dict)
+        ],
+    }
+
+
+LEGAL_SUFFIX = re.compile(r"\b(inc|llc|ltd|limited|gmbh|corp|corporation|plc|ag|sa|bv|oy|pty)\.?$", re.I)
+
+
+def org_slugs(company: str) -> list[str]:
+    base = company.strip().lower()
+    variants = [base, LEGAL_SUFFIX.sub("", base).strip()]
+    slugs = []
+    for variant in variants:
+        slug = re.sub(r"[^a-z0-9]+", "-", variant).strip("-")
+        if slug and slug not in slugs:
+            slugs.append(slug)
+    return slugs
+
+
+@collector("github_org_repos", ("company",), "GitHub repositories owned by the company", order=52)
+async def github_org_repos(ctx: Context) -> dict:
+    slugs = org_slugs(ctx.target)
+    slug = slugs[0]
+    response = None
+    for candidate in slugs:
+        response = await ctx.http.get(
+            f"{API}/orgs/{quote(candidate)}/repos",
+            namespace="github",
+            params={"per_page": "50", "sort": "updated"},
+            headers=_headers(ctx),
+            empty_on=(404,),
+        )
+        if response.status_code != 404:
+            slug = candidate
+            break
+    if response is None or response.status_code == 404:
+        return {"org": slug, "tried": slugs, "found": False, "repositories": []}
+    payload = response.json()
+    if not isinstance(payload, list):
+        return {"org": slug, "found": False, "repositories": []}
+    return {
+        "org": slug,
+        "found": True,
+        "repositories": [
+            {
+                "name": r.get("name"),
+                "description": (r.get("description") or "")[:200],
+                "language": r.get("language"),
+                "stars": r.get("stargazers_count"),
+                "url": r.get("html_url"),
+            }
+            for r in payload
+            if isinstance(r, dict)
+        ],
+    }

osintkit/collectors/pastes.py

@@ -0,0 +1,237 @@
+"""Paste dumps and public code mentions."""
+
+from __future__ import annotations
+
+import re
+from urllib.parse import quote
+
+from .. import errors
+from .base import Context, collector
+
+PSBDMP = "https://psbdmp.ws/api/v3"
+TAG_RE = re.compile(r"<[^>]+>")
+SPACE_RE = re.compile(r"\s+")
+GIST_RE = re.compile(r'href="/([A-Za-z0-9][A-Za-z0-9-]*)/([0-9a-f]{20,})"')
+
+
+def snippet(text: str, needle: str, width: int = 140) -> str:
+    """Plain-text window around the needle."""
+    clean = SPACE_RE.sub(" ", TAG_RE.sub(" ", text or "")).strip()
+    if not clean:
+        return ""
+    index = clean.lower().find(needle.lower())
+    if index == -1:
+        return clean[:width]
+    start = max(0, index - width // 2)
+    end = min(len(clean), index + len(needle) + width // 2)
+    return ("…" if start else "") + clean[start:end] + ("…" if end < len(clean) else "")
+
+
+@collector("psbdmp_search", ("email", "username", "domain"), "Psbdmp paste archive mentions", order=70)
+async def psbdmp_search(ctx: Context) -> dict:
+    try:
+        response = await ctx.http.get(
+            f"{PSBDMP}/search/{quote(ctx.target)}",
+            namespace="psbdmp",
+            empty_on=(404, 500),
+            timeout=ctx.http_timeout,
+        )
+    except (errors.SourceUnavailable, errors.RateLimited) as exc:
+        return await _gist_fallback(ctx, f"psbdmp unavailable ({exc})")
+
+    if response.status_code != 200:
+        return await _gist_fallback(ctx, f"psbdmp HTTP {response.status_code}")
+    try:
+        body = response.json()
+    except ValueError:
+        return await _gist_fallback(ctx, "psbdmp returned non-JSON")
+    entries = body.get("data") if isinstance(body, dict) else None
+    if not isinstance(entries, list):
+        return await _gist_fallback(ctx, "psbdmp returned an unexpected shape")
+    pastes = []
+    for entry in entries[:25]:
+        if not isinstance(entry, dict):
+            continue
+        paste_id = entry.get("id")
+        pastes.append(
+            {
+                "id": paste_id,
+                "url": f"https://pastebin.com/{paste_id}" if paste_id else None,
+                "archive_url": f"https://psbdmp.ws/api/v3/dump/{paste_id}" if paste_id else None,
+                "tags": entry.get("tags"),
+                "length": entry.get("len"),
+                "created": entry.get("time"),
+                "snippet": (entry.get("snippet") or "")[:400],
+            }
+        )
+    if not pastes:
+        return await _gist_fallback(ctx, "psbdmp had no hits")
+    return {"count": body.get("count", len(pastes)), "pastes": pastes, "source": "psbdmp"}
+
+
+async def _gist_fallback(ctx: Context, note: str) -> dict:
+    """Public GitHub Gist search, used when the paste archive is unreachable."""
+    try:
+        response = await ctx.http.get(
+            "https://gist.github.com/search",
+            namespace="gist",
+            params={"q": ctx.target},
+            accept="text/html",
+            empty_on=(403, 404, 429, 500),
+            timeout=ctx.http_timeout,
+        )
+    except (errors.SourceUnavailable, errors.RateLimited) as exc:
+        return {"count": 0, "pastes": [], "note": f"{note}; gist fallback failed ({exc})"}
+    if response.status_code != 200:
+        return {"count": 0, "pastes": [], "note": f"{note}; gist search HTTP {response.status_code}"}
+
+    seen: set[str] = set()
+    pastes = []
+    for owner, gist_id in GIST_RE.findall(response.text):
+        if gist_id in seen:
+            continue
+        seen.add(gist_id)
+        pastes.append(
+            {
+                "id": gist_id,
+                "url": f"https://gist.github.com/{owner}/{gist_id}",
+                "author": owner,
+            }
+        )
+        if len(pastes) >= 15:
+            break
+    return {"count": len(pastes), "pastes": pastes, "source": "github-gists", "note": note}
+
+
+@collector("code_mentions", ("email", "domain"), "Public code/repository mentions", order=71)
+async def code_mentions(ctx: Context) -> dict:
+    response = await ctx.http.get(
+        "https://grep.app/api/search",
+        namespace="grep.app",
+        params={"q": ctx.target},
+        empty_on=(404, 429),
+    )
+    if response.status_code != 200:
+        return {"count": 0, "hits": []}
+    try:
+        body = response.json()
+    except ValueError:
+        return {"count": 0, "hits": []}
+    facet = body.get("facets") if isinstance(body, dict) else None
+    hits_raw = body.get("hits", {}).get("hits", []) if isinstance(body, dict) else []
+    hits = []
+    for hit in hits_raw[:20]:
+        if not isinstance(hit, dict):
+            continue
+        repo = (hit.get("repo") or {}).get("raw") if isinstance(hit.get("repo"), dict) else None
+        path = (hit.get("path") or {}).get("raw") if isinstance(hit.get("path"), dict) else None
+        snippet = ((hit.get("content") or {}).get("snippet") or "")[:400]
+        hits.append({"repo": repo, "path": path, "snippet": snippet})
+    total = None
+    if isinstance(facet, dict):
+        total = facet.get("repo", {}).get("total") if isinstance(facet.get("repo"), dict) else None
+    return {"count": len(hits), "total_repos": total, "hits": hits}
+
+
+@collector("hackertarget_hostsearch", ("domain",), "Passive DNS host search", order=72)
+async def hackertarget_hostsearch(ctx: Context) -> dict:
+    response = await ctx.http.get(
+        "https://api.hackertarget.com/hostsearch/",
+        namespace="hackertarget",
+        params={"q": ctx.target},
+        accept="text/plain",
+        empty_on=(404, 429),
+    )
+    if response.status_code != 200:
+        return {"hosts": []}
+    text = response.text.strip()
+    if text.lower().startswith("error") or "API count exceeded" in text:
+        return {"hosts": [], "note": text[:120]}
+    hosts = []
+    for line in text.splitlines():
+        if "," not in line:
+            continue
+        name, _, ip = line.partition(",")
+        hosts.append({"host": name.strip(), "ip": ip.strip()})
+    return {"count": len(hosts), "hosts": hosts[:100]}
+
+
+@collector("hn_mentions", ("email", "username", "domain", "company"), "Hacker News mentions", order=73)
+async def hn_mentions(ctx: Context) -> dict:
+    response = await ctx.http.get(
+        "https://hn.algolia.com/api/v1/search",
+        namespace="hn",
+        params={"query": ctx.target, "hitsPerPage": "20"},
+        empty_on=(404, 429),
+        timeout=ctx.http_timeout,
+    )
+    if response.status_code != 200:
+        return {"count": 0, "mentions": []}
+    try:
+        body = response.json()
+    except ValueError:
+        return {"count": 0, "mentions": []}
+    hits = body.get("hits") if isinstance(body, dict) else None
+    if not isinstance(hits, list):
+        return {"count": 0, "mentions": []}
+    mentions = []
+    for hit in hits[:20]:
+        if not isinstance(hit, dict):
+            continue
+        text = hit.get("comment_text") or hit.get("story_text") or ""
+        mentions.append(
+            {
+                "title": hit.get("title") or hit.get("story_title"),
+                "url": hit.get("url") or (f"https://news.ycombinator.com/item?id={hit.get('objectID')}"),
+                "author": hit.get("author"),
+                "created": hit.get("created_at"),
+                "snippet": snippet(text, ctx.target),
+            }
+        )
+    return {"count": len(mentions), "total": body.get("nbHits"), "mentions": mentions}
+
+
+@collector(
+    "stackoverflow_mentions",
+    ("email", "username", "domain", "company"),
+    "Stack Exchange mentions",
+    order=74,
+)
+async def stackoverflow_mentions(ctx: Context) -> dict:
+    response = await ctx.http.get(
+        "https://api.stackexchange.com/2.3/search/excerpts",
+        namespace="stackexchange",
+        params={
+            "q": ctx.target,
+            "site": "stackoverflow",
+            "pagesize": "20",
+            "order": "desc",
+            "sort": "relevance",
+        },
+        empty_on=(400, 404, 429),
+        timeout=ctx.http_timeout,
+    )
+    if response.status_code != 200:
+        return {"count": 0, "items": []}
+    try:
+        body = response.json()
+    except ValueError:
+        return {"count": 0, "items": []}
+    raw = body.get("items") if isinstance(body, dict) else None
+    if not isinstance(raw, list):
+        return {"count": 0, "items": []}
+    items = []
+    for item in raw[:20]:
+        if not isinstance(item, dict):
+            continue
+        items.append(
+            {
+                "title": item.get("title"),
+                "url": item.get("question_id") and f"https://stackoverflow.com/q/{item.get('question_id')}",
+                "item_type": item.get("item_type"),
+                "score": item.get("score"),
+                "tags": item.get("tags"),
+                "excerpt": snippet(item.get("excerpt") or "", ctx.target),
+            }
+        )
+    return {"count": len(items), "items": items}

osintkit/collectors/ports.py

@@ -0,0 +1,91 @@
+"""Lightweight TCP connect probes against common ports."""
+
+from __future__ import annotations
+
+import asyncio
+
+from ..http import cache_hit, cache_put
+from .base import Context, collector
+
+NAMESPACE = "ports"
+
+PORTS: dict[int, str] = {
+    21: "ftp",
+    22: "ssh",
+    23: "telnet",
+    25: "smtp",
+    53: "dns",
+    80: "http",
+    110: "pop3",
+    143: "imap",
+    443: "https",
+    445: "smb",
+    465: "smtps",
+    587: "submission",
+    993: "imaps",
+    995: "pop3s",
+    3306: "mysql",
+    3389: "rdp",
+    5432: "postgres",
+    6379: "redis",
+    8080: "http-alt",
+    8443: "https-alt",
+    9200: "elasticsearch",
+    27017: "mongodb",
+}
+
+CONCURRENCY = 24
+TIMEOUT = 2.5
+
+
+async def _probe(host: str, port: int, service: str, semaphore: asyncio.Semaphore) -> dict | None:
+    async with semaphore:
+        try:
+            reader, writer = await asyncio.wait_for(asyncio.open_connection(host, port), timeout=TIMEOUT)
+        except (TimeoutError, OSError):
+            return None
+        banner = ""
+        try:
+            banner_bytes = await asyncio.wait_for(reader.read(120), timeout=1.0)
+            banner = banner_bytes.decode("utf-8", "replace").strip().splitlines()[0][:120]
+        except (TimeoutError, OSError, IndexError):
+            banner = ""
+        finally:
+            writer.close()
+            try:
+                await writer.wait_closed()
+            except OSError:
+                pass
+        return {"port": port, "service": service, "banner": banner or None}
+
+
+@collector("tcp_ports", ("domain",), "Open TCP ports on common services", order=80, ttl=3600)
+def selected_ports(override: list[str] | None) -> dict[int, str]:
+    if not override:
+        return dict(PORTS)
+    chosen: dict[int, str] = {}
+    for raw in override:
+        port = int(raw)
+        chosen[port] = PORTS.get(port, "unknown")
+    return dict(sorted(chosen.items()))
+
+
+async def tcp_ports(ctx: Context) -> dict:
+    ports = selected_ports(ctx.settings.extra.get("ports"))
+    host = ctx.target.lower()
+    key = f"{host}|{','.join(str(port) for port in ports)}"
+    cached = cache_hit(ctx.settings, NAMESPACE, key, ctx.cache_ttl)
+    if cached is not None:
+        return cached
+
+    semaphore = asyncio.Semaphore(CONCURRENCY)
+    results = await asyncio.gather(*(_probe(host, port, service, semaphore) for port, service in ports.items()))
+    open_ports = [r for r in results if r]
+    payload = {
+        "host": host,
+        "scanned": len(ports),
+        "open": sorted(open_ports, key=lambda r: r["port"]),
+        "note": "single connect probe from the local network; filtered ports may be missed",
+    }
+    cache_put(ctx.settings, NAMESPACE, key, payload)
+    return payload

osintkit/collectors/subdomains.py

@@ -0,0 +1,102 @@
+"""Passive subdomain enumeration: certificate logs plus passive DNS plus resolution."""
+
+from __future__ import annotations
+
+import asyncio
+
+import dns.asyncresolver
+
+from ..http import cache_hit, cache_put
+from .base import Context, collector
+from .ct import _query
+from .pastes import hackertarget_hostsearch
+
+NAMESPACE = "subdomains"
+
+COMMON_PREFIXES = (
+    "www",
+    "mail",
+    "api",
+    "dev",
+    "staging",
+    "test",
+    "vpn",
+    "admin",
+    "cdn",
+    "static",
+    "docs",
+    "status",
+    "git",
+    "ci",
+    "app",
+)
+
+
+async def _resolve(name: str, resolvers: list[str]) -> str | None:
+    resolver = dns.asyncresolver.Resolver(configure=False)
+    resolver.nameservers = list(resolvers)
+    resolver.timeout = 3.0
+    resolver.lifetime = 4.0
+    try:
+        answer = await resolver.resolve(name, "A", raise_on_no_answer=False, lifetime=4.0)
+    except Exception:
+        return None
+    if answer.rrset is None:
+        return None
+    return str(next(iter(answer))).rstrip(".")
+
+
+@collector(
+    "passive_subdomains",
+    ("domain",),
+    "Subdomains from cert logs, passive DNS, resolution",
+    order=25,
+    ttl=3600,
+)
+async def passive_subdomains(ctx: Context) -> dict:
+    domain = ctx.target.lower()
+    key = f"{domain}|{','.join(ctx.settings.resolvers)}"
+    cached = cache_hit(ctx.settings, NAMESPACE, key, ctx.cache_ttl)
+    if cached is not None:
+        return cached
+    entries = await _query(ctx, f"%.{domain}", "crt")
+    names: set[str] = set()
+
+    for entry in entries:
+        for line in f"{entry.get('name_value') or ''}\n{entry.get('common_name') or ''}".splitlines():
+            name = line.strip().lower().lstrip("*.").strip()
+            if name.endswith("." + domain) and "." in name:
+                names.add(name)
+
+    try:
+        passive = await hackertarget_hostsearch(ctx)
+    except Exception:
+        passive = {"hosts": []}
+    for row in passive.get("hosts") or []:
+        name = str(row.get("host") or "").lower()
+        if name.endswith("." + domain):
+            names.add(name)
+
+    candidates = sorted(names)
+    resolved = await asyncio.gather(*(_resolve(n, ctx.settings.resolvers) for n in candidates[:80]))
+    live = {name: ip for name, ip in zip(candidates[:80], resolved, strict=True) if ip}
+
+    unresolved_prefixes = []
+    for prefix in COMMON_PREFIXES:
+        candidate = f"{prefix}.{domain}"
+        if candidate in names:
+            continue
+        ip = await _resolve(candidate, ctx.settings.resolvers)
+        if ip:
+            unresolved_prefixes.append({"host": candidate, "ip": ip})
+
+    result = {
+        "domain": domain,
+        "discovered_count": len(names),
+        "live_hosts": [{"host": h, "ip": ip} for h, ip in sorted(live.items())],
+        "common_prefix_hits": unresolved_prefixes,
+        "sources": ["crt.sh", "hackertarget", "dns"],
+        "note": "passive only: no brute force beyond a short common-prefix list",
+    }
+    cache_put(ctx.settings, NAMESPACE, key, result)
+    return result

osintkit/collectors/whois_lookup.py

@@ -0,0 +1,72 @@
+"""WHOIS registration data."""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+import whois
+
+from ..http import cache_hit, cache_put
+from .base import Context, collector
+
+NAMESPACE = "whois"
+FIELD_LIMIT = 1500
+
+
+def _clean(value: Any) -> Any:
+    if value is None:
+        return None
+    if isinstance(value, list):
+        return [_clean(v) for v in value[:20]]
+    if hasattr(value, "isoformat"):
+        return value.isoformat()
+    if isinstance(value, str):
+        return value.strip()[:FIELD_LIMIT]
+    return str(value)[:FIELD_LIMIT]
+
+
+def _blocking_lookup(domain: str) -> dict:
+    record = whois.whois(domain)
+    if record is None:
+        return {}
+    data = dict(record)
+    keys = (
+        "domain_name",
+        "registrar",
+        "whois_server",
+        "referral_url",
+        "updated_date",
+        "creation_date",
+        "expiration_date",
+        "name_servers",
+        "status",
+        "emails",
+        "dnssec",
+        "org",
+        "country",
+        "state",
+        "city",
+    )
+    out: dict[str, Any] = {}
+    for key in keys:
+        if data.get(key):
+            out[key] = _clean(data[key])
+    return out
+
+
+@collector("whois_registration", ("domain",), "WHOIS registration record", order=40, timeout=30)
+async def whois_registration(ctx: Context) -> dict:
+    domain = ctx.target.lower()
+    cached = cache_hit(ctx.settings, NAMESPACE, domain)
+    if cached is not None:
+        return cached
+    try:
+        limit = ctx.http_timeout or 25
+        result = await asyncio.wait_for(asyncio.to_thread(_blocking_lookup, domain), timeout=limit)
+    except TimeoutError:
+        return {"note": "whois lookup timed out"}
+    except Exception as exc:
+        return {"note": f"whois unavailable: {type(exc).__name__}"}
+    cache_put(ctx.settings, NAMESPACE, domain, result)
+    return result