osintkit 0.1.1__py3-none-any.whl

osintkit/collectors/accounts.py

@@ -0,0 +1,83 @@
+"""Username correlation across public profiles that answer 404 for unknown handles."""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+from urllib.parse import quote
+
+from .base import Context, collector
+
+SITES: tuple[tuple[str, str], ...] = (
+    ("github", "https://github.com/{u}"),
+    ("gitlab", "https://gitlab.com/{u}"),
+    ("codeberg", "https://codeberg.org/{u}"),
+    ("bitbucket", "https://bitbucket.org/{u}"),
+    ("keybase", "https://keybase.io/{u}"),
+    ("pypi", "https://pypi.org/user/{u}/"),
+    ("npm", "https://www.npmjs.com/~{u}"),
+    ("reddit", "https://www.reddit.com/user/{u}"),
+    ("mastodon.social", "https://mastodon.social/@{u}"),
+    ("dev.to", "https://dev.to/{u}"),
+    ("dribbble", "https://dribbble.com/{u}"),
+    ("behance", "https://www.behance.net/{u}"),
+    ("kaggle", "https://www.kaggle.com/{u}"),
+    ("huggingface", "https://huggingface.co/{u}"),
+    ("codepen", "https://codepen.io/{u}"),
+    ("twitch", "https://www.twitch.tv/{u}"),
+    ("soundcloud", "https://soundcloud.com/{u}"),
+    ("replit", "https://replit.com/@{u}"),
+)
+
+SEMAPHORE = 8
+
+
+async def _probe(ctx: Context, site: str, template: str) -> dict | None:
+    handle = ctx.target
+    url = template.format(u=quote(handle))
+    try:
+        status = await ctx.http.head_ok(url, namespace="presence")
+    except Exception as exc:
+        return {"site": site, "url": url, "status": "error", "detail": type(exc).__name__}
+    if status == 404:
+        return None
+    return {
+        "site": site,
+        "url": url,
+        "status": "found" if status < 400 else "unreachable",
+        "http_status": status,
+    }
+
+
+@collector("username_presence", ("username",), "Public profile presence", order=60, ttl=21600)
+async def username_presence(ctx: Context) -> dict:
+    semaphore = asyncio.Semaphore(SEMAPHORE)
+
+    async def guarded(site: str, template: str) -> dict | None:
+        async with semaphore:
+            return await _probe(ctx, site, template)
+
+    results = await asyncio.gather(*(guarded(s, t) for s, t in SITES))
+    found = [r for r in results if r and r["status"] == "found"]
+    unknown = [r for r in results if r and r["status"] != "found"]
+    return {
+        "handle": ctx.target,
+        "checked": len(SITES),
+        "found": found,
+        "unconfirmed": unknown,
+    }
+
+
+@collector("gravatar", ("email",), "Gravatar presence for the address", order=61)
+async def gravatar(ctx: Context) -> dict:
+    digest = hashlib.md5(ctx.target.strip().lower().encode("utf-8")).hexdigest()
+    url = f"https://gravatar.com/avatar/{digest}?d=404"
+    try:
+        status = await ctx.http.head_ok(url, namespace="gravatar")
+    except Exception as exc:
+        return {"checked": False, "detail": type(exc).__name__}
+    return {
+        "checked": True,
+        "present": status == 200,
+        "profile": f"https://gravatar.com/{digest}" if status == 200 else None,
+    }

osintkit/collectors/base.py

@@ -0,0 +1,138 @@
+"""Collector registry and safe execution."""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass, replace
+from typing import Any
+
+from .. import errors
+from ..config import Settings
+from ..http import HttpClient
+from ..models import CollectorResult
+
+
+@dataclass(slots=True)
+class Context:
+    target: str
+    target_type: str
+    settings: Settings
+    http: HttpClient
+    cache_ttl: int | None = None
+    http_timeout: float | None = None
+
+    @property
+    def hibp_key(self) -> str | None:
+        return self.settings.hibp_api_key
+
+    @property
+    def github_token(self) -> str | None:
+        return self.settings.github_token
+
+
+CollectorFunc = Callable[[Context], Awaitable[Any]]
+
+
+@dataclass(frozen=True, slots=True)
+class Collector:
+    name: str
+    target_types: tuple[str, ...]
+    func: CollectorFunc
+    description: str = ""
+    requires: str | None = None
+    order: int = 100
+    ttl: int | None = None
+    timeout: float | None = None
+
+
+REGISTRY: list[Collector] = []
+
+
+def collector(
+    name: str,
+    target_types: tuple[str, ...],
+    description: str = "",
+    *,
+    requires: str | None = None,
+    order: int = 100,
+    ttl: int | None = None,
+    timeout: float | None = None,
+) -> Callable[[CollectorFunc], CollectorFunc]:
+    def register(func: CollectorFunc) -> CollectorFunc:
+        REGISTRY.append(
+            Collector(
+                name=name,
+                target_types=target_types,
+                func=func,
+                description=description,
+                requires=requires,
+                order=order,
+                ttl=ttl,
+                timeout=timeout,
+            )
+        )
+        return func
+
+    return register
+
+
+def collectors_for(target_type: str) -> list[Collector]:
+    found = [c for c in REGISTRY if target_type in c.target_types]
+    return sorted(found, key=lambda c: (c.order, c.name))
+
+
+async def run_collector(spec: Collector, ctx: Context) -> CollectorResult:
+    if spec.requires and not _has_requirement(spec.requires, ctx):
+        hint = {
+            "hibp": "HIBP_API_KEY not set",
+            "github": "GITHUB_TOKEN not set",
+        }.get(spec.requires, f"{spec.requires} unavailable")
+        return CollectorResult(
+            collector=spec.name,
+            target=ctx.target,
+            target_type=ctx.target_type,
+            ok=False,
+            skipped=True,
+            note=f"skipped: {hint}",
+        )
+
+    if spec.ttl is not None or spec.timeout is not None:
+        ctx = replace(
+            ctx,
+            cache_ttl=spec.ttl if spec.ttl is not None else ctx.cache_ttl,
+            http_timeout=spec.timeout,
+        )
+
+    started = time.perf_counter()
+    try:
+        data = await spec.func(ctx)
+    except errors.RateLimited as exc:
+        return _result(spec, ctx, started, ok=False, error=f"rate limited: {exc}")
+    except errors.SourceUnavailable as exc:
+        return _result(spec, ctx, started, ok=False, error=str(exc))
+    except TimeoutError:
+        return _result(spec, ctx, started, ok=False, error="timed out")
+    except Exception as exc:  # a broken source must never sink the run
+        return _result(spec, ctx, started, ok=False, error=f"{type(exc).__name__}: {exc}")
+
+    return _result(spec, ctx, started, ok=True, data=data)
+
+
+def _result(spec: Collector, ctx: Context, started: float, **kwargs: Any) -> CollectorResult:
+    elapsed_ms = int((time.perf_counter() - started) * 1000)
+    return CollectorResult(
+        collector=spec.name,
+        target=ctx.target,
+        target_type=ctx.target_type,
+        elapsed_ms=elapsed_ms,
+        **kwargs,
+    )
+
+
+def _has_requirement(requirement: str, ctx: Context) -> bool:
+    if requirement == "hibp":
+        return bool(ctx.hibp_key)
+    if requirement == "github":
+        return bool(ctx.github_token)
+    return True

osintkit/collectors/breaches.py

@@ -0,0 +1,163 @@
+"""Breach exposure checks against public, login-free indexes."""
+
+from __future__ import annotations
+
+from urllib.parse import quote
+
+from .. import errors
+from .base import Context, collector
+
+HIBP = "https://haveibeenpwned.com/api/v3"
+MAX_LISTED = 80
+
+
+def _summarise_hibp(entries: list[dict]) -> list[dict]:
+    out = []
+    for entry in entries:
+        out.append(
+            {
+                "name": entry.get("Name") or entry.get("Title"),
+                "domain": entry.get("Domain"),
+                "date": entry.get("BreachDate"),
+                "added": entry.get("AddedDate"),
+                "pwn_count": entry.get("PwnCount"),
+                "data_types": entry.get("DataClasses") or [],
+                "verified": entry.get("IsVerified"),
+                "description": (entry.get("Description") or "")[:500],
+                "source": "haveibeenpwned",
+            }
+        )
+    return out
+
+
+@collector(
+    "hibp_breaches",
+    ("email",),
+    "Have I Been Pwned breach membership",
+    requires="hibp",
+    order=10,
+)
+async def hibp_breaches(ctx: Context) -> list[dict]:
+    headers = {"hibp-api-key": ctx.hibp_key or "", "User-Agent": "osintkit/0.1"}
+    response = await ctx.http.get(
+        f"{HIBP}/breachedaccount/{quote(ctx.target)}",
+        namespace="hibp",
+        params={"truncateResponse": "false"},
+        headers=headers,
+        empty_on=(404,),
+    )
+    if response.status_code == 404:
+        return []
+    body = response.json()
+    if not isinstance(body, list):
+        return []
+    return _summarise_hibp(body)
+
+
+@collector(
+    "hibp_pastes",
+    ("email",),
+    "Have I Been Pwned paste exposure",
+    requires="hibp",
+    order=11,
+)
+async def hibp_pastes(ctx: Context) -> list[dict]:
+    headers = {"hibp-api-key": ctx.hibp_key or "", "User-Agent": "osintkit/0.1"}
+    response = await ctx.http.get(
+        f"{HIBP}/pasteaccount/{quote(ctx.target)}",
+        namespace="hibp",
+        headers=headers,
+        empty_on=(404,),
+    )
+    if response.status_code == 404:
+        return []
+    body = response.json()
+    if not isinstance(body, list):
+        return []
+    return [
+        {
+            "source": entry.get("Source"),
+            "id": entry.get("Id"),
+            "title": entry.get("Title"),
+            "date": entry.get("Date"),
+            "email_count": entry.get("EmailCount"),
+            "origin": "haveibeenpwned-pastes",
+        }
+        for entry in body
+    ]
+
+
+@collector("xposed_or_not", ("email",), "XposedOrNot public breach lookup", order=12)
+async def xposed_or_not(ctx: Context) -> dict:
+    response = await ctx.http.get(
+        f"https://api.xposedornot.com/v1/check-email/{quote(ctx.target)}",
+        namespace="xposedornot",
+        empty_on=(404, 422),
+    )
+    if response.status_code != 200:
+        return {"found": 0, "breaches": []}
+    try:
+        body = response.json()
+    except ValueError:
+        return {"found": 0, "breaches": []}
+    raw = body.get("breaches") or []
+    total_found = sum(len(item) if isinstance(item, list) else 1 for item in raw)
+    names: list[str] = []
+    for item in raw:
+        if isinstance(item, list):
+            names.extend(str(n) for n in item)
+        elif isinstance(item, str):
+            names.append(item)
+    details = body.get("breach_details") if isinstance(body, dict) else None
+    breaches = []
+    if isinstance(details, dict):
+        for entry in details.get("breaches") or []:
+            if not isinstance(entry, dict):
+                continue
+            breaches.append(
+                {
+                    "name": entry.get("breach") or entry.get("name"),
+                    "domain": entry.get("domain"),
+                    "date": entry.get("xposed_date") or entry.get("breached_date"),
+                    "data_types": entry.get("xposed_data") or entry.get("data_classes") or [],
+                    "source": "xposedornot",
+                }
+            )
+    if not breaches:
+        breaches = [{"name": name, "source": "xposedornot"} for name in names]
+    breaches.sort(key=lambda item: str(item.get("name") or "").lower())
+    return {
+        "found": max(total_found, len(breaches)),
+        "listed": len(breaches[:MAX_LISTED]),
+        "breaches": breaches[:MAX_LISTED],
+    }
+
+
+@collector("leakcheck_public", ("email",), "LeakCheck public (free tier)", order=13)
+async def leakcheck_public(ctx: Context) -> dict:
+    response = await ctx.http.get(
+        "https://leakcheck.io/api/public",
+        namespace="leakcheck",
+        params={"check": ctx.target},
+        empty_on=(404, 400),
+    )
+    if response.status_code != 200:
+        raise errors.SourceUnavailable("leakcheck public endpoint unavailable")
+    try:
+        body = response.json()
+    except ValueError:
+        return {"found": 0, "sources": []}
+    sources = []
+    for entry in body.get("result") or []:
+        for src in entry.get("sources") or []:
+            sources.append(
+                {
+                    "name": src.get("name"),
+                    "date": src.get("date"),
+                    "source": "leakcheck",
+                }
+            )
+    deduped: dict[str, dict] = {}
+    for src in sources:
+        deduped.setdefault(str(src.get("name")), src)
+    return {"found": len(deduped), "sources": list(deduped.values())}

osintkit/collectors/ct.py

@@ -0,0 +1,165 @@
+"""Certificate transparency via crt.sh."""
+
+from __future__ import annotations
+
+import json
+import re
+from urllib.parse import quote
+
+from .base import Context, collector
+
+CRT = "https://crt.sh/"
+
+
+def _entries(body: object) -> list[dict]:
+    if not isinstance(body, list):
+        return []
+    return [e for e in body if isinstance(e, dict)]
+
+
+def _parse(response: object) -> list[dict]:
+    """crt.sh sometimes wraps JSON in an HTML header block."""
+    text = getattr(response, "text", "").strip()
+    if text.startswith("[BEGIN_HEADERS]"):
+        text = text.split("[END_HEADERS]", 1)[-1].replace("<BR>", "").strip()
+    if not text.startswith(("[", "{")):
+        return []
+    try:
+        return _entries(json.loads(text))
+    except ValueError:
+        return []
+
+
+def _names(entries: list[dict], domain: str) -> list[str]:
+    seen: set[str] = set()
+    out: list[str] = []
+    for entry in entries:
+        blob = f"{entry.get('name_value') or ''}\n{entry.get('common_name') or ''}"
+        for line in blob.splitlines():
+            name = line.strip().lower().lstrip("*.").strip()
+            if not name or name == domain or "." not in name:
+                continue
+            if not name.endswith("." + domain):
+                continue
+            if name in seen:
+                continue
+            seen.add(name)
+            out.append(name)
+    return sorted(out)
+
+
+async def _query(ctx: Context, query: str, namespace: str) -> list[dict]:
+    response = await ctx.http.get(
+        CRT,
+        namespace=namespace,
+        params={"q": query, "output": "json", "exclude": "expired"},
+        accept="application/json",
+        empty_on=(404,),
+        timeout=ctx.http_timeout,
+    )
+    if response.status_code != 200:
+        return []
+    return _parse(response)
+
+
+@collector(
+    "crt_sh_subdomains",
+    ("domain",),
+    "Subdomains from certificate transparency",
+    order=20,
+    timeout=90,
+    ttl=21600,
+)
+async def crt_sh_subdomains(ctx: Context) -> list[str]:
+    entries = await _query(ctx, f"%.{ctx.target}", "crt")
+    return _names(entries, ctx.target.lower())
+
+
+@collector("crt_sh_certificates", ("domain",), "Issued certificates from crt.sh", order=21, timeout=90)
+async def crt_sh_certificates(ctx: Context) -> list[dict]:
+    entries = await _query(ctx, f"%.{ctx.target}", "crt")
+    certs = []
+    for entry in entries[:60]:
+        issuer = str(entry.get("issuer_name") or "")
+        certs.append(
+            {
+                "id": entry.get("id"),
+                "common_name": entry.get("common_name"),
+                "names": (entry.get("name_value") or "").splitlines()[:10],
+                "issuer": issuer.split("O=")[-1].split(",")[0] if "O=" in issuer else issuer[:80],
+                "not_before": entry.get("not_before"),
+                "not_after": entry.get("not_after"),
+                "serial": entry.get("serial_number"),
+                "source": "crt.sh",
+            }
+        )
+    return certs
+
+
+def _company_domains(company: str) -> list[str]:
+    """Guess likely apex domains from a company name."""
+    slug = re.sub(r"[^a-z0-9]+", "", company.lower())
+    slug = re.sub(r"(inc|llc|ltd|limited|gmbh|corp|corporation|plc)$", "", slug)
+    candidates = []
+    for suffix in (".com", ".net", ".io"):
+        candidate = slug + suffix
+        if len(slug) >= 2 and candidate not in candidates:
+            candidates.append(candidate)
+    return candidates
+
+
+def _summarise(entries: list[dict]) -> dict:
+    domains: dict[str, int] = {}
+    for entry in entries:
+        for name in (entry.get("name_value") or "").splitlines():
+            name = name.strip().lower().lstrip("*.")
+            if "." in name and not name.startswith("."):
+                domains[name] = domains.get(name, 0) + 1
+    ranked = sorted(domains.items(), key=lambda kv: kv[1], reverse=True)[:50]
+    return {
+        "certificates_seen": len(entries),
+        "domains": [{"domain": d, "certificate_count": n} for d, n in ranked],
+        "sample_certificates": [
+            {
+                "common_name": e.get("common_name"),
+                "issuer": e.get("issuer_name"),
+                "not_before": e.get("not_before"),
+            }
+            for e in entries[:20]
+        ],
+    }
+
+
+@collector(
+    "crt_sh_organization",
+    ("company",),
+    "Certificates issued to an organisation",
+    order=20,
+    timeout=90,
+    ttl=21600,
+)
+async def crt_sh_organization(ctx: Context) -> dict:
+    entries = await _query(ctx, f"O={quote(ctx.target)}", "crt")
+    if entries:
+        return {**_summarise(entries), "method": "organization match"}
+
+    entries = []
+    matched: list[str] = []
+    for domain in _company_domains(ctx.target):
+        found = await _query(ctx, f"%.{domain}", "crt")
+        if found:
+            entries.extend(found)
+            matched.append(domain)
+        if len(entries) > 400:
+            break
+    if not entries:
+        return {
+            **_summarise([]),
+            "method": "no organization match",
+            "note": "crt.sh had no O= matches and the guessed domains were empty",
+        }
+    return {
+        **_summarise(entries),
+        "method": "guessed domain identity",
+        "matched_domains": matched,
+    }

osintkit/collectors/dns.py

@@ -0,0 +1,130 @@
+"""DNS record collection through public resolvers."""
+
+from __future__ import annotations
+
+import asyncio
+
+import dns.asyncresolver
+import dns.exception
+import dns.resolver
+
+from ..http import cache_hit, cache_put
+from .base import Context, collector
+
+RECORD_TYPES = ("A", "AAAA", "MX", "TXT", "NS", "CNAME")
+NAMESPACE = "dns"
+
+
+def _key(ctx: Context, host: str) -> str:
+    return f"{host}|{','.join(ctx.settings.resolvers)}"
+
+
+def _resolver(ctx: Context) -> dns.asyncresolver.Resolver:
+    resolver = dns.asyncresolver.Resolver(configure=False)
+    resolver.nameservers = list(ctx.settings.resolvers)
+    resolver.timeout = 4.0
+    resolver.lifetime = 6.0
+    return resolver
+
+
+def _format(rtype: str, record: object) -> str:
+    if rtype == "MX":
+        return f"{getattr(record, 'preference', 0)} {getattr(record, 'exchange', record)}".strip()
+    if rtype == "TXT":
+        strings = getattr(record, "strings", None)
+        if strings:
+            return b"".join(strings).decode("utf-8", "replace")
+        return str(record).strip('"')
+    return str(record).rstrip(".")
+
+
+def _spf_all(records: list[str]) -> str | None:
+    """Classify the SPF `all` qualifier: -all is strict, +all is permissive."""
+    for record in records:
+        for token in record.split():
+            lowered = token.lower()
+            if lowered.endswith("all") and lowered.lstrip("~-+?").startswith("all"):
+                qualifier = lowered[:-3]
+                return f"{qualifier}all" if qualifier else "all"
+    return None
+
+
+def _dmarc_policy(record: str) -> str | None:
+    for part in record.split(";"):
+        key, _, value = part.partition("=")
+        if key.strip().lower() == "p":
+            return value.strip().lower() or None
+    return None
+
+
+async def _lookup(ctx: Context, host: str, rtype: str) -> list[str]:
+    resolver = _resolver(ctx)
+    try:
+        answer = await resolver.resolve(host, rtype, raise_on_no_answer=False, lifetime=6.0)
+    except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.exception.Timeout):
+        return []
+    except dns.resolver.NoNameservers:
+        return []
+    if answer.rrset is None:
+        return []
+    return [_format(rtype, rd) for rd in answer]
+
+
+async def _all_records(ctx: Context, host: str) -> dict[str, list[str]]:
+    results = await asyncio.gather(*(_lookup(ctx, host, rt) for rt in RECORD_TYPES))
+    return {rtype: value for rtype, value in zip(RECORD_TYPES, results, strict=True) if value}
+
+
+@collector("dns_records", ("domain",), "A/AAAA/MX/TXT/NS/CNAME records", order=30, ttl=3600)
+async def dns_records(ctx: Context) -> dict:
+    host = ctx.target.lower()
+    key = _key(ctx, host)
+    cached = cache_hit(ctx.settings, NAMESPACE, key, ctx.cache_ttl)
+    if cached is not None:
+        return cached
+    records = await _all_records(ctx, host)
+    cache_put(ctx.settings, NAMESPACE, key, records)
+    return records
+
+
+@collector("dns_mail_posture", ("domain",), "SPF, DMARC, and DKIM hints", order=31, ttl=3600)
+async def dns_mail_posture(ctx: Context) -> dict:
+    host = ctx.target.lower()
+    key = f"mail:{_key(ctx, host)}"
+    cached = cache_hit(ctx.settings, NAMESPACE, key, ctx.cache_ttl)
+    if cached is not None:
+        return cached
+
+    txt = await _lookup(ctx, host, "TXT")
+    spf = [t for t in txt if t.lower().startswith("v=spf1")]
+
+    dmarc = await _lookup(ctx, f"_dmarc.{host}", "TXT") if not host.startswith("_dmarc.") else []
+    dmarc = [t for t in dmarc if t.lower().startswith("v=dmarc1")]
+
+    dkim_selectors = ("default", "google", "selector1", "selector2", "k1", "mail")
+    dkim_hits = []
+    for selector in dkim_selectors:
+        found = await _lookup(ctx, f"{selector}._domainkey.{host}", "TXT")
+        if found:
+            dkim_hits.append({"selector": selector, "present": True})
+
+    spf_all = _spf_all(spf)
+    result = {
+        "spf": spf,
+        "spf_present": bool(spf),
+        "spf_all": spf_all,
+        "spf_strict": spf_all == "-all",
+        "dmarc": dmarc,
+        "dmarc_present": bool(dmarc),
+        "dmarc_policy": _dmarc_policy(dmarc[0]) if dmarc else None,
+        "dkim_selectors_found": dkim_hits,
+    }
+    cache_put(ctx.settings, NAMESPACE, key, result)
+    return result
+
+
+@collector("dns_wildcard", ("domain",), "Wildcard DNS detection", order=32, ttl=3600)
+async def dns_wildcard(ctx: Context) -> dict:
+    probe = f"osintkit-wildcard-probe.{ctx.target.lower()}"
+    answers = await _lookup(ctx, probe, "A")
+    return {"probe": probe, "wildcard": bool(answers), "answers": answers}