osintkit 0.1.1__py3-none-any.whl

osintkit/synthesis.py

@@ -0,0 +1,176 @@
+"""Prompt construction, model call, strict validation, and the single repair retry."""
+
+from __future__ import annotations
+
+import json
+import time
+from typing import Any
+
+from .errors import SourceUnavailable, SynthesisError
+from .llm import LLMClient
+from .models import Analysis, PromptBundle
+
+SYSTEM_PROMPT = """\
+You are a defensive OSINT analyst. You receive raw results gathered from public sources \
+about a single target. Work only from the supplied data plus well-known public facts about \
+the platforms named in it. Never invent a breach, an account, or a hostname.
+
+Produce a compact, source-grounded briefing for someone doing a personal privacy check or \
+authorised research.
+
+Rules:
+- Cite the supplied source name in every breach entry. Mark dates as found; use null when unknown.
+- If a source reports more breaches than it listed, say so and keep the 15 most material, ordering
+  credential-bearing and recent incidents first.
+- Distinguish confirmed findings from plausible inference in the summary and notes.
+- Risk level reflects exposure only: high means credential-bearing breaches or a wide,
+  live attack surface.
+- Recommendations must be concrete and actionable, not generic advice.
+- If a source failed or was skipped, note the gap in infrastructure_notes.
+
+Return ONLY a JSON object, no prose and no code fences, with exactly these keys:
+{
+  "summary": "3-6 sentence overview of the public footprint",
+  "breaches": [{"name": "str", "date": "YYYY-MM-DD or null", "data_types": ["str"], "source": "str"}],
+  "associated_accounts": [
+    {"platform": "str", "url": "str or null", "handle": "str or null", "note": "str or null"}
+  ],
+  "infrastructure_notes": ["str"],
+  "subdomains": ["str"],
+  "risk_level": "low" | "medium" | "high",
+  "recommendations": ["str"]
+}
+Use empty arrays when a section has no findings.\
+"""
+
+USER_TEMPLATE = """\
+Target: {target}
+Target type: {target_type}
+
+Raw public-source data:
+```json
+{payload}
+```
+
+Write the briefing now as strict JSON.\
+"""
+
+REPAIR_PROMPT = """\
+That response was not valid against the required schema. Error: {error}
+
+Return the corrected JSON object only. All seven keys are required, arrays instead of prose \
+for every list field, and risk_level must be exactly one of "low", "medium", or "high".\
+"""
+
+
+def build_messages(bundle: PromptBundle) -> list[dict[str, str]]:
+    return [
+        {"role": "system", "content": SYSTEM_PROMPT},
+        {
+            "role": "user",
+            "content": USER_TEMPLATE.format(
+                target=bundle.target,
+                target_type=bundle.target_type,
+                payload=bundle.to_prompt_json(),
+            ),
+        },
+    ]
+
+
+def render_prompt(bundle: PromptBundle) -> str:
+    parts = []
+    for message in build_messages(bundle):
+        parts.append(f"### {message['role']}\n\n{message['content']}")
+    return "\n\n".join(parts)
+
+
+def extract_json(text: str) -> dict[str, Any]:
+    cleaned = text.strip()
+    if cleaned.startswith("```"):
+        cleaned = cleaned.split("\n", 1)[-1]
+        if cleaned.rstrip().endswith("```"):
+            cleaned = cleaned.rstrip()[:-3]
+    start = cleaned.find("{")
+    end = cleaned.rfind("}")
+    if start == -1 or end == -1 or end <= start:
+        raise SynthesisError("no JSON object in model output")
+    blob = cleaned[start : end + 1]
+    try:
+        parsed = json.loads(blob)
+    except json.JSONDecodeError as exc:
+        raise SynthesisError(f"invalid JSON: {exc.msg} at offset {exc.pos}") from exc
+    if not isinstance(parsed, dict):
+        raise SynthesisError("model output was JSON but not an object")
+    return parsed
+
+
+async def synthesize(
+    client: LLMClient,
+    bundle: PromptBundle,
+    *,
+    logger: Any | None = None,
+    max_tokens: int = 2400,
+) -> Analysis:
+    messages = build_messages(bundle)
+
+    settings = getattr(client, "settings", None)
+
+    async def call(current: list[dict[str, str]], attempt: int) -> str:
+        if logger is not None:
+            logger.request(
+                base_url=getattr(settings, "base_url", "unknown"),
+                model=getattr(settings, "model", "unknown"),
+                attempt=attempt,
+                dry_run=False,
+                messages=current,
+                max_tokens=max_tokens,
+            )
+        started = time.perf_counter()
+        try:
+            try:
+                content = await client.chat(
+                    current,
+                    max_tokens=max_tokens,
+                    response_format={"type": "json_object"},
+                )
+            except SourceUnavailable as exc:
+                if "HTTP 400" not in str(exc):
+                    raise
+                content = await client.chat(current, max_tokens=max_tokens)
+        except Exception as exc:
+            if logger is not None:
+                logger.response(
+                    attempt=attempt,
+                    ok=False,
+                    error=str(exc),
+                    elapsed_ms=int((time.perf_counter() - started) * 1000),
+                    content=None,
+                )
+            raise
+        if logger is not None:
+            logger.response(
+                attempt=attempt,
+                ok=True,
+                error=None,
+                elapsed_ms=int((time.perf_counter() - started) * 1000),
+                content=content,
+            )
+        return content
+
+    raw = ""
+    try:
+        raw = await call(messages, 1)
+        return Analysis.model_validate(extract_json(raw))
+    except (SynthesisError, ValueError) as first_error:
+        repair = [
+            *messages,
+            {"role": "assistant", "content": raw or "(empty)"},
+            {"role": "user", "content": REPAIR_PROMPT.format(error=str(first_error)[:400])},
+        ]
+        raw_retry = await call(repair, 2)
+        try:
+            return Analysis.model_validate(extract_json(raw_retry))
+        except (SynthesisError, ValueError) as second_error:
+            raise SynthesisError(
+                f"model output still invalid after one repair attempt: {second_error}"
+            ) from second_error

osintkit/targets.py

@@ -0,0 +1,65 @@
+"""Target typing and normalisation."""
+
+from __future__ import annotations
+
+import re
+
+from .errors import TargetParseError
+
+EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[A-Za-z]{2,}$")
+DOMAIN_RE = re.compile(r"^(?=.{1,253}$)(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)+[A-Za-z]{2,24}$")
+USERNAME_RE = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9._-]{0,38})$")
+COMPANY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9 .,&'()+-]*$")
+COMPANY_HINTS = re.compile(r"\b(inc|llc|ltd|gmbh|corp|corporation|limited|plc|ag|sa|bv|oy|pty)\b", re.I)
+
+VALID_TYPES = ("email", "username", "domain", "company")
+
+
+def normalise(target: str) -> str:
+    target = target.strip()
+    if target.lower().startswith("http://") or target.lower().startswith("https://"):
+        target = target.split("://", 1)[1]
+    target = target.strip().strip("<>\"'")
+    if "@" not in target:
+        target = target.split("/", 1)[0]
+    if target.lower().startswith("www."):
+        target = target[4:]
+    return target.rstrip("/")
+
+
+def detect_type(target: str) -> str:
+    if EMAIL_RE.match(target):
+        return "email"
+    if DOMAIN_RE.match(target):
+        return "domain"
+    if " " in target or COMPANY_HINTS.search(target):
+        return "company"
+    if USERNAME_RE.match(target):
+        return "username"
+    raise TargetParseError(f"could not tell what {target!r} is; pass --type explicitly")
+
+
+def resolve(target: str, declared: str | None = None) -> tuple[str, str]:
+    clean = normalise(target)
+    if not clean:
+        raise TargetParseError("empty target")
+    kind = (declared or "").lower() or None
+    if kind and kind not in VALID_TYPES:
+        raise TargetParseError(f"unknown type {kind!r}; expected one of {', '.join(VALID_TYPES)}")
+    if kind:
+        _check_shape(clean, kind)
+        return clean, kind
+    kind = detect_type(clean)
+    _check_shape(clean, kind)
+    return clean, kind
+
+
+def _check_shape(target: str, kind: str) -> None:
+    if kind == "email" and not EMAIL_RE.match(target):
+        raise TargetParseError(f"{target!r} is not a valid email")
+    if kind == "domain" and not DOMAIN_RE.match(target):
+        raise TargetParseError(f"{target!r} is not a valid domain")
+    if kind == "username" and not USERNAME_RE.match(target):
+        raise TargetParseError(f"{target!r} is not a valid username")
+    if kind == "company" and not COMPANY_RE.match(target):
+        raise TargetParseError(f"{target!r} does not look like a company name")

osintkit-0.1.1.dist-info/METADATA

@@ -0,0 +1,176 @@
+Metadata-Version: 2.5
+Name: osintkit
+Version: 0.1.1
+Summary: Public-data OSINT aggregator for emails, usernames, domains, and companies
+Project-URL: Homepage, https://github.com/sjdsjadjajdjsa/osintkit
+Project-URL: Issues, https://github.com/sjdsjadjajdjsa/osintkit/issues
+Author: osintkit contributors
+License-Expression: MIT
+License-File: LICENSE
+Keywords: cli,osint,privacy,recon,security
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Information Technology
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Security
+Classifier: Typing :: Typed
+Requires-Python: >=3.12
+Requires-Dist: dnspython>=2.6
+Requires-Dist: httpx>=0.27
+Requires-Dist: pydantic>=2.7
+Requires-Dist: python-whois>=0.9
+Requires-Dist: rich>=13.7
+Requires-Dist: typer>=0.12
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest-cov>=5; extra == 'dev'
+Requires-Dist: pytest>=8.2; extra == 'dev'
+Requires-Dist: respx>=0.21; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# osintkit
+
+[![ci](https://github.com/sjdsjadjajdjsa/osintkit/actions/workflows/ci.yml/badge.svg)](https://github.com/sjdsjadjajdjsa/osintkit/actions/workflows/ci.yml)
+[![python](https://img.shields.io/badge/python-3.12%20%7C%203.13-blue)](pyproject.toml)
+[![license](https://img.shields.io/badge/license-MIT-green)](LICENSE)
+[![ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+
+Public-data OSINT for one email, username, domain, or company: gathered from free sources,
+briefed by an LLM acting as a defensive analyst, rendered as a colour-coded report.
+
+`osintkit` uses only public sources: certificate transparency, public DNS, WHOIS, public GitHub
+search, free breach indexes, paste archives, and public profile pages. It never logs in on your
+behalf and never touches a paid data broker.
+
+<img src="docs/demo.svg" alt="osintkit report showing risk, breaches, and recommendations" width="820">
+
+A full trimmed report, including breach and account sections, is in
+[docs/sample-output.md](docs/sample-output.md).
+
+## Why
+
+Checking what the public record says about you is tedious: a dozen sites, a dozen formats, no
+synthesis. `osintkit` runs those checks concurrently, keeps the raw evidence, and turns it into
+a briefing with a risk level and concrete hardening advice. Everything is local, every source
+is named, and every gap is reported instead of hidden.
+
+Intended for personal privacy checks, authorised pre-engagement recon, and quick due diligence
+on a domain or company. Findings are leads to verify, not facts.
+
+## Install
+
+```bash
+uv tool install git+https://github.com/sjdsjadjajdjsa/osintkit
+# or
+pipx install git+https://github.com/sjdsjadjajdjsa/osintkit
+```
+
+[Releases](https://github.com/sjdsjadjajdjsa/osintkit/releases) carry a signed-checksum wheel;
+`pip install osintkit-0.1.0-py3-none-any.whl` works from one. From a clone,
+`pip install -e .` also works. Python 3.12+ only.
+
+## Quick start
+
+```bash
+export API_KEY=...                 # model key, any OpenAI-compatible endpoint
+osintkit doctor --deep             # prove the key and endpoint work
+osintkit lookup alice@example.com
+```
+
+No key? `osintkit lookup example.com --raw` still collects everything; `--dry-run` shows the
+exact synthesis prompt.
+
+Any OpenAI-compatible endpoint works. `--provider venice|openrouter|openai` picks a preset, and
+a recognisable key prefix picks one for you; `--base-url` and `--model` always win.
+
+Optional keys: `HIBP_API_KEY` turns on Have I Been Pwned, `GITHUB_TOKEN` lifts GitHub from 60 to
+5000 requests/hour. Everything else needs no credentials.
+
+## Commands
+
+| Command | Purpose |
+| --- | --- |
+| `lookup` | One target, one report (`--email`, `--username`, `--domain`, `--company`) |
+| `batch` | A CSV of targets, run concurrently, reported in order |
+| `diff` | What changed between two archived runs |
+| `history` | Past runs, replayable by id |
+| `doctor` | Key, endpoint, and directory checks (`--deep` verifies auth) |
+| `collectors` | Every collector and what it touches |
+| `cache` / `prune` | Inspect, clear, or age out local state |
+| `schema` | The JSON contract the model must satisfy |
+
+Output: default colour report, `--json`, `--markdown` (they combine), `--raw` for collected
+data, `--dry-run` for the prompt. Exit codes: `0` ok, `1` synthesis or partial failure, `2` bad
+input.
+
+## How it works
+
+```
+target -> type detection -> collectors (parallel, cached, retried)
+                                 |
+                                 v
+                     merge + risk signals (deterministic)
+                                 |
+                                 v
+           LLM briefing -> pydantic validation -> one repair retry
+                                 |
+                                 v
+              colour report . JSON . Markdown . archive
+```
+
+Collectors degrade to a logged gap, never a dead run; a report is produced even when sources or
+the model fail.
+
+- **Merged breaches.** Three indexes listing one incident collapse into one row with every
+  source and the earliest date.
+- **Risk floor.** Breach count and credential types set a floor the model may exceed but never
+  undercut.
+- **Cached everything.** 24h default, shorter for volatile sources; repeat prompts cost nothing.
+- **Validated output.** Strict JSON schema, one repair round-trip, then a clean failure.
+
+## Collectors
+
+Two dozen sources across four target types, all free and login-free: HIBP (key), XposedOrNot,
+LeakCheck, Gravatar, crt.sh (subdomains, certificates, organisation search), public DNS
+(A/AAAA/MX/TXT/NS/CNAME plus SPF, DMARC, DKIM, wildcard), passive subdomains, WHOIS, TCP
+connect probes, GitHub (profile, commits, user search, org repos), psbdmp with a Gist fallback,
+grep.app, Hacker News, Stack Exchange, and username presence. The full table is in
+[docs/REFERENCE.md](docs/REFERENCE.md#collectors) or via `osintkit collectors`.
+
+## Safety and legality
+
+- Public data only. No logins, no scraping behind auth, no paid brokers.
+- Targets are typed and normalised before any request.
+- Requests retry on `429`/`5xx` with backoff and honour GitHub quota headers.
+- Results are evidence, not ground truth; the report says which sources were unavailable.
+- Cache, logs, and reports stay under `~/.osintkit` (override with `OSINTKIT_HOME`). Retention
+  is 30 days by default.
+
+## Limitations
+
+WHOIS is often redacted. GitHub search is shallow. Free breach indexes lag. Port probing only
+sees what your network can reach. Model summaries can overstate weak signals. Read the coverage
+panel: an empty section may mean "nothing found" or "source skipped".
+
+## Development
+
+```bash
+git clone https://github.com/sjdsjadjajdjsa/osintkit
+cd osintkit
+uv sync --extra dev
+pytest
+uvx ruff check src tests
+```
+
+Tests never touch the network: HTTP is mocked with `respx`, DNS and WHOIS are monkeypatched, and
+a recorded model reply covers the full-stack path. CI runs on 3.12 and 3.13. See
+[CONTRIBUTING.md](CONTRIBUTING.md) for the collector contract and
+[docs/architecture.md](docs/architecture.md) for the module map.
+
+## Licence
+
+MIT. Security reports: [SECURITY.md](SECURITY.md).
+
+`osintkit` is independent tooling. It is not affiliated with any source it queries.

osintkit-0.1.1.dist-info/RECORD

@@ -0,0 +1,33 @@
+osintkit/__init__.py,sha256=nzOmu0EQ5zD3bXHXAG09l7UM6UN-hO_5H9kFS-vByGg,71
+osintkit/cli.py,sha256=IOn7YGApgP6Dt1oO-U8Jc_m9Hmm99dgkDAoxlZw3ZkM,25214
+osintkit/config.py,sha256=mYIazpKR4R1A75q4QiEWB00D4tjAQ3qWtpQxf94XvhM,4314
+osintkit/diff.py,sha256=fY4Y-bEmqugUkZsul23zVwC3YBRchHqwphl0NuvpCRI,3536
+osintkit/doctor.py,sha256=Fvyrg6Ke_5W6XYi8FAFwI9L1ReihE4dSPYtL409X52Y,4176
+osintkit/errors.py,sha256=LpWYG6-BJeSOC7TZhJz_FxOMw_Nqwag3Odpm6nojeUw,1145
+osintkit/http.py,sha256=wt0sWXpEpREJZrFaR--7OuEsyaCbFdENTw3fOmbm2AM,9461
+osintkit/llm.py,sha256=jG_ubL_3_X324V46-Cu_0tZmSN2kAhuMFN92IdpoLdE,3237
+osintkit/logging_store.py,sha256=erYmgS-TAqP7uXBhB2KCmMUWHLDR-7OcSOLXJrWGGWU,6226
+osintkit/models.py,sha256=nRyg4rfRoU81koug53z5NuY-oX9uSlmgqAJejlP1JkI,7034
+osintkit/normalise.py,sha256=eR_cSFHG4W_6zg5YqlbGizQF_ycrhBLd02lvFBPjjQM,4508
+osintkit/pipeline.py,sha256=cKeI2vDf8F2GjOwbwFIReAZDATug_9iwThLODhnjv-Y,8072
+osintkit/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+osintkit/render.py,sha256=hJB87iwVjy-AU_p8rRTtHpkGoOjrb_RpVqf0hGasy1U,8459
+osintkit/retention.py,sha256=asCc5vNXxdyYLZ3V1LnlFzJxrOwDJYRdXkcFg4BkgLA,2332
+osintkit/synthesis.py,sha256=ZXGLV7RYjC2W6oNimC2TOLeVdLueXrycNcTOi3eGL8E,6101
+osintkit/targets.py,sha256=U7v5gobZmrWCF0nfKqBJkLO5U2o02_hckEcCZnP79wM,2447
+osintkit/collectors/__init__.py,sha256=H1dbtafNPTa9YV0ox8VUZma8PLRM6vk8JOtH3XmYRuM,482
+osintkit/collectors/accounts.py,sha256=gzLB5VBkHs7xReBpyfye12q1OBYxYAgr7gfFk33RRTU,2911
+osintkit/collectors/base.py,sha256=aDJtrfpuxdjMtS1nsPho1lXa_IX0B5IUzftJvPpwJqY,3892
+osintkit/collectors/breaches.py,sha256=HSwecmDdxuDkFVX33IUz6qTxvyetNVFZUCuGp452Vdk,5308
+osintkit/collectors/ct.py,sha256=HJBe9z0qTeqASLyMUT4WbuOsPwShzufuuoYoR9585f8,5232
+osintkit/collectors/dns.py,sha256=dceEKn01iUU6cd3FsJFBypb1uRYXmgw9Iihcid_d-BU,4519
+osintkit/collectors/github.py,sha256=BGusnCVp9WnABst_apqv5uAs9aeVc15WESE4nO8Nz-8,6468
+osintkit/collectors/pastes.py,sha256=0Zk3Qi3LG-iMTS2mSdStRuHSw7xN5sM0WzbZ0pexN7M,8810
+osintkit/collectors/ports.py,sha256=pXsRPcF5Z-0tnM7wP_0hGpJdwRIAf0P7IekfflNHK_k,2700
+osintkit/collectors/subdomains.py,sha256=i-osO4MyFAPcIrZ0blD3f8jIqauvPLaZ8QC36iaE_oo,3002
+osintkit/collectors/whois_lookup.py,sha256=tkXalwMdEh4DQnVrECnF77ZrvBd5kFOh6xr-tVr2GMM,1859
+osintkit-0.1.1.dist-info/METADATA,sha256=nFwcyLijcELGMC00PFTE8znwFRwZa-QJhFxya6bYx-4,7482
+osintkit-0.1.1.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+osintkit-0.1.1.dist-info/entry_points.txt,sha256=_aW-Q6e3ikA8kWDSUI9EtCTwzECK00H5S8tWtLVkB4Q,46
+osintkit-0.1.1.dist-info/licenses/LICENSE,sha256=RqsRrtEfRQOE5mURJPG7DJB1IUTa7XODM8FlMS7gUEM,1078
+osintkit-0.1.1.dist-info/RECORD,,

osintkit-0.1.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

osintkit-0.1.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+osintkit = osintkit.cli:app

osintkit-0.1.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 osintkit contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.