reconkit 0.1.0__py3-none-any.whl

reconkit/__init__.py

@@ -0,0 +1,22 @@
+"""reconkit.
+
+Public API: Noir Stack / Hexarch / Judicium surface lives under `reconkit.noir`.
+Internal modules remain available but are not the supported public interface.
+"""
+
+__version__ = "0.1.0"
+
+from reconkit.noir.primitives import (  # noqa: E402
+    EvidenceArtifact,
+    EvidenceBundle,
+    ProvenanceRecord,
+    VerificationResult,
+)
+
+__all__ = [
+    "__version__",
+    "EvidenceArtifact",
+    "EvidenceBundle",
+    "ProvenanceRecord",
+    "VerificationResult",
+]

reconkit/cli.py

@@ -0,0 +1,218 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+import typer
+from rich.console import Console
+
+from reconkit.noir import public_api
+from reconkit.noir.primitives import VerificationResult, json_dumps_audit
+
+app = typer.Typer(add_completion=False, no_args_is_help=True)
+console = Console()
+
+evidence_app = typer.Typer(no_args_is_help=True)
+boundary_app = typer.Typer(no_args_is_help=True)
+verify_app = typer.Typer(no_args_is_help=True)
+app.add_typer(evidence_app, name="evidence")
+app.add_typer(boundary_app, name="boundary")
+app.add_typer(verify_app, name="verify")
+
+
+def _emit(result: VerificationResult) -> None:
+    # Audit-grade requirement: stdout must be machine-parseable JSON.
+    # Rich is excellent at presentation and equally excellent at line-wrapping.
+    # We only need one of those properties.
+    typer.echo(json_dumps_audit(result.to_json_dict()))
+
+
+@evidence_app.command("acquire-web")
+def evidence_acquire_web(
+    url: str,
+    trust_boundary: str = typer.Option(
+        "public-internet",
+        help="Declared trust boundary for acquisition (audited, not inferred).",
+    ),
+    max_bytes: int = typer.Option(2_000_000, help="Max download size in bytes"),
+    timeout_s: float = typer.Option(20.0, help="Timeout in seconds"),
+    rps: float = typer.Option(1.0, help="Max requests per second"),
+    obey_robots: bool = typer.Option(True, help="Respect robots.txt"),
+    include_payload: bool = typer.Option(False, help="Include payload in output (base64)."),
+):
+    """Acquire a web response as evidence.
+
+    Conservative note: this is acquisition, not authentication. The system is many things.
+    Omniscient is not currently one of them.
+    """
+
+    artifact = public_api.acquire_web_artifact(
+        url,
+        trust_boundary=trust_boundary,
+        max_bytes=max_bytes,
+        timeout_s=timeout_s,
+        rps=rps,
+        obey_robots=obey_robots,
+    )
+    _emit(
+        VerificationResult(
+            verification_status="informational",
+            decision_basis="http-get acquisition within declared trust boundary",
+            provenance_confidence="medium",
+            subject={"url": url, "trust_boundary": trust_boundary},
+            findings={"artifact": artifact.to_json_dict(include_payload=include_payload)},
+            limitations=[
+                "Transport acquisition does not assert publisher identity or content authenticity.",
+            ],
+        )
+    )
+
+
+@evidence_app.command("derive-links")
+def evidence_derive_links(
+    url: str,
+    timeout_s: float = typer.Option(20.0, help="Timeout in seconds"),
+    rps: float = typer.Option(1.0, help="Max requests per second"),
+    obey_robots: bool = typer.Option(True, help="Respect robots.txt"),
+    include_payload: bool = typer.Option(False, help="Include payload in output (base64)."),
+):
+    """Acquire a web response and derive a link-list artifact."""
+
+    web = public_api.acquire_web_artifact(url, timeout_s=timeout_s, rps=rps, obey_robots=obey_robots)
+    derived = public_api.derive_links(web, base_url=url)
+    _emit(
+        VerificationResult(
+            verification_status="informational",
+            decision_basis="derived link extraction from acquired HTML",
+            provenance_confidence="medium",
+            subject={"url": url},
+            findings={
+                "source_artifact": web.to_json_dict(include_payload=False),
+                "derived_artifact": derived.to_json_dict(include_payload=include_payload),
+            },
+            limitations=[
+                "Link extraction is heuristic and may miss dynamically generated URLs.",
+            ],
+        )
+    )
+
+
+@evidence_app.command("derive-text")
+def evidence_derive_text(
+    url: str,
+    timeout_s: float = typer.Option(20.0, help="Timeout in seconds"),
+    rps: float = typer.Option(1.0, help="Max requests per second"),
+    obey_robots: bool = typer.Option(True, help="Respect robots.txt"),
+    include_payload: bool = typer.Option(False, help="Include payload in output (base64)."),
+):
+    """Acquire a web response and derive a readable-text artifact."""
+
+    web = public_api.acquire_web_artifact(url, timeout_s=timeout_s, rps=rps, obey_robots=obey_robots)
+    derived = public_api.derive_readable_text(web, base_url=url)
+    _emit(
+        VerificationResult(
+            verification_status="informational",
+            decision_basis="derived readable text from acquired HTML",
+            provenance_confidence="medium",
+            subject={"url": url},
+            findings={
+                "source_artifact": web.to_json_dict(include_payload=False),
+                "derived_artifact": derived.to_json_dict(include_payload=include_payload),
+            },
+            limitations=[
+                "Readability extraction removes scripts/styles and may omit critical context.",
+            ],
+        )
+    )
+
+
+@boundary_app.command("dns")
+def boundary_dns(domain: str, include_payload: bool = typer.Option(False, help="Include payload in output (base64).")):
+    """Observe DNS answers across the public-internet boundary."""
+
+    artifact = public_api.boundary_dns(domain)
+    _emit(
+        VerificationResult(
+            verification_status="informational",
+            decision_basis="DNS resolution observation",
+            provenance_confidence="medium",
+            subject={"domain": domain, "trust_boundary": "public-internet"},
+            findings={"artifact": artifact.to_json_dict(include_payload=include_payload)},
+            limitations=[
+                "DNS is an administrative naming system; answers are not identity proofs.",
+            ],
+        )
+    )
+
+
+@boundary_app.command("whois")
+def boundary_whois(domain: str, include_payload: bool = typer.Option(False, help="Include payload in output (base64).")):
+    """Observe WHOIS/RDAP administrative metadata across the public-internet boundary."""
+
+    artifact = public_api.boundary_whois(domain)
+    _emit(
+        VerificationResult(
+            verification_status="informational",
+            decision_basis="WHOIS observation",
+            provenance_confidence="low",
+            subject={"domain": domain, "trust_boundary": "public-internet"},
+            findings={"artifact": artifact.to_json_dict(include_payload=include_payload)},
+            limitations=[
+                "WHOIS fields vary by registrar/TLD and may be redacted or stale.",
+            ],
+        )
+    )
+
+
+@boundary_app.command("ip")
+def boundary_ip(ip: str, include_payload: bool = typer.Option(False, help="Include payload in output (base64).")):
+    """Observe RDAP/ASN administrative metadata across the public-internet boundary."""
+
+    artifact = public_api.boundary_ip(ip)
+    _emit(
+        VerificationResult(
+            verification_status="informational",
+            decision_basis="RDAP/ASN observation",
+            provenance_confidence="medium",
+            subject={"ip": ip, "trust_boundary": "public-internet"},
+            findings={"artifact": artifact.to_json_dict(include_payload=include_payload)},
+            limitations=[
+                "Administrative allocations do not imply actor attribution.",
+            ],
+        )
+    )
+
+
+@verify_app.command("file-hash")
+def verify_file_hash(
+    path: Path,
+    algo: str = typer.Option("sha256", help="md5|sha1|sha256"),
+):
+    """Verify (informational): compute a cryptographic digest over file bytes."""
+
+    _emit(public_api.verify_file_hash(path, algo=algo))
+
+
+@verify_app.command("file-type")
+def verify_file_type(path: Path):
+    """Verify (informational): identify file type by signature."""
+
+    _emit(public_api.verify_file_type(path))
+
+
+@verify_app.command("strings")
+def verify_strings(
+    path: Path,
+    min_len: int = typer.Option(6, "--min-len"),
+    max_strings: Optional[int] = typer.Option(None, "--max"),
+):
+    """Verify (informational): extract printable strings as a triage aid."""
+
+    _emit(public_api.verify_strings(path, min_len=min_len, max_strings=max_strings))
+
+
+@verify_app.command("exif")
+def verify_exif(path: Path):
+    """Verify (informational): extract EXIF tags."""
+
+    _emit(public_api.verify_exif(path))

reconkit/core/rate_limit.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+
+
+@dataclass
+class RateLimiter:
+    """Simple, process-local rate limiter.
+
+    This intentionally stays basic (no jitter, no token bucket) to be easy to understand and
+    replicate.
+    """
+
+    requests_per_second: float = 1.0
+    _next_allowed_at: float = 0.0
+
+    def wait(self) -> None:
+        if self.requests_per_second <= 0:
+            return
+        now = time.monotonic()
+        if now < self._next_allowed_at:
+            time.sleep(self._next_allowed_at - now)
+        self._next_allowed_at = time.monotonic() + (1.0 / self.requests_per_second)

reconkit/forensics/exif.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import exifread
+
+
+def extract_exif(path: Path) -> dict[str, str]:
+    with path.open("rb") as f:
+        tags = exifread.process_file(f, details=False)
+
+    # normalize to plain strings
+    out: dict[str, str] = {}
+    for k, v in tags.items():
+        out[str(k)] = str(v)
+    return out

reconkit/forensics/hashing.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+
+
+_SUPPORTED = {"md5", "sha1", "sha256"}
+
+
+def hash_file(path: Path, *, algo: str = "sha256", chunk_size: int = 1024 * 1024) -> str:
+    algo = algo.lower()
+    if algo not in _SUPPORTED:
+        raise ValueError(f"Unsupported algo: {algo}. Choose one of: {sorted(_SUPPORTED)}")
+
+    h = hashlib.new(algo)
+    with path.open("rb") as f:
+        while True:
+            chunk = f.read(chunk_size)
+            if not chunk:
+                break
+            h.update(chunk)
+    return h.hexdigest()

reconkit/forensics/identify.py

@@ -0,0 +1,12 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import filetype
+
+
+def identify_file(path: Path) -> dict[str, str | None]:
+    kind = filetype.guess(str(path))
+    if kind is None:
+        return {"path": str(path), "ext": None, "mime": None}
+    return {"path": str(path), "ext": kind.extension, "mime": kind.mime}

reconkit/forensics/strings.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Iterator, Optional
+
+
+_PRINTABLE = set(range(0x20, 0x7F))  # basic ASCII printable
+
+
+def extract_strings(
+    path: Path,
+    *,
+    min_len: int = 6,
+    max_strings: Optional[int] = None,
+    chunk_size: int = 1024 * 1024,
+) -> Iterator[str]:
+    """Extract basic ASCII printable strings.
+
+    Intentionally simple (easy to replicate). For deeper analysis, consider specialized tools.
+    """
+
+    buf = bytearray()
+    yielded = 0
+
+    def flush() -> Iterator[str]:
+        nonlocal buf, yielded
+        if len(buf) >= min_len:
+            yielded += 1
+            yield buf.decode("ascii", errors="ignore")
+        buf = bytearray()
+
+    with path.open("rb") as f:
+        while True:
+            chunk = f.read(chunk_size)
+            if not chunk:
+                break
+            for b in chunk:
+                if b in _PRINTABLE:
+                    buf.append(b)
+                else:
+                    for s in flush():
+                        yield s
+                        if max_strings is not None and yielded >= max_strings:
+                            return
+
+    for s in flush():
+        yield s

reconkit/noir/__init__.py

@@ -0,0 +1,54 @@
+"""Noir Stack / Hexarch / Judicium public interface.
+
+Deliberately conservative: explicit provenance over convenience.
+If you want a tool that invents facts, there are easier ways to do that.
+"""
+
+from .errors import (
+	AcquisitionFailureError,
+	EvidencePolicyError,
+	EvidenceSizeLimitError,
+	NoirStackError,
+	ProvenanceRequiredError,
+	RobotsDisallowedError,
+	TrustBoundaryError,
+)
+from .primitives import EvidenceArtifact, EvidenceBundle, ProvenanceRecord, VerificationResult
+from .public_api import (
+	acquire_web_artifact,
+	boundary_dns,
+	boundary_ip,
+	boundary_whois,
+	bundle,
+	derive_links,
+	derive_readable_text,
+	verify_exif,
+	verify_file_hash,
+	verify_file_type,
+	verify_strings,
+)
+
+__all__ = [
+	"EvidenceArtifact",
+	"EvidenceBundle",
+	"ProvenanceRecord",
+	"VerificationResult",
+	"NoirStackError",
+	"TrustBoundaryError",
+	"ProvenanceRequiredError",
+	"EvidencePolicyError",
+	"RobotsDisallowedError",
+	"EvidenceSizeLimitError",
+	"AcquisitionFailureError",
+	"acquire_web_artifact",
+	"derive_links",
+	"derive_readable_text",
+	"boundary_dns",
+	"boundary_whois",
+	"boundary_ip",
+	"verify_file_hash",
+	"verify_file_type",
+	"verify_strings",
+	"verify_exif",
+	"bundle",
+]

reconkit/noir/errors.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+
+class NoirStackError(Exception):
+    """Base class for Noir Stack domain errors."""
+
+
+class TrustBoundaryError(NoirStackError):
+    """Operation attempted across an unapproved or undefined trust boundary."""
+
+
+class ProvenanceRequiredError(NoirStackError):
+    """Raised when an operation requires provenance and none is provided."""
+
+
+class EvidencePolicyError(NoirStackError):
+    """Raised when evidence acquisition violates a declared policy."""
+
+
+class RobotsDisallowedError(EvidencePolicyError):
+    """Raised when robots.txt forbids acquisition for the declared agent."""
+
+
+class EvidenceSizeLimitError(EvidencePolicyError):
+    """Raised when an artifact exceeds declared acquisition limits."""
+
+
+class AcquisitionFailureError(NoirStackError):
+    """Raised when acquisition fails within the declared boundary."""

reconkit/noir/primitives.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+import base64
+import hashlib
+import json
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Literal, Optional
+from uuid import uuid4
+
+VerificationStatus = Literal[
+    "verified",
+    "unverified",
+    "inconclusive",
+    "blocked",
+    "informational",
+]
+
+ProvenanceConfidence = Literal[
+    "high",
+    "medium",
+    "low",
+    "unknown",
+]
+
+
+def _utc_now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+@dataclass(frozen=True)
+class ProvenanceRecord:
+    """Audit-grade statement of how an artifact was obtained.
+
+    This is not a promise of truth. It is a statement of method, boundary, and limits.
+    """
+
+    source: str
+    acquisition_method: str
+    trust_boundary: str
+    observed_at: datetime = field(default_factory=_utc_now)
+    observer: str = "reconkit.judicium"
+    notes: list[str] = field(default_factory=list)
+
+
+@dataclass(frozen=True)
+class EvidenceArtifact:
+    """Opaque artifact plus provenance.
+
+    Treat payload as immutable evidence; derive additional artifacts rather than mutating.
+    """
+
+    artifact_id: str
+    kind: str
+    payload: bytes
+    media_type: Optional[str]
+    encoding: Optional[str]
+    provenance: ProvenanceRecord
+    sha256: str
+    size_bytes: int
+
+    @staticmethod
+    def from_bytes(
+        *,
+        kind: str,
+        payload: bytes,
+        provenance: ProvenanceRecord,
+        media_type: Optional[str] = None,
+        encoding: Optional[str] = None,
+        artifact_id: Optional[str] = None,
+    ) -> "EvidenceArtifact":
+        digest = hashlib.sha256(payload).hexdigest()
+        return EvidenceArtifact(
+            artifact_id=artifact_id or str(uuid4()),
+            kind=kind,
+            payload=payload,
+            media_type=media_type,
+            encoding=encoding,
+            provenance=provenance,
+            sha256=digest,
+            size_bytes=len(payload),
+        )
+
+    def to_json_dict(self, *, include_payload: bool = False) -> dict[str, Any]:
+        out: dict[str, Any] = {
+            "artifact_id": self.artifact_id,
+            "kind": self.kind,
+            "sha256": self.sha256,
+            "size_bytes": self.size_bytes,
+            "media_type": self.media_type,
+            "encoding": self.encoding,
+            "provenance": {
+                "source": self.provenance.source,
+                "acquisition_method": self.provenance.acquisition_method,
+                "trust_boundary": self.provenance.trust_boundary,
+                "observed_at": self.provenance.observed_at.isoformat(),
+                "observer": self.provenance.observer,
+                "notes": list(self.provenance.notes),
+            },
+        }
+        if include_payload:
+            out["payload_b64"] = base64.b64encode(self.payload).decode("ascii")
+        return out
+
+
+@dataclass(frozen=True)
+class EvidenceBundle:
+    """A set of artifacts intended to be assessed together."""
+
+    bundle_id: str
+    artifacts: list[EvidenceArtifact]
+    created_at: datetime = field(default_factory=_utc_now)
+    notes: list[str] = field(default_factory=list)
+
+    @staticmethod
+    def of(*artifacts: EvidenceArtifact, notes: Optional[list[str]] = None) -> "EvidenceBundle":
+        return EvidenceBundle(
+            bundle_id=str(uuid4()),
+            artifacts=list(artifacts),
+            notes=list(notes or []),
+        )
+
+
+@dataclass(frozen=True)
+class VerificationResult:
+    """Result of a verification action.
+
+    Required fields for structured output:
+    - decision_basis
+    - verification_status
+    - provenance_confidence
+    """
+
+    verification_status: VerificationStatus
+    decision_basis: str
+    provenance_confidence: ProvenanceConfidence
+
+    subject: dict[str, Any] = field(default_factory=dict)
+    findings: dict[str, Any] = field(default_factory=dict)
+    limitations: list[str] = field(default_factory=list)
+
+    def to_json_dict(self) -> dict[str, Any]:
+        return {
+            "verification_status": self.verification_status,
+            "decision_basis": self.decision_basis,
+            "provenance_confidence": self.provenance_confidence,
+            "subject": self.subject,
+            "findings": self.findings,
+            "limitations": list(self.limitations),
+        }
+
+
+def json_dumps_audit(obj: Any) -> str:
+    return json.dumps(obj, indent=2, sort_keys=True, default=str)

reconkit/noir/public_api.py

@@ -0,0 +1,221 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Optional
+
+from reconkit.forensics import exif as exif_mod
+from reconkit.forensics import hashing, identify, strings as strings_mod
+from reconkit.osint import dns as dns_mod
+from reconkit.osint import ip as ip_mod
+from reconkit.osint import whois as whois_mod
+from reconkit.scraping import extract, fetch
+
+from .errors import NoirStackError
+from .primitives import EvidenceArtifact, EvidenceBundle, ProvenanceRecord, VerificationResult
+
+
+def acquire_web_artifact(
+    url: str,
+    *,
+    trust_boundary: str = "public-internet",
+    rps: float = 1.0,
+    timeout_s: float = 20.0,
+    max_bytes: int = 2_000_000,
+    obey_robots: bool = True,
+    user_agent: str = "reconkit/0.1 (+https://localhost)",
+) -> EvidenceArtifact:
+    """Acquire a web artifact as evidence.
+
+    Conservatism note: this only asserts transport-level acquisition. It does not assert truth.
+    """
+
+    prov = ProvenanceRecord(
+        source=url,
+        acquisition_method="http-get",
+        trust_boundary=trust_boundary,
+        notes=[
+            "Transport acquisition only; content authenticity is not asserted.",
+        ],
+    )
+
+    try:
+        result = fetch.fetch_url(
+            url,
+            max_bytes=max_bytes,
+            timeout_s=timeout_s,
+            rps=rps,
+            obey_robots=obey_robots,
+            user_agent=user_agent,
+        )
+    except NoirStackError:
+        raise
+
+    media_type = result.headers.get("content-type")
+    return EvidenceArtifact.from_bytes(
+        kind="web-response",
+        payload=result.body,
+        media_type=media_type,
+        encoding=result.encoding,
+        provenance=prov,
+    )
+
+
+def derive_links(artifact: EvidenceArtifact, *, base_url: str) -> EvidenceArtifact:
+    links = extract.extract_links(artifact.payload, base_url=base_url)
+    payload = ("\n".join(links) + "\n").encode("utf-8")
+    prov = ProvenanceRecord(
+        source=artifact.artifact_id,
+        acquisition_method="derived-links",
+        trust_boundary=artifact.provenance.trust_boundary,
+        notes=[
+            "Derived artifact; link extraction is heuristic and HTML-dependent.",
+        ],
+    )
+    return EvidenceArtifact.from_bytes(
+        kind="derived-links",
+        payload=payload,
+        media_type="text/plain",
+        encoding="utf-8",
+        provenance=prov,
+    )
+
+
+def derive_readable_text(artifact: EvidenceArtifact, *, base_url: str) -> EvidenceArtifact:
+    text = extract.readable_text(artifact.payload, base_url=base_url)
+    payload = (text + "\n").encode("utf-8")
+    prov = ProvenanceRecord(
+        source=artifact.artifact_id,
+        acquisition_method="derived-readable-text",
+        trust_boundary=artifact.provenance.trust_boundary,
+        notes=[
+            "Derived artifact; readability extraction removes scripts/styles and may omit context.",
+        ],
+    )
+    return EvidenceArtifact.from_bytes(
+        kind="derived-readable-text",
+        payload=payload,
+        media_type="text/plain",
+        encoding="utf-8",
+        provenance=prov,
+    )
+
+
+def boundary_dns(domain: str) -> EvidenceArtifact:
+    data = dns_mod.lookup(domain)
+    prov = ProvenanceRecord(
+        source=domain,
+        acquisition_method="dns-resolve",
+        trust_boundary="public-internet",
+        notes=[
+            "DNS answers are boundary observations, not identity proofs.",
+        ],
+    )
+    payload = json.dumps(data, indent=2, sort_keys=True).encode("utf-8")
+    return EvidenceArtifact.from_bytes(
+        kind="boundary-dns",
+        payload=payload,
+        media_type="application/json",
+        encoding="utf-8",
+        provenance=prov,
+    )
+
+
+def boundary_whois(domain: str) -> EvidenceArtifact:
+    data = whois_mod.lookup(domain)
+    prov = ProvenanceRecord(
+        source=domain,
+        acquisition_method="whois",
+        trust_boundary="public-internet",
+        notes=[
+            "WHOIS data is registrar-provided and may be redacted or stale.",
+        ],
+    )
+    payload = json.dumps(data, indent=2, sort_keys=True, default=str).encode("utf-8")
+    return EvidenceArtifact.from_bytes(
+        kind="boundary-whois",
+        payload=payload,
+        media_type="application/json",
+        encoding="utf-8",
+        provenance=prov,
+    )
+
+
+def boundary_ip(ip: str) -> EvidenceArtifact:
+    data = ip_mod.lookup(ip)
+    prov = ProvenanceRecord(
+        source=ip,
+        acquisition_method="ip-whois-rdap",
+        trust_boundary="public-internet",
+        notes=[
+            "RDAP/WHOIS is administrative metadata; attribution remains a separate claim.",
+        ],
+    )
+    payload = json.dumps(data, indent=2, sort_keys=True, default=str).encode("utf-8")
+    return EvidenceArtifact.from_bytes(
+        kind="boundary-ip",
+        payload=payload,
+        media_type="application/json",
+        encoding="utf-8",
+        provenance=prov,
+    )
+
+
+def verify_file_hash(path: Path, *, algo: str = "sha256") -> VerificationResult:
+    digest = hashing.hash_file(path, algo=algo)
+    return VerificationResult(
+        verification_status="informational",
+        decision_basis=f"computed {algo} over file bytes",
+        provenance_confidence="high",
+        subject={"path": str(path), "algo": algo},
+        findings={"digest": digest},
+        limitations=[
+            "Hash computation verifies byte stability, not file origin.",
+        ],
+    )
+
+
+def verify_file_type(path: Path) -> VerificationResult:
+    info = identify.identify_file(path)
+    return VerificationResult(
+        verification_status="informational",
+        decision_basis="magic-byte signature guess",
+        provenance_confidence="high",
+        subject={"path": str(path)},
+        findings=info,
+        limitations=[
+            "File type guess is heuristic; containers and polyglots may evade identification.",
+        ],
+    )
+
+
+def verify_exif(path: Path) -> VerificationResult:
+    data = exif_mod.extract_exif(path)
+    return VerificationResult(
+        verification_status="informational",
+        decision_basis="EXIF tag extraction",
+        provenance_confidence="high",
+        subject={"path": str(path)},
+        findings={"exif": data},
+        limitations=[
+            "EXIF may be absent, altered, or stripped; authenticity is not asserted.",
+        ],
+    )
+
+
+def verify_strings(path: Path, *, min_len: int = 6, max_strings: Optional[int] = None) -> VerificationResult:
+    strings = list(strings_mod.extract_strings(path, min_len=min_len, max_strings=max_strings))
+    return VerificationResult(
+        verification_status="informational",
+        decision_basis=f"ASCII printable strings, min_len={min_len}",
+        provenance_confidence="high",
+        subject={"path": str(path), "min_len": min_len, "max_strings": max_strings},
+        findings={"strings": strings},
+        limitations=[
+            "This extractor is ASCII-only and will miss many encodings.",
+        ],
+    )
+
+
+def bundle(*artifacts: EvidenceArtifact, notes: Optional[list[str]] = None) -> EvidenceBundle:
+    return EvidenceBundle.of(*artifacts, notes=notes)

reconkit/osint/dns.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+import dns.resolver
+
+
+def lookup(domain: str) -> dict[str, list[str]]:
+    resolver = dns.resolver.Resolver()
+    record_types = ["A", "AAAA", "MX", "TXT", "NS", "CNAME", "SOA"]
+    result: dict[str, list[str]] = {}
+
+    for rtype in record_types:
+        try:
+            answers = resolver.resolve(domain, rtype)
+            result[rtype] = [str(a).strip() for a in answers]
+        except Exception:
+            result[rtype] = []
+
+    return result

reconkit/osint/ip.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from typing import Any
+
+from ipwhois import IPWhois
+
+
+def lookup(ip: str) -> dict[str, Any]:
+    obj = IPWhois(ip)
+    data = obj.lookup_rdap(depth=1)
+
+    # Keep a small, stable subset for display
+    return {
+        "query": ip,
+        "asn": data.get("asn"),
+        "asn_cidr": data.get("asn_cidr"),
+        "asn_country_code": data.get("asn_country_code"),
+        "asn_description": data.get("asn_description"),
+        "network": {
+            "name": (data.get("network") or {}).get("name"),
+            "country": (data.get("network") or {}).get("country"),
+            "cidr": (data.get("network") or {}).get("cidr"),
+        },
+        "objects": list((data.get("objects") or {}).keys()),
+    }

reconkit/osint/whois.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+from typing import Any
+
+import whois
+
+
+def lookup(domain: str) -> dict[str, Any]:
+    """Perform a WHOIS lookup and normalize to a JSON-friendly dict."""
+
+    w = whois.whois(domain)
+    # python-whois returns a dict-like object with some date fields.
+    data: dict[str, Any] = dict(w)
+    return data

reconkit/scraping/extract.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from urllib.parse import urljoin
+
+from bs4 import BeautifulSoup
+
+
+def extract_links(html: bytes, *, base_url: str) -> list[str]:
+    soup = BeautifulSoup(html, "lxml")
+    out: list[str] = []
+    for a in soup.select("a[href]"):
+        href = a.get("href")
+        if not href or not isinstance(href, str):
+            continue
+        out.append(urljoin(base_url, href))
+    # de-dup while preserving order
+    seen: set[str] = set()
+    unique: list[str] = []
+    for u in out:
+        if u in seen:
+            continue
+        seen.add(u)
+        unique.append(u)
+    return unique
+
+
+def readable_text(html: bytes, *, base_url: str | None = None) -> str:
+    soup = BeautifulSoup(html, "lxml")
+
+    for tag in soup(["script", "style", "noscript"]):
+        tag.decompose()
+
+    text = soup.get_text("\n")
+    lines = [line.strip() for line in text.splitlines()]
+    lines = [line for line in lines if line]
+    return "\n".join(lines)

reconkit/scraping/fetch.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional
+
+import httpx
+
+from reconkit.core.rate_limit import RateLimiter
+from reconkit.scraping import robots
+from reconkit.noir.errors import AcquisitionFailureError, EvidenceSizeLimitError, RobotsDisallowedError
+
+
+@dataclass
+class FetchResult:
+    url: str
+    status_code: int
+    headers: dict[str, str]
+    body: bytes
+    encoding: Optional[str]
+
+
+def fetch_url(
+    url: str,
+    *,
+    max_bytes: int = 2_000_000,
+    timeout_s: float = 20.0,
+    rps: float = 1.0,
+    obey_robots: bool = True,
+    user_agent: str = "reconkit/0.1 (+https://localhost)",
+    retries: int = 2,
+) -> FetchResult:
+    """Fetch a URL with conservative defaults.
+
+    No anti-bot bypass behavior is implemented (no proxy rotation, no fingerprinting evasion).
+    """
+
+    limiter = RateLimiter(requests_per_second=rps)
+    limiter.wait()
+
+    headers = {"User-Agent": user_agent, "Accept": "text/html,application/xhtml+xml,*/*;q=0.8"}
+    timeout = httpx.Timeout(timeout_s)
+
+    with httpx.Client(follow_redirects=True, timeout=timeout, headers=headers) as client:
+        if obey_robots and not robots.can_fetch(url, user_agent=user_agent, timeout_s=min(10.0, timeout_s), client=client):
+            raise RobotsDisallowedError(
+                "Acquisition denied by robots.txt policy. "
+                "Trust boundary: remote publisher policy; provenance cannot be improved by bypass."
+            )
+
+        for attempt in range(retries + 1):
+            try:
+                with client.stream("GET", url) as resp:
+                    resp.raise_for_status()
+                    chunks: list[bytes] = []
+                    size = 0
+                    for chunk in resp.iter_bytes():
+                        chunks.append(chunk)
+                        size += len(chunk)
+                        if max_bytes and size > max_bytes:
+                            raise EvidenceSizeLimitError(
+                                f"Evidence size limit exceeded (max_bytes={max_bytes}). "
+                                "Trust boundary: acquisition constraints; partial evidence is not emitted by default."
+                            )
+                    body = b"".join(chunks)
+
+                    return FetchResult(
+                        url=str(resp.url),
+                        status_code=resp.status_code,
+                        headers={k: v for k, v in resp.headers.items()},
+                        body=body,
+                        encoding=resp.encoding,
+                    )
+            except (httpx.TimeoutException, httpx.NetworkError):
+                if attempt >= retries:
+                    raise AcquisitionFailureError(
+                        "Acquisition failed within the declared trust boundary (network/timeout). "
+                        "Decision basis: transport failure; provenance confidence unchanged."
+                    )
+                limiter.wait()
+
+        raise RuntimeError("Unreachable")

reconkit/scraping/robots.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+from urllib.parse import urlparse
+from urllib.robotparser import RobotFileParser
+
+import httpx
+
+_CACHE: dict[str, RobotFileParser] = {}
+
+
+def robots_url(url: str) -> str:
+    parsed = urlparse(url)
+    return f"{parsed.scheme}://{parsed.netloc}/robots.txt"
+
+
+def can_fetch(
+    url: str,
+    *,
+    user_agent: str,
+    timeout_s: float = 10.0,
+    client: httpx.Client | None = None,
+) -> bool:
+    """Return whether `user_agent` is allowed to fetch `url` per robots.txt.
+
+    Safe defaults:
+    - If robots.txt cannot be fetched/parsed, default to allow.
+    - Caches robots parsers per (scheme, netloc).
+    """
+
+    parsed = urlparse(url)
+    key = f"{parsed.scheme}://{parsed.netloc}"
+    rp = _CACHE.get(key)
+    if rp is None:
+        rp = RobotFileParser()
+        robots_txt = f"{key}/robots.txt"
+        rp.set_url(robots_txt)
+
+        try:
+            if client is None:
+                with httpx.Client(timeout=httpx.Timeout(timeout_s), follow_redirects=True) as c:
+                    resp = c.get(robots_txt)
+            else:
+                resp = client.get(robots_txt)
+
+            if resp.status_code >= 400:
+                # treat missing/blocked robots as allow
+                _CACHE[key] = rp
+                return True
+
+            rp.parse(resp.text.splitlines())
+        except Exception:
+            _CACHE[key] = rp
+            return True
+
+        _CACHE[key] = rp
+
+    try:
+        return bool(rp.can_fetch(user_agent, url))
+    except Exception:
+        return True

reconkit-0.1.0.dist-info/METADATA

@@ -0,0 +1,108 @@
+Metadata-Version: 2.4
+Name: reconkit
+Version: 0.1.0
+Summary: A small, safe-by-default toolkit for scraping, OSINT, and file triage.
+Author: reconkit
+License: MIT
+License-File: LICENSE
+Keywords: forensics,osint,scraping,triage
+Requires-Python: >=3.10
+Requires-Dist: beautifulsoup4>=4.12
+Requires-Dist: dnspython>=2.6
+Requires-Dist: exifread>=3.0
+Requires-Dist: filetype>=1.2
+Requires-Dist: httpx>=0.27
+Requires-Dist: ipwhois>=1.2
+Requires-Dist: lxml>=5.1
+Requires-Dist: python-whois>=0.9
+Requires-Dist: rich>=13.7
+Requires-Dist: typer>=0.12
+Provides-Extra: dev
+Requires-Dist: build>=1.2; extra == 'dev'
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.4; extra == 'dev'
+Requires-Dist: twine>=5.1; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# reconkit (Judicium surface)
+
+`reconkit` is a **conservative, safe-by-default** toolkit that exposes a Noir Stack / Hexarch / Judicium-flavored public API.
+
+It is designed to produce **audit-grade structured outputs** that explicitly state:
+- what was observed (evidence)
+- how it was observed (provenance)
+- what is and is not being asserted (verification limits)
+
+This project is intended for lawful use on systems/data you own or have permission to assess.
+
+## Noir Stack primitives
+
+Public primitives:
+- `EvidenceArtifact`: immutable bytes + provenance
+- `EvidenceBundle`: a set of artifacts for joint assessment
+- `ProvenanceRecord`: acquisition/derivation metadata + trust boundary
+- `VerificationResult`: structured decision output
+
+All structured outputs include:
+- `decision_basis`
+- `verification_status`
+- `provenance_confidence`
+
+Because this is Judicium, not divination.
+
+## Guarantees
+
+What the system will reliably do:
+- Preserve evidence payload bytes as acquired/derived, and compute stable SHA-256 for artifacts.
+- Emit structured, JSON-serializable outputs suitable for logging and later review.
+- Respect declared acquisition constraints (e.g., `max_bytes`) and treat them as policy boundaries.
+- Respect `robots.txt` by default for web acquisition.
+
+## Assumptions
+
+What the system assumes unless you override it:
+- `trust_boundary` is declared explicitly for acquisitions. The tool will not invent one.
+- Web acquisition is transport-level observation, not identity/authenticity proof.
+- Public-internet boundary data (DNS/WHOIS/RDAP) is administrative metadata.
+
+## Non-goals
+
+What the system explicitly does not do:
+- No anti-bot bypassing, fingerprint evasion, proxy-rotation orchestration, or credentialed scraping.
+- No claims of attribution, intent, or identity. (You can do that in your report; the tool will not do it for you.)
+- No “automatic truth.” The system is already busy being correct.
+
+## Install
+
+```bash
+python -m pip install -e ".[dev]"
+```
+
+## CLI quickstart (Noir terminology)
+
+All commands emit audit-grade JSON.
+
+```bash
+# Evidence acquisition / derivation
+reconkit evidence acquire-web "https://example.com" --trust-boundary public-internet
+reconkit evidence derive-links "https://example.com"
+reconkit evidence derive-text "https://example.com"
+
+# Boundary observations (OSINT-ish, but we say the quiet part out loud)
+reconkit boundary dns example.com
+reconkit boundary whois example.com
+reconkit boundary ip 8.8.8.8
+
+# Verification (informational)
+reconkit verify file-hash .\somefile.bin --algo sha256
+reconkit verify file-type .\somefile.bin
+reconkit verify strings .\somefile.bin --min-len 6
+reconkit verify exif .\photo.jpg
+```
+
+## Notes
+
+- If acquisition is blocked by `robots.txt`, that is not a “bug.” It is a boundary condition.
+- If you need features that reduce accountability, you are in the wrong repository. Convenient.

reconkit-0.1.0.dist-info/RECORD

@@ -0,0 +1,26 @@
+reconkit/__init__.py,sha256=-640z75cTTFaQnqNm43lxM_F9_gQVywAnp6ouC5ZVp8,496
+reconkit/cli.py,sha256=KZXT07CtZTvxEsbdRGIFPZOP0aBFnktyZ4YH-87MDY8,8365
+reconkit/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reconkit/core/rate_limit.py,sha256=fOsav7k-yIFVJNeijSf8Fh2p4Cd-hpLkXmmnws-sCPQ,667
+reconkit/forensics/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reconkit/forensics/exif.py,sha256=jwXoxy437iCF9SWWqhzUexR3MGL8iCVMMBwuqS-EewQ,367
+reconkit/forensics/hashing.py,sha256=09YNWEYNLSong9l9-qm8chzFy22t65FOnzaI2pkP-sI,595
+reconkit/forensics/identify.py,sha256=N_R8n5tMTuOkLvAl95aSqUEliAXtGULQZDC8hmSP-6k,341
+reconkit/forensics/strings.py,sha256=RU_n61URl2-9dA0Hj1NVi8mjRT87ma4tHDFnCB2F4ZQ,1245
+reconkit/noir/__init__.py,sha256=s-Ey2F6gV1a7bbqb-ObcKDAB_oyANRtalZKO2hVWrpI,1225
+reconkit/noir/errors.py,sha256=qvs_790TQuxtSs8DPi97C2hYZtu7bTCH2LJfWjR0Hjs,888
+reconkit/noir/primitives.py,sha256=PKcvs-EoZUO6yXm3AYFt_0wl7w3wnroTG0bwkfv7tlE,4549
+reconkit/noir/public_api.py,sha256=Wdp-WotZyZXktz9lw5nPKC6CNW4vX2hO5TfgMbkYVsc,7347
+reconkit/osint/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reconkit/osint/dns.py,sha256=4qMKgeDOxbz2AJLLSWClLPUWfCWldz-OUNFhuffwj1c,507
+reconkit/osint/ip.py,sha256=OBSRX8chGrYZKtJrWS0fbLL-iQuqzjDCwzWw4XZw1us,769
+reconkit/osint/whois.py,sha256=_sNM5kGdo_ems2q80Elzrxh3BnOMfv815bomFno4CfM,353
+reconkit/scraping/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reconkit/scraping/extract.py,sha256=IOqdum232kTnrap0ibcMSbsfBPT5-PXpTb8pWVVp9v8,1012
+reconkit/scraping/fetch.py,sha256=uGz8EYpkQzm8DIlqePXdH8T5gGOigKCdAq9x7PBqdgY,3096
+reconkit/scraping/robots.py,sha256=xOfAoc3ZhGGqpqPzHFxIid9wFYdYmANTMME-0T5-yR8,1596
+reconkit-0.1.0.dist-info/METADATA,sha256=XRqdONqZ_AplrigKOOCNg4iNcM7usXAc0SuWDb17JC4,3754
+reconkit-0.1.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+reconkit-0.1.0.dist-info/entry_points.txt,sha256=QmRcskGj_VbAP3HyCDGbm5uQ8XxYcRl9VSp-0Rfq5Rw,46
+reconkit-0.1.0.dist-info/licenses/LICENSE,sha256=1brMukln6lIrsZyBqbIiwIWTOMMgX1yWQwdamL-MJKs,1092
+reconkit-0.1.0.dist-info/RECORD,,

reconkit-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.28.0
+Root-Is-Purelib: true
+Tag: py3-none-any

reconkit-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+reconkit = reconkit.cli:app

reconkit-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Noir Stack LLC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.