dorksmith 0.1.0__py3-none-any.whl

dorksmith/__init__.py

@@ -0,0 +1,56 @@
+"""dorksmith — deterministic search-dork generator and OSINT/SOCMINT query engine.
+
+    from dorksmith import generate, expand_handle, validate_query
+    result = generate("example.com", "domain", "public-documents", options={"fileTypes": ["pdf"]})
+    for v in result.variants:
+        print(v.label, v.query)
+
+The engine is pure: same request + same catalogs = same output. Pass ``catalogs=load_catalogs(path)``
+to run with edited catalogs; the bundled ones are used otherwise.
+"""
+from __future__ import annotations
+
+from urllib.parse import quote as _url_quote
+
+from .analyzer import GOOGLE_WORD_LIMIT, QueryAnalysis, analyze_query
+from .catalogs import Catalogs, bundled_catalog_files, bundled_catalogs, load_catalogs
+from .errors import InputValidationError
+from .generator import (
+    DEFAULT_LIMITS, INPUT_TYPES, GenerateResult, Limits, OperatorUse, ValidateQueryResult, Variant, build_context,
+    expand_template, generate, validate_query,
+)
+from .handles import HANDLE_NOTICE, NOT_CHECKED, HandleExpandResult, HandleQuery, ProfileCandidate, escape_data_string, expand_handle
+from .infer import infer_input_type
+from .normalizer import (
+    canonicalize, domain_label, normalize_text, try_normalize_domain, try_normalize_email, try_normalize_url,
+    try_normalize_username, try_parse_filename, try_parse_iso_date,
+)
+from .placeholders import ALL_PLACEHOLDERS, OPTIONAL_BY_DEFAULT, placeholders_in
+from .quoting import exclusion, looks_like_syntax, operator_value, or_group, quote, safe_term, safe_terms, unquote
+from .validator import validate_catalogs
+
+__version__ = "0.1.0"
+
+
+def catalog_version(catalogs: Catalogs | None = None) -> str:
+    """Version string of the catalogs in use (from intents.json)."""
+    return (catalogs or bundled_catalogs()).catalog_version
+
+
+def search_url(query: str, engine: str = "google", catalogs: Catalogs | None = None) -> str:
+    """Search-engine URL for a query, built locally with percent-encoding."""
+    c = catalogs or bundled_catalogs()
+    template = c.operators.get(engine, {}).get("searchUrlTemplate", "https://www.google.com/search?q={query}")
+    return template.replace("{query}", _url_quote(query, safe="-_.!~*'()"))
+
+
+__all__ = [
+    "__version__", "ALL_PLACEHOLDERS", "Catalogs", "DEFAULT_LIMITS", "GOOGLE_WORD_LIMIT", "GenerateResult", "HANDLE_NOTICE",
+    "HandleExpandResult", "HandleQuery", "INPUT_TYPES", "InputValidationError", "Limits", "NOT_CHECKED", "OPTIONAL_BY_DEFAULT",
+    "OperatorUse", "ProfileCandidate", "QueryAnalysis", "ValidateQueryResult", "Variant", "analyze_query", "build_context",
+    "bundled_catalog_files", "bundled_catalogs", "canonicalize", "catalog_version", "domain_label", "escape_data_string",
+    "exclusion", "expand_handle", "expand_template", "generate", "infer_input_type", "load_catalogs", "looks_like_syntax",
+    "normalize_text", "operator_value", "or_group", "placeholders_in", "quote", "safe_term", "safe_terms", "search_url",
+    "try_normalize_domain", "try_normalize_email", "try_normalize_url", "try_normalize_username", "try_parse_filename",
+    "try_parse_iso_date", "unquote", "validate_catalogs", "validate_query",
+]

dorksmith/analyzer.py

@@ -0,0 +1,151 @@
+"""Quote-aware operator scanner with plain-language warnings."""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from .catalogs import Json
+from .normalizer import is_letter
+
+GOOGLE_WORD_LIMIT = 32
+
+
+@dataclass(frozen=True)
+class QueryAnalysis:
+    operators: list[str]
+    warnings: list[str]
+    has_errors: bool
+
+
+def _tokenize(query: str) -> list[tuple[str, bool]]:
+    tokens: list[tuple[str, bool]] = []
+    i, n = 0, len(query)
+    while i < n:
+        if query[i].isspace():
+            i += 1
+            continue
+        if query[i] == '"':
+            end = query.find('"', i + 1)
+            if end < 0:
+                end = n - 1
+            tokens.append((query[i + 1: max(i + 1, end)], True))
+            i = end + 1
+            continue
+        start = i
+        while i < n and not query[i].isspace():
+            if query[i] == '"':
+                end = query.find('"', i + 1)
+                i = n if end < 0 else end + 1
+                continue
+            i += 1
+        tokens.append((query[start:i], False))
+    return tokens
+
+
+def analyze_query(query: str, catalog: Json, by_token: dict[str, Json]) -> QueryAnalysis:
+    operators: list[str] = []
+    warnings: list[str] = []
+    has_errors = False
+
+    def use(token: str) -> None:
+        if token not in operators:
+            operators.append(token)
+
+    depth = 0
+    tokens = _tokenize(query)
+    quote_count = query.count('"')
+    if quote_count > 0:
+        use('"')
+    if quote_count % 2 == 1:
+        warnings.append("Unbalanced double quotes: the last phrase will not be treated as exact.")
+        has_errors = True
+
+    for text, quoted in tokens:
+        if quoted:
+            if "*" in text:
+                use("*")
+            continue
+        t = text
+        if not t:
+            continue
+        if t == "OR":
+            use("OR"); continue
+        if t == "AND":
+            use("AND"); continue
+        if t == "|":
+            use("|"); continue
+        if t in ("or", "and"):
+            warnings.append(f"Lower-case '{t}' is treated as an ordinary word; use upper-case {t.upper()} for a boolean operator.")
+            continue
+
+        body = t
+        while body and body[0] == "(":
+            depth += 1; use("("); body = body[1:]
+        closing = 0
+        while body and body[-1] == ")":
+            closing += 1; body = body[:-1]
+        depth -= closing
+
+        if body.startswith("AROUND("):
+            use("AROUND("); continue
+        if len(body) > 1 and body[0] == "-":
+            use("-"); body = body[1:]
+        elif len(body) > 1 and body[0] == "+":
+            use("+"); body = body[1:]
+        elif len(body) > 1 and body[0] == "~":
+            use("~"); body = body[1:]
+        if len(body) > 1 and body[0] == "#":
+            use("#"); continue
+        if len(body) > 1 and body[0] == "@":
+            use("@"); continue
+        if body == "*":
+            use("*"); continue
+        if ".." in body and any(ch.isdigit() for ch in body):
+            use(".."); continue
+        if len(body) > 1 and body[0] == "$" and all(ch.isdigit() or ch in ".," for ch in body[1:]):
+            use("$"); continue
+
+        colon = body.find(":")
+        if colon > 0 and all(is_letter(ch) for ch in body[:colon]):
+            prefix = body[: colon + 1].lower()
+            value = body[colon + 1:]
+            op = by_token.get(prefix)
+            if op:
+                use(prefix)
+                if prefix == "filetype:" and value.startswith("."):
+                    warnings.append("filetype: values take no leading dot (filetype:pdf).")
+                if prefix == "site:" and "://" in value:
+                    warnings.append("site: values take no scheme (site:example.com).")
+                if prefix in ("before:", "after:") and len(value) not in (4, 10):
+                    warnings.append(f"{prefix} expects YYYY-MM-DD or YYYY.")
+                if op.get("takesValue") and not value:
+                    warnings.append(f"{prefix} has no value.")
+                    has_errors = True
+            elif len(body[:colon]) <= 15 and "://" not in body:
+                warnings.append(f"'{prefix}' is not a known {catalog.get('engineName', '')} operator and will be searched as plain text.")
+
+    if depth != 0:
+        warnings.append("Unbalanced parentheses.")
+        has_errors = True
+
+    for token in operators:
+        op = by_token.get(token)
+        if not op:
+            continue
+        support = op.get("support")
+        caveats = op.get("caveats") or []
+        if support == "deprecated":
+            warnings.append(f"{op['token']} is deprecated: {caveats[0] if caveats else 'it no longer works.'}")
+        elif support == "unreliable":
+            warnings.append(f"{op['token']} is unreliable: {caveats[0] if caveats else 'results are inconsistent.'}")
+        elif support == "unknown":
+            warnings.append(f"{op['token']} has unknown support status.")
+
+    word_count = sum(1 for text, _ in tokens if text)
+    if word_count > GOOGLE_WORD_LIMIT:
+        warnings.append(f"Query has {word_count} terms; Google ignores everything after the first {GOOGLE_WORD_LIMIT}.")
+
+    return QueryAnalysis(operators, warnings, has_errors)
+
+
+_ = re  # keep re available for callers that extend the analyzer

dorksmith/catalogs.py

@@ -0,0 +1,73 @@
+"""Catalog loading. Catalogs are plain dicts mirroring the JSON files in the repository's data/ directory."""
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from importlib import resources
+from pathlib import Path
+from typing import Any
+
+Json = dict[str, Any]
+
+
+@dataclass(frozen=True)
+class Catalogs:
+    """The four catalogs the engine consumes. ``operators`` is keyed by engine id."""
+
+    operators: dict[str, Json]
+    intents: Json
+    platforms: Json
+    file_types: Json
+    _operator_index: dict[str, dict[str, Json]] = field(default_factory=dict, repr=False, compare=False)
+
+    @property
+    def catalog_version(self) -> str:
+        return str(self.intents.get("catalogVersion", ""))
+
+    def operator_index(self, engine: str) -> dict[str, Json]:
+        """token -> operator definition for an engine (cached)."""
+        if engine not in self._operator_index:
+            index: dict[str, Json] = {}
+            for op in self.operators[engine].get("operators", []):
+                index.setdefault(op["token"], op)
+            self._operator_index[engine] = index
+        return self._operator_index[engine]
+
+    def intent(self, intent_id: str) -> Json | None:
+        for i in self.intents.get("intents", []):
+            if i.get("id") == intent_id:
+                return i
+        return None
+
+
+def _assemble(files: dict[str, Json]) -> Catalogs:
+    operators = {v["engine"]: v for k, v in files.items() if k.startswith("operators.")}
+    return Catalogs(operators=operators, intents=files["intents.json"], platforms=files["platforms.json"], file_types=files["filetypes.json"])
+
+
+def load_catalogs(directory: str | Path) -> Catalogs:
+    """Load catalogs from a directory containing operators.<engine>.json, intents.json, platforms.json, filetypes.json."""
+    d = Path(directory)
+    files = {p.name: json.loads(p.read_text(encoding="utf-8")) for p in sorted(d.glob("*.json"))}
+    return _assemble(files)
+
+
+def bundled_catalog_files() -> dict[str, Json]:
+    """The raw catalog files embedded in the package, keyed by file name."""
+    root = resources.files("dorksmith") / "data"
+    out: dict[str, Json] = {}
+    for entry in sorted(root.iterdir(), key=lambda e: e.name):
+        if entry.name.endswith(".json"):
+            out[entry.name] = json.loads(entry.read_text(encoding="utf-8"))
+    return out
+
+
+_bundled: Catalogs | None = None
+
+
+def bundled_catalogs() -> Catalogs:
+    """The catalogs embedded in this package (loaded once)."""
+    global _bundled
+    if _bundled is None:
+        _bundled = _assemble(bundled_catalog_files())
+    return _bundled

dorksmith/cli.py

@@ -0,0 +1,124 @@
+"""Command line: dorksmith generate|handle|validate|intents|operators|platforms|filetypes."""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from . import __version__, bundled_catalogs, catalog_version, expand_handle, generate, infer_input_type, search_url, validate_query
+from .errors import InputValidationError
+
+
+def _csv(value: str | None) -> list[str]:
+    return [v.strip() for v in value.split(",") if v.strip()] if value else []
+
+
+def _parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="dorksmith", description=f"Deterministic search-dork generator (catalog {catalog_version()}, package {__version__}).")
+    sub = p.add_subparsers(dest="command", required=True)
+
+    g = sub.add_parser("generate", help="generate ranked query variants")
+    g.add_argument("input", nargs="+", help="target text (domain, name, handle, email, phrase, ...)")
+    g.add_argument("--type", dest="input_type", help="input type; inferred when omitted")
+    g.add_argument("--intent", help="intent id; first compatible intent when omitted")
+    g.add_argument("--filetypes", help="comma-separated extensions")
+    g.add_argument("--exclude", help="comma-separated excluded terms")
+    g.add_argument("--after"); g.add_argument("--before"); g.add_argument("--site")
+    g.add_argument("--max", type=int, dest="max_variants")
+    g.add_argument("--organization"); g.add_argument("--location"); g.add_argument("--role"); g.add_argument("--display-name", dest="display_name")
+    g.add_argument("--urls", action="store_true", help="print a search URL under each variant")
+    g.add_argument("--json", action="store_true")
+
+    h = sub.add_parser("handle", help="expand a handle into profile URLs and queries")
+    h.add_argument("username")
+    h.add_argument("--categories"); h.add_argument("--platforms"); h.add_argument("--max", type=int, dest="max_platforms")
+    h.add_argument("--json", action="store_true")
+
+    v = sub.add_parser("validate", help="analyse a hand-written query")
+    v.add_argument("query", nargs="+")
+    v.add_argument("--json", action="store_true")
+
+    for name in ("intents", "operators", "platforms", "filetypes"):
+        sub.add_parser(name, help=f"list the {name} catalog").add_argument("--json", action="store_true")
+    return p
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _parser().parse_args(argv)
+    catalogs = bundled_catalogs()
+    try:
+        if args.command == "generate":
+            text = " ".join(args.input)
+            known = {e["ext"] for e in catalogs.file_types["extensions"]}
+            input_type = args.input_type or infer_input_type(text, known) or "keyword"
+            intent = args.intent or next(i["id"] for i in catalogs.intents["intents"] if input_type in i["compatibleInputTypes"])
+            r = generate(text, input_type, intent, options={
+                "fileTypes": _csv(args.filetypes), "excludeTerms": _csv(args.exclude), "after": args.after, "before": args.before, "site": args.site,
+                "maxVariants": args.max_variants, "organization": args.organization, "location": args.location, "role": args.role, "displayName": args.display_name,
+            })
+            if args.json:
+                print(json.dumps(r.to_dict(), indent=2, ensure_ascii=False))
+                return 0
+            print(f"# {r.input_type} \"{r.normalized_input}\" · intent {r.intent} · catalog {r.catalog_version}")
+            for var in r.variants:
+                print(f"\n[{var.label}] {var.id}\n  {var.query}\n  ↳ {var.explanation}")
+                for w in var.warnings:
+                    print(f"  ⚠ {w}")
+                if args.urls:
+                    print(f"  {search_url(var.query)}")
+        elif args.command == "handle":
+            r = expand_handle(args.username, categories=_csv(args.categories), platform_ids=_csv(args.platforms), max_platforms=args.max_platforms)
+            if args.json:
+                print(json.dumps(r.to_dict(), indent=2, ensure_ascii=False))
+                return 0
+            print(f"# {r.normalized_username} · {len(r.profiles)} platforms · {r.notice}")
+            for p in r.profiles:
+                print(f"{p.platform_name:<28} {p.url or '(no reliable URL)'}")
+                if p.caveat:
+                    print(f"{'':<29}⚠ {p.caveat}")
+            print("\n# queries")
+            for q in r.queries:
+                print(f"  {q.query}")
+        elif args.command == "validate":
+            r = validate_query(" ".join(args.query))
+            if args.json:
+                print(json.dumps(r.to_dict(), indent=2, ensure_ascii=False))
+                return 0
+            for o in r.operators:
+                print(f"{o.token:<12} {o.support:<11} {o.name}")
+            for w in r.warnings:
+                print(f"⚠ {w}")
+            return 2 if r.has_errors else 0
+        elif args.command == "intents":
+            rows = catalogs.intents["intents"]
+            if args.json:
+                print(json.dumps([{k: i[k] for k in ("id", "label", "group", "compatibleInputTypes", "safety")} for i in rows], indent=2))
+            else:
+                for i in rows:
+                    print(f"{i['id']:<28} {','.join(i['compatibleInputTypes']):<52} {i['label']}")
+        elif args.command == "operators":
+            if args.json:
+                print(json.dumps(catalogs.operators, indent=2))
+            else:
+                for o in next(iter(catalogs.operators.values()))["operators"]:
+                    print(f"{o['token']:<14} {o['support']:<11} {'generates' if o.get('generate') else '         '}  {o['name']}")
+        elif args.command == "platforms":
+            if args.json:
+                print(json.dumps(catalogs.platforms, indent=2))
+            else:
+                for p in catalogs.platforms["platforms"]:
+                    print(f"{p['id']:<18} {p['category']:<20} {'' if p.get('enabled', True) else '(disabled) '}{p['profileUrlTemplate']}")
+        elif args.command == "filetypes":
+            if args.json:
+                print(json.dumps(catalogs.file_types, indent=2))
+            else:
+                for e in catalogs.file_types["extensions"]:
+                    print(f"{e['ext']:<12} {e['group']:<11} {e['label']}")
+        return 0
+    except InputValidationError as e:
+        print(f"error: {e.message}" + (f" ({e.field})" if e.field else ""), file=sys.stderr)
+        return 3 if e.unprocessable else 2
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

dorksmith/data/filetypes.json

@@ -0,0 +1,68 @@
+{
+  "schemaVersion": 1,
+  "catalogVersion": "2026-09-24",
+  "groups": [
+    { "id": "documents", "label": "Office documents", "extensions": ["pdf", "docx", "doc", "pptx", "ppt", "xlsx", "xls", "odt", "odp", "ods", "rtf", "txt", "csv"] },
+    { "id": "data", "label": "Data & configuration", "extensions": ["json", "xml", "yaml", "yml", "csv", "sql", "ini", "cfg", "conf", "env", "toml", "properties"] },
+    { "id": "logs", "label": "Logs & diagnostics", "extensions": ["log", "txt", "out", "err", "dmp", "trace"] },
+    { "id": "archives", "label": "Archives & backups", "extensions": ["zip", "tar", "gz", "tgz", "7z", "rar", "bak", "old", "sql"] },
+    { "id": "code", "label": "Source & build", "extensions": ["js", "map", "py", "php", "java", "cs", "sh", "ps1", "rb", "go"] },
+    { "id": "media", "label": "Media", "extensions": ["jpg", "jpeg", "png", "gif", "svg", "mp4", "mp3"] }
+  ],
+  "extensions": [
+    { "ext": "pdf",  "label": "PDF document",           "group": "documents", "weight": 100 },
+    { "ext": "docx", "label": "Word document",          "group": "documents", "weight": 95 },
+    { "ext": "doc",  "label": "Word 97-2003 document",  "group": "documents", "weight": 70 },
+    { "ext": "pptx", "label": "PowerPoint presentation","group": "documents", "weight": 90 },
+    { "ext": "ppt",  "label": "PowerPoint 97-2003",     "group": "documents", "weight": 60 },
+    { "ext": "xlsx", "label": "Excel workbook",         "group": "documents", "weight": 92 },
+    { "ext": "xls",  "label": "Excel 97-2003 workbook", "group": "documents", "weight": 65 },
+    { "ext": "odt",  "label": "OpenDocument text",      "group": "documents", "weight": 40 },
+    { "ext": "odp",  "label": "OpenDocument presentation", "group": "documents", "weight": 35 },
+    { "ext": "ods",  "label": "OpenDocument spreadsheet", "group": "documents", "weight": 35 },
+    { "ext": "rtf",  "label": "Rich text",              "group": "documents", "weight": 30 },
+    { "ext": "txt",  "label": "Plain text",             "group": "documents", "weight": 80 },
+    { "ext": "csv",  "label": "CSV data",               "group": "data",      "weight": 85 },
+    { "ext": "json", "label": "JSON data",              "group": "data",      "weight": 88, "defensive": true },
+    { "ext": "xml",  "label": "XML data",               "group": "data",      "weight": 82, "defensive": true },
+    { "ext": "yaml", "label": "YAML configuration",     "group": "data",      "weight": 75, "defensive": true },
+    { "ext": "yml",  "label": "YAML configuration",     "group": "data",      "weight": 74, "defensive": true },
+    { "ext": "sql",  "label": "SQL dump / script",      "group": "data",      "weight": 78, "defensive": true },
+    { "ext": "ini",  "label": "INI configuration",      "group": "data",      "weight": 60, "defensive": true },
+    { "ext": "cfg",  "label": "Configuration file",     "group": "data",      "weight": 58, "defensive": true },
+    { "ext": "conf", "label": "Configuration file",     "group": "data",      "weight": 62, "defensive": true },
+    { "ext": "env",  "label": "Environment file",       "group": "data",      "weight": 70, "defensive": true },
+    { "ext": "toml", "label": "TOML configuration",     "group": "data",      "weight": 45, "defensive": true },
+    { "ext": "properties", "label": "Java properties",  "group": "data",      "weight": 50, "defensive": true },
+    { "ext": "log",  "label": "Log file",               "group": "logs",      "weight": 84, "defensive": true },
+    { "ext": "out",  "label": "Process output",         "group": "logs",      "weight": 30, "defensive": true },
+    { "ext": "err",  "label": "Error output",           "group": "logs",      "weight": 30, "defensive": true },
+    { "ext": "dmp",  "label": "Memory dump",            "group": "logs",      "weight": 35, "defensive": true },
+    { "ext": "trace","label": "Trace file",             "group": "logs",      "weight": 28, "defensive": true },
+    { "ext": "zip",  "label": "ZIP archive",            "group": "archives",  "weight": 76, "defensive": true },
+    { "ext": "tar",  "label": "TAR archive",            "group": "archives",  "weight": 55, "defensive": true },
+    { "ext": "gz",   "label": "Gzip archive",           "group": "archives",  "weight": 56, "defensive": true },
+    { "ext": "tgz",  "label": "Gzipped TAR archive",    "group": "archives",  "weight": 50, "defensive": true },
+    { "ext": "7z",   "label": "7-Zip archive",          "group": "archives",  "weight": 48, "defensive": true },
+    { "ext": "rar",  "label": "RAR archive",            "group": "archives",  "weight": 46, "defensive": true },
+    { "ext": "bak",  "label": "Backup file",            "group": "archives",  "weight": 72, "defensive": true },
+    { "ext": "old",  "label": "Old / renamed file",     "group": "archives",  "weight": 52, "defensive": true },
+    { "ext": "js",   "label": "JavaScript",             "group": "code",      "weight": 60 },
+    { "ext": "map",  "label": "Source map",             "group": "code",      "weight": 66, "defensive": true },
+    { "ext": "py",   "label": "Python source",          "group": "code",      "weight": 50 },
+    { "ext": "php",  "label": "PHP source",             "group": "code",      "weight": 48 },
+    { "ext": "java", "label": "Java source",            "group": "code",      "weight": 40 },
+    { "ext": "cs",   "label": "C# source",              "group": "code",      "weight": 40 },
+    { "ext": "sh",   "label": "Shell script",           "group": "code",      "weight": 54, "defensive": true },
+    { "ext": "ps1",  "label": "PowerShell script",      "group": "code",      "weight": 52, "defensive": true },
+    { "ext": "rb",   "label": "Ruby source",            "group": "code",      "weight": 30 },
+    { "ext": "go",   "label": "Go source",              "group": "code",      "weight": 30 },
+    { "ext": "jpg",  "label": "JPEG image",             "group": "media",     "weight": 40 },
+    { "ext": "jpeg", "label": "JPEG image",             "group": "media",     "weight": 30 },
+    { "ext": "png",  "label": "PNG image",              "group": "media",     "weight": 40 },
+    { "ext": "gif",  "label": "GIF image",              "group": "media",     "weight": 20 },
+    { "ext": "svg",  "label": "SVG image",              "group": "media",     "weight": 25 },
+    { "ext": "mp4",  "label": "MP4 video",              "group": "media",     "weight": 25 },
+    { "ext": "mp3",  "label": "MP3 audio",              "group": "media",     "weight": 20 }
+  ]
+}