hashident 0.1.0__py3-none-any.whl

hashident/__init__.py

@@ -0,0 +1,3 @@
+"""hashident: identify likely hash algorithms from a hash string."""
+
+__version__ = "0.1.0"

hashident/cli.py

@@ -0,0 +1,228 @@
+"""Command-line interface for hashident."""
+
+from __future__ import annotations
+
+import argparse
+import functools
+import itertools
+import os
+import sys
+import textwrap
+from collections.abc import Iterable, Iterator, Sequence
+from dataclasses import dataclass
+from typing import Any, NoReturn
+
+from hashident import __version__, json_output
+from hashident.engine import identify
+from hashident.inputs import InputError, InputItem, gather
+from hashident.models import Context, Result
+from hashident.output import WIDTH, CompactTable, Positions, format_result, report_rule
+from hashident.style import Styler
+from hashident.terminal import COLOR_MODES, use_color
+from hashident.tips import ContextTip, context_tips
+
+_EPILOG = """\
+Input: hashes given as arguments are reported first, in order, then the
+lines of -f PATH. With neither, hashes are read from standard input when it
+is piped or redirected. In files and piped input, blank lines and lines
+starting with # are skipped. UTF-8 and UTF-16 (with a BOM) are understood.
+
+Output: one hash gets a full report with reasons and crack commands.
+Several hashes get a compact table, one row each (argument index or line
+number, best format, confidence, Hashcat mode), then a count per format.
+With -v, every hash gets a full report instead, under a rule naming it,
+and low-confidence candidates get details as well.
+
+JSON: --json prints one JSON object per hash, each on its own line (JSON
+Lines), never colored. --json ignores -v: JSON always includes everything.
+Keys, in schema_version 1: schema_version, input, source, position,
+context, matched, candidates, tips. Each candidate: rank, format, score,
+confidence, hashcat_mode, john_format, strength, strength_note, reasons,
+commands, notes. Each tip: context, top_format. Absent values are null.
+
+Quoting: hashes that contain $ (bcrypt, md5crypt, sha512crypt) or start
+with * (MySQL) must be wrapped in single quotes, in PowerShell and bash
+alike. Otherwise the shell expands them before hashident sees them:
+  hashident '$2a$05$LhayLxezLhK1LhWvKxCyLOj0j1u.Kj0jZ0pEmm134uzrQlFvQJLF6'
+
+Color: --color auto (the default) colors output only when it goes to a
+terminal and the NO_COLOR environment variable is unset or empty. Order of
+precedence: --color/--no-color, then NO_COLOR, then terminal detection.
+Piped or redirected output is plain unless you pass --color always.
+
+Exit status:
+  0  every hash matched a known format
+  1  none did
+  3  some did and some did not
+  2  invalid arguments, or input that cannot be read or has no hashes"""
+
+EXIT_ALL_MATCHED = 0
+EXIT_NONE_MATCHED = 1
+EXIT_ERROR = 2
+EXIT_SOME_MATCHED = 3
+
+# Help and errors are always plain: argparse decides on its own color (3.14+)
+# before --color is parsed, so it could not honour our flags anyway.
+_PARSER_OPTIONS: dict[str, Any] = {"color": False} if sys.version_info >= (3, 14) else {}
+
+
+class _Parser(argparse.ArgumentParser):
+    """ArgumentParser whose error messages are wrapped to 80 columns."""
+
+    def error(self, message: str) -> NoReturn:
+        self.print_usage(sys.stderr)
+        text = textwrap.fill(f"{self.prog}: error: {message}", width=WIDTH, subsequent_indent="  ")
+        self.exit(2, text + "\n")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = _Parser(
+        prog="hashident",
+        description="Identify likely hash algorithms from a hash string.",
+        epilog=_EPILOG,
+        formatter_class=functools.partial(argparse.RawDescriptionHelpFormatter, width=WIDTH),
+        **_PARSER_OPTIONS,
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"%(prog)s {__version__}",
+    )
+    parser.add_argument(
+        "--from",
+        dest="context",
+        type=str.lower,
+        choices=[c.value for c in Context],
+        help="where the hash came from; ranks formats common there higher",
+    )
+    color = parser.add_mutually_exclusive_group()
+    color.add_argument(
+        "--color",
+        type=str.lower,
+        choices=COLOR_MODES,
+        help="when to use color (default: auto)",
+    )
+    color.add_argument(
+        "--no-color",
+        dest="color",
+        action="store_const",
+        const="never",
+        help="same as --color never",
+    )
+    parser.set_defaults(color="auto")
+    parser.add_argument(
+        "-f",
+        "--file",
+        metavar="PATH",
+        help="read hashes from PATH, one per line; - means standard input",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="full report for every hash, with details of low-confidence candidates too",
+    )
+    parser.add_argument(
+        "--json",
+        action="store_true",
+        help="print JSON, one object per hash per line (see JSON below)",
+    )
+    parser.add_argument("hashes", nargs="*", metavar="hash", help="a hash to identify")
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    try:
+        items = gather(args.hashes, args.file)
+        if items is None:
+            parser.print_help()
+            return EXIT_ALL_MATCHED
+
+        first, second = next(items, None), next(items, None)
+        if first is None:
+            return _fail(parser, "no hashes found in the input; see hashident --help")
+        several = second is not None
+        values = itertools.chain((first, second) if several else (first,), items)
+
+        tally = _Tally()
+        analysed = _analyse(values, args.context, tally)
+        if args.json:
+            # Checked before use_color, which may change the console's mode.
+            _print_json(analysed)
+        else:
+            style = Styler(enabled=use_color(args.color, sys.stdout, os.environ))
+            if several and not args.verbose:
+                _print_compact(analysed, CompactTable(_positions(args), style))
+            else:
+                _print_reports(analysed, style, several=several, show_low=args.verbose)
+        return tally.exit_status()
+    except InputError as exc:
+        return _fail(parser, str(exc))
+
+
+_Analysed = Iterator[tuple[InputItem, Result, tuple[ContextTip, ...]]]
+
+
+@dataclass
+class _Tally:
+    """Counts of values with and without a match, for the exit status."""
+
+    matched: int = 0
+    unmatched: int = 0
+
+    def exit_status(self) -> int:
+        if not self.unmatched:
+            return EXIT_ALL_MATCHED
+        return EXIT_SOME_MATCHED if self.matched else EXIT_NONE_MATCHED
+
+
+def _analyse(items: Iterable[InputItem], context: str | None, tally: _Tally) -> _Analysed:
+    """Identify each value as it arrives, counting matches into ``tally``."""
+    for item in items:
+        result = identify(item.value, context=context)
+        if result.candidates:
+            tally.matched += 1
+        else:
+            tally.unmatched += 1
+        yield item, result, context_tips(result)
+
+
+def _print_json(analysed: _Analysed) -> None:
+    for item, result, tips in analysed:
+        print(json_output.dumps(json_output.to_record(item, result, tips)))
+
+
+def _print_reports(analysed: _Analysed, style: Styler, *, several: bool, show_low: bool) -> None:
+    """Full reports; with several values, each under a rule naming it."""
+    for index, (item, result, tips) in enumerate(analysed):
+        if several:
+            if index:
+                print()
+            print(report_rule(item))
+        print(format_result(result, tips=tips, style=style, show_low=show_low))
+
+
+def _print_compact(analysed: _Analysed, table: CompactTable) -> None:
+    print("\n".join(table.header()))
+    for item, result, tips in analysed:
+        print(table.row(item, result, tips))
+    print("\n".join(table.footer()))
+
+
+def _positions(args: argparse.Namespace) -> Positions:
+    """What the compact table's first column counts, given the inputs used."""
+    if not args.hashes:
+        return Positions.LINE  # -f, or piped stdin
+    return Positions.ARGUMENT if args.file is None else Positions.MIXED
+
+
+def _fail(parser: argparse.ArgumentParser, message: str) -> int:
+    """Print a one-line error (never wrapped: it may hold a path)."""
+    print(f"{parser.prog}: error: {message}", file=sys.stderr)
+    return EXIT_ERROR
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

hashident/commands.py

@@ -0,0 +1,39 @@
+"""Ready-to-run cracking commands for a signature.
+
+Kept separate from text formatting so other output formats (e.g. JSON)
+can reuse the same commands and notes.
+"""
+
+from __future__ import annotations
+
+from hashident.models import Signature
+from hashident.signatures import MYSQL41
+
+HASH_FILE = "hash.txt"
+WORDLIST = "wordlist.txt"
+
+
+def crack_commands(sig: Signature) -> tuple[str, ...]:
+    """Hashcat and John wordlist commands, for the tools that support ``sig``."""
+    commands = []
+    if sig.hashcat_mode is not None:
+        commands.append(f"hashcat -m {sig.hashcat_mode} -a 0 {HASH_FILE} {WORDLIST}")
+    if sig.john_format is not None:
+        commands.append(f"john --format={sig.john_format} --wordlist={WORDLIST} {HASH_FILE}")
+    return tuple(commands)
+
+
+def tool_notes(sig: Signature, value: str) -> tuple[str, ...]:
+    """Notes on preparing ``value`` for the tools, when it needs changes."""
+    if sig is MYSQL41:
+        if value.startswith("*"):
+            return (
+                "For hashcat, save the hash without the leading '*' (mode 300 "
+                "expects exactly 40 hex characters). John's mysql-sha1 format "
+                "needs the '*', so keep it for john.",
+            )
+        return (
+            "For john, add a leading '*' (its mysql-sha1 format expects '*' "
+            "plus 40 hex characters). Hashcat mode 300 takes the hash as is.",
+        )
+    return ()

hashident/context.py

@@ -0,0 +1,56 @@
+"""Context hints: where the hash was found (the --from option).
+
+A hint boosts signatures that are typical for that environment. It never
+penalises the others; it only changes which plausible format is preferred.
+
+Boost size
+----------
+``CONTEXT_BOOST`` is 50 points on the 0-100 popularity scale, where tiers
+are 20 points apart (see ``hashident.signatures``):
+
+* It overturns a popularity gap of up to two tiers (40 points) but not
+  three (60). NTLM (60) beats MD5 (100) with --from windows (110 vs 100),
+  but a rare format (20) cannot overtake a ubiquitous one (100) on a hint
+  alone (70 vs 100).
+* After such a flip the two are 10 points apart, well within the engine's
+  ``MEDIUM_MAX_GAP`` of 20, so both stay MEDIUM. A hint changes the best
+  guess; it does not turn an ambiguous match into a certain one.
+"""
+
+from __future__ import annotations
+
+from hashident.models import Adjustment, Context, Signature
+
+CONTEXT_BOOST = 50.0
+"""Points added to a signature whose contexts include the hint."""
+
+_DESCRIPTIONS: dict[Context, str] = {
+    Context.WINDOWS: "common in Windows environments",
+    Context.LINUX: "common in Linux environments",
+    Context.WEB: "common in web applications",
+    Context.DB: "common in databases",
+}
+
+
+def parse_context(value: Context | str | None) -> Context | None:
+    """Turn a hint like ``"Windows"`` into a ``Context``; None means no hint.
+
+    Raises ValueError, listing the valid options, for an unknown string.
+    """
+    if value is None or isinstance(value, Context):
+        return value
+    if not isinstance(value, str):
+        raise TypeError(f"context must be a Context, str or None, not {type(value).__name__}")
+    try:
+        return Context(value.strip().casefold())
+    except ValueError:
+        valid = ", ".join(c.value for c in Context)
+        raise ValueError(f"Invalid context {value!r}; choose one of: {valid}") from None
+
+
+def context_adjustments(sig: Signature, context: Context | None) -> list[Adjustment]:
+    """The boost for ``sig`` under ``context``, if it fits; otherwise none."""
+    if context is None or context not in sig.contexts:
+        return []
+    reason = f"{_DESCRIPTIONS[context]} (--from {context.value})"
+    return [Adjustment(delta=CONTEXT_BOOST, reason=reason)]

hashident/engine.py

@@ -0,0 +1,157 @@
+"""Identification engine: turns a raw string into ranked candidates.
+
+Pipeline for one input:
+
+1. Normalize (``Result`` strips surrounding whitespace).
+2. Match the normalized value against every signature.
+3. Score each match: popularity, then any score adjustments
+   (currently the context hint, see ``hashident.context``).
+4. Rank: score descending, then name, so output is deterministic.
+5. Label confidence and add reasons that depend on the other matches.
+
+Confidence thresholds
+---------------------
+* Exactly one signature matches: HIGH. The shape alone pins down the format.
+* Several match: the input is genuinely ambiguous, so nothing is HIGH,
+  with or without a context hint.
+  A candidate within ``MEDIUM_MAX_GAP`` points of the top score is MEDIUM
+  (the top candidate always is); anything further behind is LOW.
+
+``MEDIUM_MAX_GAP`` is 20 because popularity tiers are 20 points apart
+(see ``hashident.signatures``): a candidate within one tier of the leader
+is still a serious contender, while one two or more tiers behind is a
+long shot. MD5 (100) vs NTLM (60) gives MD5 MEDIUM and NTLM LOW.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+from hashident.context import context_adjustments, parse_context
+from hashident.models import Adjustment, Candidate, Confidence, Context, Result, Signature
+from hashident.signatures import SIGNATURES
+
+MEDIUM_MAX_GAP = 20.0
+"""Max points behind the top score for a candidate to still be MEDIUM."""
+
+_HEX_RE = re.compile(r"[0-9a-fA-F]+")
+
+# Modular crypt format identifiers whose first parameter is a cost factor.
+_MCF_COST_IDS = frozenset({"2a", "2b", "2y"})
+
+
+@dataclass(frozen=True)
+class _Scored:
+    signature: Signature
+    score: float
+    reasons: tuple[str, ...]
+
+
+def identify(
+    raw: str,
+    context: Context | str | None = None,
+    *,
+    signatures: Sequence[Signature] = SIGNATURES,
+) -> Result:
+    """Identify which hash formats ``raw`` could be, best guess first.
+
+    ``context`` is an optional hint about where the hash came from, as a
+    ``Context`` or its name (e.g. ``"windows"``, case-insensitive). An
+    unknown name raises ValueError listing the valid options.
+
+    Returns a ``Result`` with no candidates for empty input or input that
+    matches no known format; it does not raise for those cases.
+    """
+    hint = parse_context(context)
+    result = Result(raw=raw, context=hint)
+    value = result.normalized
+    if not value:
+        return result
+
+    matched = [sig for sig in signatures if sig.matches(value)]
+    scored = [_score(sig, value, hint) for sig in matched]
+    scored.sort(key=lambda s: (-s.score, s.signature.name))
+
+    candidates = tuple(_to_candidate(s, scored, value) for s in scored)
+    return Result(raw=raw, candidates=candidates, context=hint)
+
+
+def _score(sig: Signature, value: str, context: Context | None) -> _Scored:
+    """Score one matching signature: popularity plus every adjustment."""
+    score = float(sig.popularity)
+    reasons = [_shape_reason(sig, value)]
+    for adj in _adjustments(sig, value, context):
+        score += adj.delta
+        reasons.append(adj.reason)
+    return _Scored(signature=sig, score=score, reasons=tuple(reasons))
+
+
+def _adjustments(sig: Signature, value: str, context: Context | None) -> list[Adjustment]:
+    """Score adjustments for one candidate.
+
+    This is the single place where scores are changed beyond popularity.
+    """
+    return context_adjustments(sig, context)
+
+
+def _to_candidate(item: _Scored, ranked: Sequence[_Scored], value: str) -> Candidate:
+    """Label confidence and add reasons that depend on the other matches."""
+    top = ranked[0]
+    others = [s.signature.name for s in ranked if s is not item]
+    reasons = list(item.reasons)
+
+    if not others:
+        confidence = Confidence.HIGH
+        reasons.append("no other known format has this shape")
+    else:
+        gap = top.score - item.score
+        confidence = Confidence.MEDIUM if gap <= MEDIUM_MAX_GAP else Confidence.LOW
+        reasons.append(_ranking_reason(item, top, ranked))
+        why = "same length and charset" if _HEX_RE.fullmatch(value) else "same format"
+        reasons.append(f"also matches {', '.join(others)}: {why}")
+
+    return Candidate(
+        signature=item.signature,
+        score=item.score,
+        confidence=confidence,
+        reasons=tuple(reasons),
+    )
+
+
+def _ranking_reason(item: _Scored, top: _Scored, ranked: Sequence[_Scored]) -> str:
+    """Explain this candidate's place in the final ranking.
+
+    Popularity comparisons are always true statements about popularity;
+    when an adjustment reversed the popularity order, the reason says so.
+    """
+    pop = item.signature.popularity
+    if item is top:
+        more_popular = [s.signature.name for s in ranked if s.signature.popularity > pop]
+        if not more_popular:
+            return "most common format with this shape"
+        return f"ranked above {', '.join(more_popular)} by the context hint"
+
+    leader, top_pop = top.signature.name, top.signature.popularity
+    if pop < top_pop:
+        return f"less common than {leader} with this shape"
+    if pop == top_pop and item.score == top.score:
+        return f"as common as {leader} with this shape"
+    comparison = f"more common than {leader}" if pop > top_pop else f"as common as {leader}"
+    return f"{comparison} with this shape, but {leader} matches the context hint"
+
+
+def _shape_reason(sig: Signature, value: str) -> str:
+    """Describe what about the input's shape matched this signature."""
+    if _HEX_RE.fullmatch(value):
+        return f"{len(value)} hexadecimal characters"
+    if value.startswith("*") and _HEX_RE.fullmatch(value[1:]):
+        return f"'*' followed by {len(value) - 1} hexadecimal characters"
+    if value.startswith("$") and value.count("$") >= 2:
+        parts = value.split("$")
+        reason = f"starts with {sig.name} prefix ${parts[1]}$"
+        if parts[1] in _MCF_COST_IDS and len(parts) > 2 and parts[2].isdigit():
+            reason += f" (cost {int(parts[2])})"
+        return reason
+    return f"matches the {sig.name} format ({len(value)} characters)"

hashident/inputs.py

@@ -0,0 +1,154 @@
+"""Collecting the values to identify: arguments, a file, or standard input.
+
+Order of the values, and where they come from:
+
+1. Positional arguments, in the order given, exactly as typed.
+2. Then the lines of ``-f PATH`` (``-f -`` reads standard input).
+3. With neither, standard input is read if it is piped or redirected. On a
+   terminal there is nothing to read, and ``gather`` returns None.
+
+Lines (but not arguments) are cleaned up: surrounding whitespace and the
+``\\r`` of Windows line endings are stripped, and blank lines and lines
+starting with ``#`` are skipped.
+
+Bytes are decoded as UTF-8 unless they start with a UTF-16 byte order mark,
+which is what ``>`` redirection writes in Windows PowerShell 5.1. A UTF-8
+BOM is dropped. Invalid bytes become U+FFFD instead of raising.
+
+Input is decoded in chunks and yielded one line at a time, so memory use
+depends on the longest line, not on the size of the input.
+"""
+
+from __future__ import annotations
+
+import codecs
+import sys
+from collections.abc import Iterator, Sequence
+from dataclasses import dataclass
+from typing import BinaryIO, TextIO
+
+STDIN = "-"
+ARGUMENT_SOURCE = "argument"
+STDIN_SOURCE = "<stdin>"
+
+_COMMENT = "#"
+_CHUNK_SIZE = 64 * 1024
+_TEXT_BOM = ""
+# (byte order mark, encoding of what follows it)
+_BOMS = (
+    (codecs.BOM_UTF8, "utf-8"),
+    (codecs.BOM_UTF16_LE, "utf-16-le"),
+    (codecs.BOM_UTF16_BE, "utf-16-be"),
+)
+_LONGEST_BOM = max(len(bom) for bom, _ in _BOMS)
+
+
+class InputError(Exception):
+    """Input could not be read. The message is short and meant for the user."""
+
+
+@dataclass(frozen=True)
+class InputItem:
+    """One value to identify, and where it came from."""
+
+    value: str
+    source: str
+    """``ARGUMENT_SOURCE``, ``STDIN_SOURCE``, or the path given to ``-f``."""
+    position: int
+    """1-based argument index, or 1-based line number within ``source``."""
+
+
+def gather(values: Sequence[str], file: str | None = None) -> Iterator[InputItem] | None:
+    """The values to identify, in order, or None if no input was given at all.
+
+    ``file`` is opened here rather than on first iteration, so a missing or
+    unreadable file raises ``InputError`` before anything has been yielded.
+    Errors while reading raise ``InputError`` during iteration.
+    """
+    if file is None and not values:
+        if not _is_piped(sys.stdin):
+            return None
+        file = STDIN
+    if file is None:
+        return _items(values, "", iter(()))
+    source, lines = _open_lines(file)
+    return _items(values, source, lines)
+
+
+def _items(values: Sequence[str], source: str, lines: Iterator[str]) -> Iterator[InputItem]:
+    for index, value in enumerate(values, start=1):
+        yield InputItem(value, ARGUMENT_SOURCE, index)
+    try:
+        for number, line in enumerate(lines, start=1):
+            value = line.strip()
+            if value and not value.startswith(_COMMENT):
+                yield InputItem(value, source, number)
+    except OSError as exc:
+        raise InputError(_cannot_read(source, exc)) from exc
+
+
+def _is_piped(stream: TextIO | None) -> bool:
+    """Whether ``stream`` exists and is not a terminal (a pipe or a file)."""
+    if stream is None:
+        return False
+    try:
+        return not stream.isatty()
+    except (AttributeError, ValueError):  # no isatty, or stream closed
+        return False
+
+
+def _open_lines(file: str) -> tuple[str, Iterator[str]]:
+    """``(source name, raw lines)`` for a path, or for stdin if it is ``-``."""
+    if file == STDIN:
+        stream = sys.stdin
+        if stream is None or stream.closed:
+            raise InputError("cannot read standard input: it is not available")
+        binary = getattr(stream, "buffer", None)
+        # A replacement stdin such as io.StringIO has text only, no bytes.
+        lines = _text_lines(stream) if binary is None else _decoded_lines(binary)
+        return STDIN_SOURCE, lines
+    try:
+        handle = open(file, "rb")
+    except OSError as exc:
+        raise InputError(_cannot_read(file, exc)) from exc
+    return file, _file_lines(handle)
+
+
+def _file_lines(handle: BinaryIO) -> Iterator[str]:
+    with handle:
+        yield from _decoded_lines(handle)
+
+
+def _text_lines(stream: TextIO) -> Iterator[str]:
+    for number, line in enumerate(stream):
+        yield line.removeprefix(_TEXT_BOM) if number == 0 else line
+
+
+def _decoded_lines(binary: BinaryIO) -> Iterator[str]:
+    """Decode ``binary`` chunk by chunk and yield its lines, without the ``\\n``."""
+    chunk = binary.read(_LONGEST_BOM)
+    encoding = "utf-8"
+    for bom, bom_encoding in _BOMS:
+        if chunk.startswith(bom):
+            encoding, chunk = bom_encoding, chunk[len(bom) :]
+            break
+    decoder = codecs.getincrementaldecoder(encoding)(errors="replace")
+
+    pending = ""
+    while True:
+        pending += decoder.decode(chunk)
+        *complete, pending = pending.split("\n")
+        yield from complete
+        # read1 returns what is available, so piped lines are not held back
+        # until a whole chunk has arrived.
+        chunk = binary.read1(_CHUNK_SIZE)
+        if not chunk:
+            break
+    pending += decoder.decode(b"", final=True)
+    if pending:
+        yield pending
+
+
+def _cannot_read(source: str, exc: OSError) -> str:
+    name = "standard input" if source == STDIN_SOURCE else source
+    return f"cannot read {name}: {exc.strerror or exc}"