binstat 1.0.0__py3-none-any.whl

binstat/VERSION.txt

@@ -0,0 +1 @@
+1.0.0

binstat/__info__.py

@@ -0,0 +1,19 @@
+# -*- coding: UTF-8 -*-
+"""BinStat package information.
+
+"""
+import os
+from datetime import datetime
+
+__y = str(datetime.now().year)
+__s = "2026"
+
+__author__    = "Alexandre D'Hondt"
+__copyright__ = "© {} Alexandre D'Hondt".format([__y, __s + "-" + __y][__y != __s])
+__license__   = "GPLv3 (https://www.gnu.org/licenses/gpl-3.0.fr.html)"
+__reference__ = "https://doi.org/10.5769/J201101003"
+__source__    = "https://github.com/dhondta/binstat"
+
+with open(os.path.join(os.path.dirname(__file__), "VERSION.txt")) as f:
+    __version__ = f.read().strip()
+

binstat/__init__.py

@@ -0,0 +1,110 @@
+# -*- coding: UTF-8 -*-
+import numpy as np
+import os
+
+from .features import *
+from .features import __all__ as _features
+from .model import *
+from .model import __all__ as _model
+
+
+__all__ = ["BinStat", "DEFAULT_MODEL", "THRESHOLD"] + _features + _model
+
+
+DEFAULT_MODEL = os.path.join(os.path.dirname(__file__), "model.json")
+THRESHOLD = .5
+
+
+def _files(*paths):
+    """ Yield the files from the input paths, walking folders recursively (sorted, for reproducibility). """
+    for path in paths:
+        path = str(path)
+        if os.path.isdir(path):
+            for root, dirs, files in os.walk(path):
+                dirs.sort()
+                for f in sorted(files):
+                    yield os.path.join(root, f)
+        else:
+            yield path
+
+
+class BinStat:
+    def __init__(self, model=None, threshold=THRESHOLD, logger=None, **kwargs):
+        """ Configure the detector with a model (Model instance or path to a JSON model ; None for the default one)
+             and the minimum ratio of packed blocks for an executable to be packed. """
+        if not 0. <= threshold <= 1.:
+            raise ValueError("threshold must be between 0 and 1")
+        self.model = model if isinstance(model, Model) else Model.load(model or DEFAULT_MODEL)
+        self.threshold = threshold
+        self.logger = logger
+
+    def blocks(self, executable):
+        """ Classify each block of the input executable (1 = packed, 0 = not packed). """
+        return self.model.predict(file_features(str(executable), **self.model.features_kwargs))
+
+    def detect(self, executable):
+        """ Analyze the input executable file: it is packed when the ratio of its packed blocks reaches the
+             threshold. """
+        r = self.ratio(executable)
+        return r if r == "?" else bool(r >= self.threshold)
+
+    def ratio(self, executable):
+        """ Compute the ratio of packed blocks of the input executable. """
+        try:
+            blocks = self.blocks(executable)
+        except OSError as e:
+            self._log(f"{executable}: {e.strerror or e}", OSError)
+            return "?"
+        if len(blocks) == 0:
+            self._log(f"{executable} has no block to analyze", ValueError)
+            return "?"
+        r = float(blocks.mean())
+        if self.logger:
+            self.logger.debug(f"{blocks.sum()}/{len(blocks)} blocks recognized as packed ({r:.2%}, threshold: "
+                              f"{self.threshold:.2%})")
+        return r
+
+    def _log(self, msg, exception, level="error"):
+        """ Helper for either logging a message if a logger is defined or raising an exception. """
+        if self.logger:
+            getattr(self.logger, level)(msg)
+        else:
+            raise exception(msg)
+
+    @classmethod
+    def train(cls, packed, not_packed, ngram=1, block_size=BLOCK_SIZE, minkowski_order=MINKOWSKI_ORDER, boost=30,
+              balanced=False, max_blocks=None, max_leaf_nodes=None, min_samples_leaf=1, seed=None, threshold=THRESHOLD,
+              logger=None, **kwargs):
+        """ Train a detector on packed and not packed executables, labelling all the blocks of an executable with its
+             own label.
+
+        :param packed:     list of packed executables or folders holding them
+        :param not_packed: list of not packed executables or folders holding them
+        :param balanced:   weight the blocks so that both classes weigh the same (packed blocks usually outnumber
+                            the others)
+        :param max_blocks: maximum number of blocks taken (at random) from each executable (None for all)
+        :return:           BinStat instance (its model can be saved with .model.save(path))
+        """
+        fkw = {'ngram': ngram, 'block_size': block_size, 'minkowski_order': minkowski_order}
+        rng, X, y = np.random.default_rng(seed), [], []
+        for label, paths in [(1, packed), (0, not_packed)]:
+            for path in _files(*paths):
+                try:
+                    F = file_features(path, **fkw)
+                except OSError as e:
+                    if logger:
+                        logger.warning(f"{path}: {e.strerror or e}")
+                    continue
+                if max_blocks is not None and len(F) > max_blocks:
+                    F = F[np.sort(rng.choice(len(F), max_blocks, replace=False))]
+                X.append(F)
+                y.append(np.full(len(F), label))
+                if logger:
+                    logger.debug(f"{path}: {len(F)} blocks ({['not packed', 'packed'][label]})")
+        X, y = np.vstack(X or [np.zeros((0, len(FEATURES)))]), np.concatenate(y or [np.zeros(0, dtype="int64")])
+        if logger:
+            logger.info(f"training on {len(y)} blocks ({int(y.sum())} packed, {int(len(y) - y.sum())} not packed)")
+        w = (len(y) / (2 * np.bincount(y, minlength=2)))[y] if balanced and len(set(y)) == 2 else None
+        m = train(X, y, boost=boost, max_leaf_nodes=max_leaf_nodes, min_samples_leaf=min_samples_leaf, seed=seed,
+                  sample_weight=w, balanced=balanced, **fkw)
+        return cls(m, threshold, logger)

binstat/__main__.py

@@ -0,0 +1,134 @@
+# -*- coding: UTF-8 -*-
+import hashlib
+import logging
+from argparse import ArgumentParser, RawTextHelpFormatter
+from ast import literal_eval
+from os.path import exists
+from time import perf_counter
+
+from .__info__ import __author__, __copyright__, __license__, __reference__, __source__, __version__
+from .__init__ import BinStat, BLOCK_SIZE, MINKOWSKI_ORDER, NGRAMS, THRESHOLD, file_features
+
+
+def valid_file(path):
+    if not exists(path):
+        raise ValueError("input file does not exist")
+    return path
+
+
+class Positive:
+    def __init__(self, *types, maximum=None, zero=True):
+        self._types, self._max, self._zero = types, maximum, zero
+
+    def __call__(self, string):
+        try:
+            n = literal_eval(string)
+        except (SyntaxError, ValueError):
+            raise ValueError(string)
+        if not isinstance(n, self._types) or n < 0. or not self._zero and n == 0 or \
+           self._max is not None and n > self._max:
+            raise ValueError(string)
+        return self._types[0](n)
+
+    def __repr__(self):
+        return "positive %s" % "|".join(map(lambda x: x.__name__, self._types))
+
+
+def _features_args(parser):
+    parser.add_argument("--block-size", type=Positive(int, zero=False), default=BLOCK_SIZE,
+                        help="size of the blocks, in bytes (default: %d)" % BLOCK_SIZE)
+    parser.add_argument("--minkowski-order", type=Positive(float, int, zero=False), default=MINKOWSKI_ORDER,
+                        help="order (lambda) of Minkowski's distance (default: %d)" % MINKOWSKI_ORDER)
+    parser.add_argument("--ngram", type=int, choices=NGRAMS, default=1, help="n-gram size (default: 1)")
+
+
+def _detect(args):
+    r = BinStat(args.model, args.threshold, args.logger)
+    return r.ratio(args.path) if args.ratio else r.detect(args.path)
+
+
+def _features(args):
+    with open(args.path, "rb") as f:
+        md5 = hashlib.md5(f.read()).hexdigest()
+    kw = {k: getattr(args, k) for k in ["block_size", "minkowski_order", "ngram"]}
+    # same format as the C5.0 data file of the paper: the measures, the label, the block ID and the MD5 of the file
+    return "\n".join(",".join(["%f" % v for v in row] + [args.label, f"BLOCK{i}", md5])
+                     for i, row in enumerate(file_features(args.path, **kw)))
+
+
+def _train(args):
+    kw = {k: getattr(args, k) for k in ["balanced", "block_size", "boost", "max_blocks", "max_leaf_nodes",
+                                        "minkowski_order", "ngram", "seed"]}
+    BinStat.train(args.packed, args.not_packed, logger=args.logger, **kw).model.save(args.output)
+    args.logger.info(f"model saved to {args.output}")
+
+
+def main():
+    """ Tool's main function """
+    descr = "BinStat {}\n\nAuthor   : {}\nCopyright: {}\nLicense  : {}\nReference: {}\nSource   : {}\n" \
+            "\nThis tool computes statistics on the n-grams of each block of an executable and classifies the blocks" \
+            " with boosted decision trees to determine if the executable is packed or not.\n\n"
+    descr = descr.format(__version__, __author__, __copyright__, __license__, __reference__, __source__)
+    examples = "usage examples:\n- " + "\n- ".join([
+        "binstat detect program.exe",
+        "binstat detect program.exe --threshold 0.6 --ratio",
+        "binstat features program.exe --label yes",
+        "binstat train -p packed/ -n not-packed/ -o model.json --max-blocks 100",
+    ])
+    parser = ArgumentParser(description=descr, epilog=examples, formatter_class=RawTextHelpFormatter)
+    # extra arguments are accepted after each command
+    common = ArgumentParser(add_help=False)
+    extra = common.add_argument_group("extra arguments")
+    extra.add_argument("-b", "--benchmark", action="store_true",
+                       help="enable benchmarking, output in seconds (default: False)")
+    extra.add_argument("-v", "--verbose", action="store_true", help="display debug information (default: False)")
+    sparsers = parser.add_subparsers(dest="command", required=True, help="command to be executed")
+    cmd = lambda name, help: sparsers.add_parser(name, help=help, description=help, parents=[common])
+    detect = cmd("detect", "detect whether an executable is packed")
+    detect.add_argument("path", type=valid_file, help="path to the executable file")
+    detect.add_argument("-m", "--model", help="path to a JSON model (default: the model shipped with BinStat)")
+    detect.add_argument("-r", "--ratio", action="store_true",
+                        help="output the ratio of packed blocks instead of the decision")
+    detect.add_argument("-t", "--threshold", type=Positive(float, int, maximum=1), default=THRESHOLD,
+                        help="minimum ratio of packed blocks for the executable to be packed (default: %.1f)" %
+                             THRESHOLD)
+    features = cmd("features", "compute the measures of each block (in C5.0 data format)")
+    features.add_argument("path", type=valid_file, help="path to the executable file")
+    _features_args(features)
+    features.add_argument("--label", choices=["yes", "no", "?"], default="?",
+                          help="label of the blocks, 'yes' meaning packed (default: ?, i.e. unknown)")
+    train = cmd("train", "train a model on packed and not packed executables")
+    train.add_argument("-n", "--not-packed", action="extend", nargs="+", required=True,
+                       help="not packed executables or folders holding them")
+    train.add_argument("-o", "--output", required=True, help="path to the output JSON model")
+    train.add_argument("-p", "--packed", action="extend", nargs="+", required=True,
+                       help="packed executables or folders holding them")
+    _features_args(train)
+    train.add_argument("--balanced", action="store_true",
+                       help="weight the blocks so that packed and not packed blocks weigh the same (default: False)")
+    train.add_argument("--boost", type=Positive(int, zero=False), default=30,
+                       help="number of boosting trials (default: 30)")
+    train.add_argument("--max-blocks", type=Positive(int, zero=False),
+                       help="maximum number of blocks taken at random from each executable (default: all)")
+    train.add_argument("--max-leaf-nodes", type=Positive(int, zero=False),
+                       help="maximum number of leaves of each tree (default: unlimited)")
+    train.add_argument("--seed", type=int, help="random seed, for reproducible models")
+    args = parser.parse_args()
+    logging.basicConfig()
+    args.logger = logging.getLogger("binstat")
+    args.logger.setLevel([logging.INFO, logging.DEBUG][args.verbose])
+    code = 0
+    # execute the tool
+    if args.benchmark:
+        t1 = perf_counter()
+    try:
+        r = {'detect': _detect, 'features': _features, 'train': _train}[args.command](args)
+        dt = str(perf_counter() - t1) if args.benchmark else ""
+        if r is not None:
+            print(str(r))
+        if dt != "":
+            print(dt)
+    except Exception as e:
+        args.logger.exception(e)
+        code = 1
+    return code

binstat/features.py

@@ -0,0 +1,151 @@
+# -*- coding: UTF-8 -*-
+"""Statistical module of BinStat: the thirteen block measures of Park et al.
+
+Each block of the input is described by the frequencies X_f of its n-gram values f (0 <= f < M, with M = 256**n). Except
+ Simpson's index and the entropy, the measures compare the frequencies of consecutive n-gram values, X_f and X_{f+1},
+ for 0 <= f < M - 1. As a block holds at most 1024 distinct n-grams, only the non-zero frequencies are computed, so that
+ bigrams to four-grams cost the same as unigrams.
+
+The divergences (Kullback-Leibler, Jensen-Shannon and Itakura-Saito) use decimal logarithms: with these, the measures
+ match the sample line given in the paper for a UPX-packed block, while the entropy is in bits as stated.
+"""
+import numpy as np
+
+
+__all__ = ["BLOCK_SIZE", "FEATURES", "MINKOWSKI_ORDER", "NGRAMS", "block_features", "file_features"]
+
+
+BLOCK_SIZE = 1024
+FEATURES = ["simpson", "canberra", "minkowski", "manhattan", "chebyshev", "bray_curtis", "angular_separation",
+            "correlation", "entropy", "kullback_leibler", "jensen_shannon", "itakura_saito", "total_variation"]
+MINKOWSKI_ORDER = 3
+NGRAMS = (1, 2, 3, 4)
+
+
+def _div(a, b):
+    """ Element-wise a / b, with 0 where b is 0. """
+    out = np.zeros_like(a, dtype="float64")
+    np.divide(a, b, out=out, where=b != 0)
+    return out
+
+
+def _histograms(data, ngram, block_size):
+    """ Count the n-grams of each block.
+
+    N-grams do not cross block boundaries ; the last block may be shorter than block_size, and is ignored when it is
+     shorter than the n-gram.
+
+    :return: (number of n-grams per block, sorted keys block * M + value of the n-grams, their counts)
+    """
+    a = np.frombuffer(data, dtype="uint8")
+    n_blocks = (len(a) - ngram) // block_size + 1 if len(a) >= ngram else 0
+    if n_blocks == 0:
+        return np.zeros(0, dtype="int64"), np.zeros(0, dtype="int64"), np.zeros(0, dtype="int64")
+    # value of the n-gram starting at each position (big-endian, so that f and f+1 differ in their last byte)
+    start = np.arange(len(a) - ngram + 1, dtype="int64")
+    values = np.zeros(len(start), dtype="int64")
+    for j in range(ngram):
+        values = (values << 8) | a[j:len(a) - ngram + 1 + j]
+    # keep the n-grams lying entirely inside their block
+    keep = start % block_size <= block_size - ngram
+    block = start[keep] // block_size
+    counts_per_block = np.bincount(block, minlength=n_blocks)
+    if ngram == 1:
+        # fast path: a dense histogram of 256 values per block
+        dense = np.bincount(block * 256 + values[keep], minlength=n_blocks * 256)
+        keys = np.flatnonzero(dense)
+        return counts_per_block, keys, dense[keys]
+    keys, counts = np.unique(block * 256 ** ngram + values[keep], return_counts=True)
+    return counts_per_block, keys, counts
+
+
+def _measures(n_per_block, keys, counts, ngram, minkowski_order=MINKOWSKI_ORDER):
+    """ Compute the thirteen measures of each block from its sparse n-gram histogram. """
+    m, nb = 256 ** ngram, len(n_per_block)
+    bsum = lambda block, w: np.bincount(block, weights=w, minlength=nb)
+    N = n_per_block.astype("float64")
+    blk, val = keys // m, keys % m
+    x = counts / N[blk]
+    # pairs (X_f, X_{f+1}) where at least one of both is non-zero ; all the other pairs are (0, 0)
+    f = np.union1d(keys[val <= m - 2], keys[val >= 1] - 1)
+    pb = f // m
+    i = np.searchsorted(keys, f)
+    i[i == len(keys)] = 0
+    xa = np.where(keys[i] == f, x[i], 0.)
+    j = np.searchsorted(keys, f + 1)
+    j[j == len(keys)] = 0
+    xb = np.where(keys[j] == f + 1, x[j], 0.)
+    d = np.abs(xa - xb)
+    both = (xa > 0) & (xb > 0)
+    r = np.where(both, xa / np.where(both, xb, 1.), 1.)
+    mix = (xa + xb) / 2
+    # correlation coefficient, with the mean frequency 1/M over the M values ; each (0, 0) pair contributes 1/M^2
+    #  to each sum
+    mean, zero_pairs = 1. / m, (m - 1) - bsum(pb, None)
+    cov = bsum(pb, (xa - mean) * (xb - mean)) + zero_pairs * mean ** 2
+    va = bsum(pb, (xa - mean) ** 2) + zero_pairs * mean ** 2
+    vb = bsum(pb, (xb - mean) ** 2) + zero_pairs * mean ** 2
+    chebyshev = np.zeros(nb)
+    np.maximum.at(chebyshev, pb, d)
+    manhattan = bsum(pb, d)
+    return np.column_stack([
+        # 1) Simpson's index
+        _div(bsum(blk, counts * (counts - 1.)), N * (N - 1)),
+        # 2) Canberra's distance (pairs (0, 0) are skipped)
+        bsum(pb, _div(d, xa + xb)),
+        # 3) Minkowski's distance of order lambda
+        bsum(pb, d ** minkowski_order) ** (1. / minkowski_order),
+        # 4) Manhattan's distance
+        manhattan,
+        # 5) Chebyshev's distance
+        chebyshev,
+        # 6) Bray-Curtis' distance
+        _div(manhattan, bsum(pb, xa + xb)),
+        # 7) Angular separation
+        _div(bsum(pb, xa * xb), np.sqrt(bsum(pb, xa ** 2) * bsum(pb, xb ** 2))),
+        # 8) Correlation coefficient
+        _div(cov, np.sqrt(va * vb)),
+        # 9) Entropy (bits)
+        -bsum(blk, x * np.log2(x)),
+        # 10) Kullback-Leibler's divergence (pairs where X_f or X_{f+1} is zero are skipped)
+        bsum(pb, np.where(both, xa * np.log10(r), 0.)),
+        # 11) Jensen-Shannon's divergence
+        bsum(pb, (_xlogy(xa, mix) + _xlogy(xb, mix)) / 2),
+        # 12) Itakura-Saito's divergence (pairs where X_f or X_{f+1} is zero are skipped)
+        bsum(pb, np.where(both, r - np.log10(r) - 1, 0.)),
+        # 13) Total variation
+        manhattan / 2,
+    ])
+
+
+def _xlogy(x, y):
+    """ x * log10(x / y), with 0 where x is 0. """
+    out = np.zeros_like(x)
+    nz = x > 0
+    out[nz] = x[nz] * np.log10(x[nz] / y[nz])
+    return out
+
+
+def block_features(data, ngram=1, block_size=BLOCK_SIZE, minkowski_order=MINKOWSKI_ORDER):
+    """ Compute the thirteen measures of each block of the input bytes.
+
+    :param data:            bytes to analyze
+    :param ngram:           n-gram size, from 1 (unigrams) to 4 (four-grams)
+    :param block_size:      size of the blocks, in bytes
+    :param minkowski_order: order (lambda) of Minkowski's distance
+    :return:                array of shape (number of blocks, 13), with columns in the order of FEATURES
+    """
+    if ngram not in NGRAMS:
+        raise ValueError(f"n-gram size must be one of {', '.join(map(str, NGRAMS))}")
+    if block_size < ngram:
+        raise ValueError("block size must be at least the n-gram size")
+    n_per_block, keys, counts = _histograms(bytes(data), ngram, block_size)
+    if len(n_per_block) == 0:
+        return np.zeros((0, len(FEATURES)))
+    return _measures(n_per_block, keys, counts, ngram, minkowski_order)
+
+
+def file_features(path, **kwargs):
+    """ Compute the thirteen measures of each block of a file (see block_features). """
+    with open(path, "rb") as f:
+        return block_features(f.read(), **kwargs)