picblocks 2.1.0__py3-none-any.whl

picblocks/blockhasher.py

@@ -0,0 +1,274 @@
+import hashlib
+import json
+import logging
+import os
+import re
+import struct
+import sys
+
+from smda.common.SmdaReport import SmdaFunction
+from smda.Disassembler import Disassembler
+from smda.intel.IntelInstructionEscaper import IntelInstructionEscaper
+
+# Only do basicConfig if no handlers have been configured
+if not logging.root.handlers:
+    logging.basicConfig(level=logging.INFO, format="%(asctime)-15s %(message)s")
+LOG = logging.getLogger(__name__)
+
+# Memory dumps named like Malpedia's dump/dump7_0x<base> files, never a path
+# segment such as /data/dumps/malware.exe.
+_DUMP_FILENAME_RE = re.compile(r"(?:^|[^A-Za-z0-9])dump(?:7)?_0x[0-9a-fA-F]{4,16}", re.I)
+_HEX_BASE_RE = re.compile(r"0x(?P<base_addr>[0-9a-fA-F]{1,16})", re.I)
+_ARCH_RE = re.compile(r"(?P<bitness>(x86_64|x86-64|x86_32|x86|x64|x32|amd64|i386|i686|win32|win64|32bit|64bit))", re.I)
+
+
+class BlockHasher:
+    def parseBitnessFromFilename(self, filepath):
+        # try to infer bitness from filename, in case we process a mapped image / memory dump
+        # an explicit architecture tag is the only strong signal a file name carries, so it wins
+        name = os.path.basename(filepath)
+        # a base address such as 0x64000000 contains the literal "x64", so the address tokens
+        # have to go before the architecture tag is looked for, or every dump mapped at 0x64......
+        # reads as 64bit and every one at 0x32...... as 32bit
+        without_addresses = _HEX_BASE_RE.sub("", name)
+        architecture_match = _ARCH_RE.search(without_addresses)
+        if architecture_match:
+            tag = architecture_match.group("bitness").lower()
+            parsed_bitness = 64 if "64" in tag else 32
+            LOG.info("Parsed bitness from file name: %d", parsed_bitness)
+            return parsed_bitness
+        # a base address needing more than 8 hex digits cannot be 32bit, so it settles the question.
+        # the reverse does not hold - 64bit modules are regularly mapped below 4GB - and guessing
+        # 32 from a short address overrides SMDA's estimate over the actual code with a weaker one.
+        baddr_match = _HEX_BASE_RE.search(name)
+        if baddr_match and len(baddr_match.group("base_addr").lstrip("0")) > 8:
+            LOG.info("Parsed bitness from file name: %d", 64)
+            return 64
+        LOG.warning("No bitness recognized from file name, leaving detection to SMDA.")
+        return None
+
+    def parseBaseAddrFromFilename(self, filepath):
+        # try to infer base addr from filename, in case we process a mapped image / memory dump
+        name = os.path.basename(filepath)
+        baddr_match = _HEX_BASE_RE.search(name)
+        if baddr_match:
+            parsed_base_addr = int(baddr_match.group("base_addr"), 16)
+            LOG.info("Parsed base address from file name: 0x%08x %d", parsed_base_addr, parsed_base_addr)
+            return parsed_base_addr
+        LOG.warning("No base address recognized, using 0.")
+        return 0
+
+    def readFileContent(self, file_path):
+        file_content = b""
+        with open(file_path, "rb") as fin:
+            file_content = fin.read()
+        return file_content
+
+    def _isMappedDumpFilename(self, filename):
+        name = os.path.basename(filename)
+        return bool(_DUMP_FILENAME_RE.search(name))
+
+    def _isUsableSmdaReport(self, smda_report):
+        if smda_report is None:
+            return False
+        if getattr(smda_report, "status", None) == "error":
+            return False
+        if getattr(smda_report, "xcfg", None) is None:
+            return False
+        return True
+
+    def _logSmdaReport(self, smda_report, source):
+        if self._isUsableSmdaReport(smda_report):
+            LOG.info(smda_report)
+            return
+        LOG.warning(
+            "SMDA analysis failed for %s: %s",
+            source,
+            getattr(smda_report, "message", "error"),
+        )
+
+    def _getInstructionEscaper(self, block):
+        """Prefer the architecture-specific SMDA escaper; fall back to Intel on older SMDA."""
+        smda_function = getattr(block, "smda_function", None)
+        if smda_function is not None:
+            escaper = getattr(smda_function, "_escaper", None)
+            if escaper is not None:
+                return escaper
+            report = getattr(smda_function, "smda_report", None)
+            if report is not None:
+                getter = getattr(report, "getInstructionEscaper", None)
+                if callable(getter):
+                    resolved = getter()
+                    if resolved is not None:
+                        return resolved
+            getter = getattr(type(smda_function), "getInstructionEscaper", None)
+            if callable(getter):
+                architecture = getattr(smda_function, "architecture", None)
+                if architecture is None and report is not None:
+                    architecture = getattr(report, "architecture", None)
+                try:
+                    resolved = getter(architecture)
+                except TypeError:
+                    resolved = getter()
+                if resolved is not None:
+                    return resolved
+        return IntelInstructionEscaper
+
+    def processBuffer(self, buffer, filename, bitness=None, baseaddress=None):
+        LOG.info(f"now analyzing {filename}")
+        DISASSEMBLER = Disassembler()
+        name = os.path.basename(filename)
+        # baseaddress=0 is a valid mapped base and must not be treated as "unset"
+        if self._isMappedDumpFilename(name) or "_0x" in name or baseaddress is not None:
+            BASE_ADDR = baseaddress if baseaddress is not None else self.parseBaseAddrFromFilename(filename)
+            BITNESS = bitness if bitness is not None else self.parseBitnessFromFilename(filename)
+            SMDA_REPORT = DISASSEMBLER.disassembleBuffer(buffer, BASE_ADDR, BITNESS)
+        else:
+            SMDA_REPORT = DISASSEMBLER.disassembleUnmappedBuffer(buffer)
+        SMDA_REPORT.filename = os.path.basename(filename)
+        self._logSmdaReport(SMDA_REPORT, filename)
+        blockhash_report = self.extractBlockhashes(SMDA_REPORT)
+        LOG.info("hashes extracted.")
+        return blockhash_report
+
+    def processFile(self, filepath):
+        LOG.info(f"now analyzing {filepath}")
+        INPUT_FILENAME = os.path.basename(filepath)
+        DISASSEMBLER = Disassembler()
+        if self._isMappedDumpFilename(filepath):
+            BUFFER = self.readFileContent(filepath)
+            BASE_ADDR = self.parseBaseAddrFromFilename(INPUT_FILENAME)
+            BITNESS = self.parseBitnessFromFilename(INPUT_FILENAME)
+            SMDA_REPORT = DISASSEMBLER.disassembleBuffer(BUFFER, BASE_ADDR, BITNESS)
+        else:
+            SMDA_REPORT = DISASSEMBLER.disassembleFile(filepath)
+        SMDA_REPORT.filename = os.path.basename(INPUT_FILENAME)
+        self._logSmdaReport(SMDA_REPORT, filepath)
+        blockhash_report = self.extractBlockhashes(SMDA_REPORT)
+        LOG.info("hashes extracted.")
+        return blockhash_report
+
+    def processSmda(self, smda_report):
+        blockhash_report = self.extractBlockhashes(smda_report)
+        return blockhash_report
+
+    def calculateBlockhash(self, block, lower_addr, upper_addr, hash_size=4):
+        escaper = self._getInstructionEscaper(block)
+        escaped_binary_seq = []
+        for instruction in block.getInstructions():
+            escaped = instruction.getEscapedBinary(
+                escaper,
+                escape_intraprocedural_jumps=True,
+                lower_addr=lower_addr,
+                upper_addr=upper_addr,
+            )
+            if escaped:
+                escaped_binary_seq.append(escaped)
+        as_bytes = "".join(escaped_binary_seq).encode("ascii")
+        digest = hashlib.sha256(as_bytes).digest()
+        if hash_size == 8:
+            return struct.unpack("<Q", digest[:8])[0]
+        return struct.unpack("<I", digest[:4])[0]
+
+    def getBlockhashesForFunction(
+        self, smda_function: "SmdaFunction", image_lower: int, image_upper: int, min_block_size=4, hash_size=4
+    ):
+        blockhashes: dict = {}
+        for block in smda_function.getBlocks():
+            block_len = getattr(block, "length", 0) or 0
+            if block_len >= min_block_size:
+                block_size = sum(len(ins.bytes) // 2 for ins in block.getInstructions() if ins.bytes)
+                block_hash = self.calculateBlockhash(
+                    block, lower_addr=image_lower, upper_addr=image_upper, hash_size=hash_size
+                )
+                offset_tuple = {
+                    "offset": block.offset,
+                    "length": block.length,
+                    "size": block_size,
+                }
+                if block_hash not in blockhashes:
+                    blockhashes[block_hash] = {
+                        "hash": block_hash,
+                        "count": 1,
+                        "offset_tuples": [offset_tuple],
+                        "size": block_size,
+                    }
+                else:
+                    blockhashes[block_hash]["offset_tuples"].append(offset_tuple)
+                    blockhashes[block_hash]["count"] += 1
+        return list(blockhashes.values())
+
+    def extractBlockhashes(self, smda_report, min_block_size=4):
+        family = getattr(smda_report, "family", None)
+        if family is None:
+            family = ""
+        output = {
+            "family": family,
+            "version": smda_report.version,
+            "bitness": smda_report.bitness,
+            "sha256": smda_report.sha256,
+            "filename": smda_report.filename,
+            "filesize": smda_report.binary_size,
+            "is_library": smda_report.is_library,
+            "min_block_size": min_block_size,
+            "num_hashes": 0,
+            "num_functions": 0,
+            "num_functions_hashed": 0,
+            "num_blocks": 0,
+            "num_all_blocks": 0,
+            "block_bytes": 0,
+            "blockhashes": {},
+        }
+        if not self._isUsableSmdaReport(smda_report):
+            return output
+        blockhashes = {}
+        image_lower = smda_report.base_addr or 0
+        image_upper = image_lower + (smda_report.binary_size or 0)
+        function_id = 0
+        num_all_blocks = 0
+        num_blocks = 0
+        num_functions = 0
+        num_functions_hashed = 0
+        for function in smda_report.getFunctions():
+            num_functions += 1
+            function_hashed = False
+            for block in function.getBlocks():
+                num_all_blocks += 1
+                if block.length >= min_block_size:
+                    num_blocks += 1
+                    function_hashed = True
+                    block_size = sum([len(ins.bytes) // 2 for ins in block.getInstructions()])
+                    block_hash = self.calculateBlockhash(block, lower_addr=image_lower, upper_addr=image_upper)
+                    if block_hash not in blockhashes:
+                        blockhashes[block_hash] = {}
+                    if block_size not in blockhashes[block_hash]:
+                        blockhashes[block_hash][block_size] = set()
+                    blockhashes[block_hash][block_size].add(function_id)
+                    output["block_bytes"] += block_size
+            if function_hashed:
+                num_functions_hashed += 1
+            function_id += 1
+        num_hashes = 0
+        for blockhash, by_size in blockhashes.items():
+            for size, offsets in by_size.items():
+                num_hashes += 1
+                by_size[size] = sorted(list(offsets))
+        output["num_functions"] = num_functions
+        output["num_functions_hashed"] = num_functions_hashed
+        output["num_blocks"] = num_blocks
+        output["num_all_blocks"] = num_all_blocks
+        output["num_hashes"] = num_hashes
+        output["blockhashes"] = blockhashes
+        return output
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print(f"usage: {sys.argv[0]} <target_binary_path>")
+        sys.exit(1)
+    if os.path.isfile(sys.argv[1]):
+        INPUT_FILENAME = os.path.basename(sys.argv[1])
+        hasher = BlockHasher()
+        blockhash_report = hasher.processFile(sys.argv[1])
+        with open(INPUT_FILENAME + ".blocks", "w") as fout:
+            json.dump(blockhash_report, fout, indent=1, sort_keys=True)

picblocks/blockhashmatcher.py

@@ -0,0 +1,284 @@
+import datetime
+import json
+import logging
+import math
+import os
+import sys
+from collections import Counter, defaultdict
+
+try:
+    # optionally use tqdm to render progress (should not be a package requirement)
+    import tqdm
+except Exception:
+    tqdm = None
+
+from .blockhasher import BlockHasher
+
+# Only do basicConfig if no handlers have been configured
+if not logging.root.handlers:
+    logging.basicConfig(level=logging.INFO, format="%(asctime)-15s %(message)s")
+LOG = logging.getLogger(__name__)
+
+
+def _utc_timestamp():
+    return datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def _percent(part, total):
+    if not total:
+        return 0.0
+    return 100.0 * part / total
+
+
+class BlockHashMatcher:
+    def __init__(self):
+        self.db_timestamp = _utc_timestamp()
+        self.blockhashes = {}
+        self.family_to_id = {}
+        self.family_id_to_family = {}
+        self.sample_id_to_sample = {}
+
+    def load(self, filepath):
+        """load a single blockhash report"""
+        with open(filepath) as fin:
+            blockhash_report = json.load(fin)
+            family = blockhash_report.get("family")
+            if family is None:
+                family = ""
+            if family not in self.family_to_id:
+                family_id = len(self.family_to_id)
+                self.family_to_id[family] = family_id
+                self.family_id_to_family[family_id] = family
+            family_id = self.family_to_id[family]
+            sample_id = len(self.sample_id_to_sample)
+            filename = blockhash_report.get("filename") or os.path.basename(filepath)
+            self.sample_id_to_sample[sample_id] = filename
+            for blockhash, data in (blockhash_report.get("blockhashes") or {}).items():
+                int_hash = int(blockhash)
+                if int_hash not in self.blockhashes:
+                    self.blockhashes[int_hash] = {}
+                for size, fids in data.items():
+                    int_size = int(size)
+                    if int_size not in self.blockhashes[int_hash]:
+                        self.blockhashes[int_hash][int_size] = []
+                    for fid in fids:
+                        is_library = False if "is_library" not in blockhash_report else blockhash_report["is_library"]
+                        self.blockhashes[int_hash][int_size].append((family_id, sample_id, fid, is_library))
+
+    def loadDb(self, filepath):
+        """load a previously processed database of blockhashes"""
+        with open(filepath) as fin:
+            blockhash_db = json.load(fin)
+            self.db_timestamp = blockhash_db["timestamp"]
+            self.family_to_id = blockhash_db["family_to_id"]
+            self.family_id_to_family = {int(k): v for k, v in blockhash_db["family_id_to_family"].items()}
+            self.sample_id_to_sample = {int(k): v for k, v in blockhash_db["sample_id_to_sample"].items()}
+            self.blockhashes = {
+                int(k): {int(ki): vi for ki, vi in v.items()} for k, v in blockhash_db["blockhashes"].items()
+            }
+
+    def saveDb(self, filepath):
+        """save the current database of blockhashes"""
+        parent_dir = os.path.dirname(filepath)
+        if parent_dir:
+            os.makedirs(parent_dir, exist_ok=True)
+        with open(filepath, "w") as fout:
+            json_db = {
+                "timestamp": _utc_timestamp(),
+                "family_to_id": self.family_to_id,
+                "family_id_to_family": self.family_id_to_family,
+                "sample_id_to_sample": self.sample_id_to_sample,
+                "blockhashes": self.blockhashes,
+            }
+            json.dump(json_db, fout)
+
+    def getDbStats(self):
+        """return statistics for currently loaded DB"""
+        family_ids = set()
+        library_ids = set()
+        function_ids = set()
+        num_hashes = 0
+        num_hash_and_sizes = 0
+        num_bytes = 0
+        num_bytes_unique = 0
+        hash_size_counts = Counter()
+        for block_hash, sizes in self.blockhashes.items():
+            num_hashes += 1
+            hash_size_counts[len(sizes)] += 1
+            for size, entries in sizes.items():
+                num_hash_and_sizes += 1
+                num_bytes_unique += size
+                for entry in entries:
+                    family_id, sample_id, fid, is_library = entry
+                    function_ids.add(f"{sample_id}.{fid}")
+                    num_bytes += size
+                    if is_library:
+                        library_ids.add(family_id)
+                    else:
+                        family_ids.add(family_id)
+        return {
+            "num_families": len(family_ids),
+            "num_libraries": len(library_ids),
+            "num_files": len(self.sample_id_to_sample),
+            "num_functions": len(function_ids),
+            "num_hashes": num_hashes,
+            "num_hash_and_sizes": num_hash_and_sizes,
+            "num_bytes": num_bytes,
+            "num_bytes_unique": num_bytes_unique,
+            "hash_size_counts": dict(hash_size_counts),
+        }
+
+    def match(self, blockhash_report):
+        """match a blockhash report against the database"""
+        block_bytes = blockhash_report.get("block_bytes") or 0
+        match_report = {
+            "num_families": len(self.family_to_id),
+            "num_samples": len(self.sample_id_to_sample),
+            "num_blockhashes": len(self.blockhashes),
+            "bitness": blockhash_report.get("bitness"),
+            "sha256": blockhash_report.get("sha256"),
+            "input_filename": blockhash_report.get("filename"),
+            "input_block_bytes": block_bytes,
+            "input_block_hashes": len(blockhash_report.get("blockhashes") or {}),
+            "unmatched_score": 0,
+            "unmatched_hashes": 0,
+            "unmatched_blocks": 0,
+            "family_matches": [],
+        }
+        LOG.debug(
+            f"Using {len(self.family_to_id)} families, {len(self.sample_id_to_sample)} samples with {len(self.blockhashes)} hashes for matching."
+        )
+        sample_matches = defaultdict(int)
+        # bytes
+        family_bytes = defaultdict(int)
+        non_library_bytes = defaultdict(int)
+        adj_family_bytes = defaultdict(float)
+        unique_family_bytes = defaultdict(int)
+        # blocks
+        family_blocks = defaultdict(int)
+        non_library_blocks = defaultdict(int)
+        adj_family_blocks = defaultdict(float)
+        unique_family_blocks = defaultdict(int)
+        unmatched_score = 0
+        unmatched_blocks = 0
+        unmatched_hashes = 0
+        for blockhash, data in (blockhash_report.get("blockhashes") or {}).items():
+            int_hash = int(blockhash)
+            for size, fids in data.items():
+                int_size = int(size)
+                n_instances = len(fids)
+                db_entries = None
+                if int_hash in self.blockhashes and int_size in self.blockhashes[int_hash]:
+                    db_entries = self.blockhashes[int_hash][int_size]
+                if not db_entries:
+                    unmatched_hashes += 1
+                    unmatched_score += int_size * n_instances
+                    unmatched_blocks += n_instances
+                    continue
+                families = set(entry[0] for entry in db_entries)
+                has_library = any(entry[3] for entry in db_entries)
+                family_adjustment_value = 1 if len(families) < 3 else 1 + int(math.log(len(families), 2))
+                # once per query function id, matching how reports store fids
+                for _fid in fids:
+                    credited_families = set()
+                    credited_samples = set()
+                    for entry in db_entries:
+                        family_id, sample_id, fid, is_library = entry
+                        if family_id not in credited_families:
+                            credited_families.add(family_id)
+                            family_bytes[family_id] += int_size
+                            family_blocks[family_id] += 1
+                            if not has_library:
+                                non_library_bytes[family_id] += int_size
+                                non_library_blocks[family_id] += 1
+                                adj_family_bytes[family_id] += int_size / family_adjustment_value
+                                adj_family_blocks[family_id] += 1 / family_adjustment_value
+                                if len(families) == 1:
+                                    unique_family_bytes[family_id] += int_size
+                                    unique_family_blocks[family_id] += 1
+                            else:
+                                # TODO we could collect the function names of functions we potentially recognize here.
+                                pass
+                        if sample_id not in credited_samples:
+                            credited_samples.add(sample_id)
+                            sample_matches[sample_id] += int_size
+        match_report["unmatched_score"] = unmatched_score
+        match_report["unmatched_blocks"] = unmatched_blocks
+        match_report["unmatched_hashes"] = unmatched_hashes
+        LOG.debug(
+            f"Input: {blockhash_report.get('filename')} "
+            f"({blockhash_report.get('family')}/{blockhash_report.get('version')}) - {block_bytes:,d} bytes."
+        )
+        LOG.debug(f"Unmatched blocks: {unmatched_blocks:,d}, {unmatched_score:,d} bytes.")
+        LOG.debug("Family matches: ")
+        index = 1
+        LOG.debug("*" * 93)
+        LOG.debug(
+            f"{'#':>2}: {'id':>5} | {'family':>30} | {'bytescore':>9} | {'%':>6} | {'nolib%':>6} | {'adj%':>6} | {'uniq%':>6}"
+        )
+        for family_id, direct_bytes in sorted(family_bytes.items(), key=lambda x: x[1], reverse=True):
+            nonlib_bytes = non_library_bytes[family_id]
+            adj_bytes = adj_family_bytes[family_id]
+            unique_bytes = unique_family_bytes[family_id]
+            family_result = {
+                "index": index,
+                "family": self.family_id_to_family[family_id],
+                "direct_bytes": direct_bytes,
+                "direct_blocks": family_blocks[family_id],
+                "direct_perc": _percent(direct_bytes, block_bytes),
+                "nonlib_bytes": int(nonlib_bytes),
+                "nonlib_blocks": non_library_blocks[family_id],
+                "nonlib_perc": _percent(nonlib_bytes, block_bytes),
+                "freq_bytes": int(adj_bytes),
+                "freq_blocks": adj_family_blocks[family_id],
+                "freq_perc": _percent(adj_bytes, block_bytes),
+                "uniq_bytes": int(unique_bytes),
+                "uniq_blocks": unique_family_blocks[family_id],
+                "uniq_perc": _percent(unique_bytes, block_bytes),
+            }
+            match_report["family_matches"].append(family_result)
+            if index < 20 or unique_bytes > 0:
+                LOG.debug(
+                    f"{index:>5,d}: {family_id:>5,d} | {self.family_id_to_family[family_id]:>30} | "
+                    f"{direct_bytes:>9,d} | {_percent(direct_bytes, block_bytes):>6.2f} | "
+                    f"{_percent(nonlib_bytes, block_bytes):>6.2f} | {_percent(adj_bytes, block_bytes):>6.2f} | "
+                    f"{_percent(unique_bytes, block_bytes):>6.2f}"
+                )
+            index += 1
+        LOG.debug("*" * 93)
+        return match_report
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print(f"usage: {sys.argv[0]} <block_files_path> <optional:target_binary_path>")
+        sys.exit(1)
+    blocks_path = sys.argv[1]
+    target = sys.argv[2] if len(sys.argv) > 2 else None
+    hasher = BlockHasher()
+    matcher = BlockHashMatcher()
+    if target is not None and os.path.isfile(target):
+        if os.path.exists("db/picblocksdb.json"):
+            print("Loading cached DB: db/picblocksdb.json")
+            matcher.loadDb("db/picblocksdb.json")
+        else:
+            print("No cached DB found, aggregating blockhash reports...")
+            dir_iter = tqdm.tqdm(os.listdir(blocks_path)) if tqdm is not None else os.listdir(blocks_path)
+            for filename in dir_iter:
+                if filename.endswith(".blocks"):
+                    matcher.load(blocks_path + os.sep + filename)
+            print("saving DB...")
+            matcher.saveDb("db/picblocksdb.json")
+        blockhash_report = hasher.processFile(target)
+        print(
+            f"#> hashed input file: {blockhash_report['num_hashes']} hashes covering {blockhash_report['block_bytes']} bytes."
+        )
+        matcher.match(blockhash_report)
+    else:
+        print("Aggregating blockhash reports to create a new DB...")
+        dir_iter = tqdm.tqdm(os.listdir(blocks_path)) if tqdm is not None else os.listdir(blocks_path)
+        for filename in dir_iter:
+            if filename.endswith(".blocks"):
+                matcher.load(blocks_path + os.sep + filename)
+        print("saving DB...")
+        matcher.saveDb("db/picblocksdb.json")

picblocks-2.1.0.dist-info/METADATA

@@ -0,0 +1,125 @@
+Metadata-Version: 2.4
+Name: picblocks
+Version: 2.1.0
+Summary: A library for code similarity estimation using PIC hashing over basic blocks.
+Author-email: Daniel Plohmann <daniel.plohmann@mailbox.org>
+License-Expression: BSD-2-Clause
+Project-URL: Homepage, https://github.com/danielplohmann/picblocks
+Classifier: Development Status :: 4 - Beta
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Security
+Classifier: Topic :: Software Development :: Disassemblers
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: smda>=4.2.13
+Provides-Extra: web
+Requires-Dist: flask; extra == "web"
+Requires-Dist: werkzeug; extra == "web"
+Requires-Dist: waitress; extra == "web"
+Requires-Dist: pymongo; extra == "web"
+Requires-Dist: tqdm; extra == "web"
+Provides-Extra: dev
+Requires-Dist: build; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: requests; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Requires-Dist: ty; extra == "dev"
+Dynamic: license-file
+
+# PicBlocks
+
+An experimental project using position-independent code hashing over basic blocks for code similarity estimation.
+
+## Usage
+
+Both module files in `./picblocks` and in `./utils` are runnable and contain examples of their usage:
+
+* `$ python -m picblocks.blockhasher <target_binary_path>` - produces a `block-report` for a single binary.
+* `$ python -m picblocks.blockhashmatcher <block_reports_path>` - creates a new `./db/picblocksdb.json` from the `block-reports` located in `<block_reports_path>`
+* `$ python -m picblocks.blockhashmatcher <block_reports_path> <target_binary_path>` - matches a binary against data stored in `./db/picblocksdb.json` if it exists, or otherwise creates `./db/picblocksdb.json` from the `block-reports` located in `<block_reports_path>`
+* `$ python -m utils.import_picblocksdb_to_mongo` assumes some mongodb configurations (please check inside the file to adapt to yours) it merely takes the json generated DB into a most easy to manage (and query)  mongodb. 
+* `$ python -m utils.make_stats` it assumes a mongodb connection (please check inside the file to adapt to yours), the generated json db into `db/picblocksdb.json` (you can change it directly in the relative varible) and the generated blocks report into `./block-reports/` folder. It builds up some statistics about detections and DB composition. The results would be available in a dedicated (and very simple) stats web ui. Without MongoDB it writes `db/stats.json` instead. 
+
+## Creating a Database
+
+The script `hash_malpedia.py` is an example of how to process a collection of binaries into `./block-reports`, which will then be aggreated into a `./db/picblocksdb.json`.
+
+## Database Evaulation
+
+In oder to quantify and to measure the quality of your detection rate you should check some basic informations about tests run against your db. 
+The simple (and preliminary) script named `make_stats.py` would build up some initial stats for you about detection rates. 
+It assumes to have a mongodb connection, the generated json db into `db/picblocksdb.json` (you can change it directly in the relative varible) and the generated blocks reports into `./block-reports/` folder (you can change it directly on the specified variable). 
+Once you run it, it takes every single block report and check it against the json database.
+It build some stats and saves all the matching results into db. 
+A dedicated web page (and a relative API) is built to show the detection rates and some more interesting statistics on your database.
+
+## Running as a Service
+
+If a `./db/picblocksdb.json` exists, you can run
+
+`$ python app.py` 
+
+to spawn a local demo server (`http://127.0.0.1:9001`) to query against.
+
+### Screenshots
+
+Just few screenshots about the initial stage of web user interface 
+The submit form. Once you have a given database (`./db/picblocksdb.json`) you can check matching from samples by submitting your samples from 
+this form.
+
+<p align="center">
+  <img src="static/img/1.png">
+</p>
+
+If the submitted sample gets some matches against the given database you should see a block similarity matrix (still under development for a better visualization)
+
+<p align="center">
+  <img src="static/img/2.png">
+</p>
+
+Finally the matching database statistics generated by the script into `utils/make_stats.py` which takes all the generated block_reports (`block-reports/`) and check them against the generated databases (`./db/picblocksdb.json`) in order to estimate the detection rate on a given database.
+
+<p align="center">
+  <img src="static/img/3.png">
+</p>
+
+## Contributors
+
+* [Daniel Plohmann](https://github.com/danielplohmann)
+* [Marco Ramilli](https://github.com/marcoramilli)
+* [Daniele Bellavista](https://github.com/dbellavista)
+* [Rony](https://github.com/r0ny123)
+
+## Scores changed in v2.1.0
+
+Up to v2.0.1 the matcher credited a family **once per distinct block hash**, while
+`block_bytes` - the denominator every percentage is divided by - counts **every block
+occurrence**. A block shared by ten functions therefore contributed its size ten times to
+the denominator and once to the numerator, so all four percentages read too low. They now
+credit once per matching function, and reported percentages go up accordingly. Measured
+over a database built from the Malpedia block reports, matching a sample that is itself in
+the database moved from 87.6-99.0% to 98.2-99.4%.
+
+Match reports produced by older versions are not comparable with new ones.
+
+A known residual gap keeps that self-match just under 100%: a report stores the function
+ids a block was seen in as a set, so the same hash occurring twice inside one function
+counts twice in `block_bytes` but once when scoring. Over the Malpedia block reports this
+is ~2.4% of `block_bytes`. Closing it would change the block report format.
+
+## Version History
+* 2026-09-13: v2.1.0 - architecture-aware PIC escaping (SMDA >= 4.2.13, AArch64/CIL/Dalvik as well as Intel), corrected matcher scoring (see above), dump/baseaddress routing, report/UI fixes, and a test suite
+* 2023-11-24: v2.0.1 - SMDA pinned to 1.12.7 before our bigger fix for PIC calculation
+* 2022-09-08: v2.0.0 - (BREAKING CHANGE) now intraprocedural control flow transfers are wildcarded by default, which should improve matching
+* 2022-08-04: v1.1.3 - extended format for blockhash representation of functions
+* 2021-10-01: v1.1.1 - added script to check detection rates and relative web interface page
+* 2021-09-28: v1.1.0 - added simple web user interface and a db connection
+* 2021-09-12: v1.0.6 - added submission form fields for bitness and base address to force overrides for those values.
+* 2021-08-24: v1.0.5 - improved parsing of bitness from submission filenames.
+* 2021-08-20: v1.0.4 - Tweaked result visualization, now showing all unique matches beyond the first 20.

picblocks-2.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+picblocks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+picblocks/blockhasher.py,sha256=lg6VOVbllLtCdthHWUnKMbZBJR48QGLDcm356kWdRzg,12293
+picblocks/blockhashmatcher.py,sha256=RPIhr6Xx7akYqWwhik5aRh8CwLBj0lEWOuR5icBExXM,13000
+picblocks-2.1.0.dist-info/licenses/LICENSE,sha256=qVIB0Uzno8yiGRqtV-K1ngYBIcF8DqYaL4Lnnx_6pDU,1287
+picblocks-2.1.0.dist-info/METADATA,sha256=Vi30mv_T0voL99IAZwwb7LYDuQC5fQlbJUwOeyC8IeQ,7079
+picblocks-2.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+picblocks-2.1.0.dist-info/top_level.txt,sha256=BRfdREZiXbo_Xfc4YhgZyfnMcRVymnF_1tIVE5KdrKA,10
+picblocks-2.1.0.dist-info/RECORD,,

picblocks-2.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

picblocks-2.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,9 @@
+Copyright (c) 2021, Daniel Plohmann
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

picblocks-2.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+picblocks