pypackerdetect 1.2.0__tar.gz → 1.2.1__tar.gz

{pypackerdetect-1.2.0 → pypackerdetect-1.2.1}/.github/workflows/python-package.yml

@@ -10,7 +10,6 @@ on:
     branches: [ "main" ]
 
 permissions:
-  id-token: write # for OIDC
   contents: read
 
 jobs:
@@ -20,7 +19,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
     steps:
     - uses: actions/checkout@v5
     - name: Set up Python ${{ matrix.python-version }}
@@ -40,31 +39,44 @@ jobs:
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
   deploy:
+    # only publish from pushes to main, never from pull requests
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     environment:
       name: pypi
-      url: ${{ needs.prepare.outputs.pypi_url }}
+      url: https://pypi.org/p/pypackerdetect
+    permissions:
+      # required for PyPI trusted publishing (OIDC)
+      id-token: write
+      contents: read
     runs-on: ubuntu-latest
     needs: build
     steps:
     - uses: actions/checkout@v5
-      with:
-        fetch-depth: 0
-    - name: Check for version change
-      uses: dorny/paths-filter@v4
-      id: filter
-      with:
-        filters: |
-          version:
-            - '**/VERSION.txt'
-    - if: steps.filter.outputs.version == 'true'
+    - name: Check whether the version is already on PyPI
+      # comparing with PyPI (not with the previous commit) also releases a version whose push did not trigger a deploy
+      id: version
+      run: |
+        version=$(tr -d '[:space:]' < src/pypackerdetect/VERSION.txt)
+        status=$(curl -s -o /dev/null -w '%{http_code}' --retry 3 "https://pypi.org/pypi/pypackerdetect/$version/json")
+        case "$status" in
+          200) echo "Version $version is already on PyPI" ; echo "new=false" >> "$GITHUB_OUTPUT" ;;
+          404) echo "Version $version is not on PyPI yet" ; echo "new=true" >> "$GITHUB_OUTPUT" ;;
+          *)   echo "::error::Unexpected HTTP status $status from PyPI" ; exit 1 ;;
+        esac
+    - if: steps.version.outputs.new == 'true'
       name: Cleanup README
       run: |
         sed -ri 's/^(##*)\s*:.*:\s*/\1 /g' README.md
         awk '{if (match($0,"## Supporters")) exit; print}' README.md > README
         mv -f README README.md
-    - if: steps.filter.outputs.version == 'true'
-      name: Build package
+    - if: steps.version.outputs.new == 'true'
+      name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: "3.13"
+    - if: steps.version.outputs.new == 'true'
+      name: Build pypackerdetect package
       run: python3 -m pip install --upgrade build && python3 -m build
-    - if: steps.filter.outputs.version == 'true'
-      name: Upload to PyPi
+    - if: steps.version.outputs.new == 'true'
+      name: Publish to PyPI
       uses: pypa/gh-action-pypi-publish@release/v1

{pypackerdetect-1.2.0 → pypackerdetect-1.2.1}/.gitignore

@@ -70,3 +70,6 @@ target/
 .test
 tmp
 TODO
+
+# PEiD search-tree caches (rebuilt at runtime)
+src/pypackerdetect/db/.*.json

pypackerdetect-1.2.1/.readthedocs.yaml

@@ -0,0 +1,13 @@
+version: 2
+
+build:
+  os: ubuntu-24.04
+  tools:
+    python: "3.13"
+
+mkdocs:
+  configuration: docs/mkdocs.yml
+
+python:
+  install:
+    - requirements: docs/requirements.txt

{pypackerdetect-1.2.0/src/pypackerdetect.egg-info → pypackerdetect-1.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pypackerdetect
-Version: 1.2.0
+Version: 1.2.1
 Summary: Re-implementation and packaging of PyPackerDetect
 Author-email: Alexandre D'Hondt <alexandre.dhondt@gmail.com>
 License:                     GNU GENERAL PUBLIC LICENSE
@@ -691,14 +691,14 @@ Classifier: License :: OSI Approved :: GNU General Public License v3 or later (G
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Security
-Requires-Python: <4,>=3.9
+Requires-Python: <4,>=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pefile
-Requires-Dist: peid>=2.3.1
+Requires-Dist: peid>=2.4.0
 Dynamic: license-file
 
-<p align="center"><img src="https://github.com/packing-box/PyPackerDetect/raw/main/docs/logo.png"></p>
+<p align="center"><img src="https://github.com/packing-box/PyPackerDetect/raw/main/docs/pages/img/logo.png"></p>
 <h1 align="center">PyPackerDetect <a href="https://twitter.com/intent/tweet?text=PyPackerDetect%20-%20Packer%20detector%20using%20multiple%20heuristics%20and%20PEiD's%20signatures%20database.%0D%0Ahttps%3a%2f%2fgithub%2ecom%2fpacking-box%2fPyPackerDetect%0D%0A&hashtags=python,pe,peid,packer,packingdetection"><img src="https://img.shields.io/badge/Tweet--lightgrey?logo=twitter&style=social" alt="Tweet" height="20"/></a></h1>
 <h3 align="center">Detect packers on PE files using heuristics and signatures.</h3>
 
@@ -723,8 +723,9 @@ $ pypackerdetect --help
 usage examples:
 - pypackerdetect program.exe
 - pypackerdetect program.exe -b
-- pypackerdetect program.exe --low-imports --unknown-sections
+- pypackerdetect program.exe --low-imports --unknown-sections   # disable these two checks
 - pypackerdetect program.exe --imports-threshold 5 --bad-sections-threshold 5
+- pypackerdetect program.exe --peid-db-path user_db.txt --sections-json sections.json
 ```
 
 ## Detection Mechanisms
@@ -732,7 +733,7 @@ usage examples:
 - PEID signatures
 - Known packer section names
 - Entrypoint in non-standard section
-- Threshhold of non-standard sections reached
+- Threshold of non-standard sections reached
 - Low number of imports
 - Overlapping entrypoint sections
 

{pypackerdetect-1.2.0 → pypackerdetect-1.2.1}/README.md

@@ -1,4 +1,4 @@
-<p align="center"><img src="https://github.com/packing-box/PyPackerDetect/raw/main/docs/logo.png"></p>
+<p align="center"><img src="https://github.com/packing-box/PyPackerDetect/raw/main/docs/pages/img/logo.png"></p>
 <h1 align="center">PyPackerDetect <a href="https://twitter.com/intent/tweet?text=PyPackerDetect%20-%20Packer%20detector%20using%20multiple%20heuristics%20and%20PEiD's%20signatures%20database.%0D%0Ahttps%3a%2f%2fgithub%2ecom%2fpacking-box%2fPyPackerDetect%0D%0A&hashtags=python,pe,peid,packer,packingdetection"><img src="https://img.shields.io/badge/Tweet--lightgrey?logo=twitter&style=social" alt="Tweet" height="20"/></a></h1>
 <h3 align="center">Detect packers on PE files using heuristics and signatures.</h3>
 
@@ -23,8 +23,9 @@ $ pypackerdetect --help
 usage examples:
 - pypackerdetect program.exe
 - pypackerdetect program.exe -b
-- pypackerdetect program.exe --low-imports --unknown-sections
+- pypackerdetect program.exe --low-imports --unknown-sections   # disable these two checks
 - pypackerdetect program.exe --imports-threshold 5 --bad-sections-threshold 5
+- pypackerdetect program.exe --peid-db-path user_db.txt --sections-json sections.json
 ```
 
 ## Detection Mechanisms
@@ -32,7 +33,7 @@ usage examples:
 - PEID signatures
 - Known packer section names
 - Entrypoint in non-standard section
-- Threshhold of non-standard sections reached
+- Threshold of non-standard sections reached
 - Low number of imports
 - Overlapping entrypoint sections
 

pypackerdetect-1.2.1/docs/mkdocs.yml

@@ -0,0 +1,60 @@
+site_author: dhondta
+site_name: PyPackerDetect
+site_description: Detect packers on PE files using heuristics and signatures
+site_url: https://pypackerdetect.readthedocs.io
+repo_url: https://github.com/packing-box/pypackerdetect
+edit_uri: edit/main/docs/pages/
+copyright: Copyright © 2021 Alexandre D'Hondt
+docs_dir: pages
+nav:
+  - Home: index.md
+  - Installation: installation.md
+  - Command-line tool: cli/pypackerdetect.md
+  - Python API: api.md
+  - How it works: internals.md
+  - Contributing: contributing.md
+extra:
+  generator: false
+  social:
+    - icon: fontawesome/solid/paper-plane
+      link: mailto:alexandre.dhondt@gmail.com
+      name: Contact Alex
+    - icon: fontawesome/brands/github
+      link: https://github.com/dhondta
+      name: Alex on GitHub
+    - icon: fontawesome/brands/linkedin
+      link: https://www.linkedin.com/in/alexandre-d-2ab2aa14/
+      name: Alex on LinkedIn
+    - icon: fontawesome/brands/twitter
+      link: https://twitter.com/alex_dhondt
+      name: Alex on Twitter
+theme:
+  name: material
+  logo: img/logo.png
+  favicon: img/icon.png
+  palette:
+    - scheme: default
+      toggle:
+        icon: material/brightness-7
+        name: Switch to dark mode
+    - scheme: slate
+      toggle:
+        icon: material/brightness-4
+        name: Switch to light mode
+  features:
+    - content.code.copy
+    - navigation.sections
+    - navigation.top
+    - toc.follow
+use_directory_urls: false
+markdown_extensions:
+  - admonition
+  - attr_list
+  - def_list
+  - pymdownx.details
+  - pymdownx.highlight
+  - pymdownx.inlinehilite
+  - pymdownx.superfences
+  - tables
+  - toc:
+      permalink: true

pypackerdetect-1.2.1/docs/pages/api.md

@@ -0,0 +1,101 @@
+# Python API
+
+The `pypackerdetect` package exposes one class:
+
+```python
+from pypackerdetect import PyPackerDetect
+```
+
+---
+
+## `PyPackerDetect`
+
+```python
+PyPackerDetect(peid=True, peid_large_db=False, peid_ep_only=True, peid_db_path=None,
+               bad_ep_sections=True, low_imports=True, packer_sections=True,
+               unknown_sections=True, import_threshold=10, unknown_sections_threshold=3,
+               bad_sections_threshold=2, logger=None, packers_json=None,
+               sections_json=None, **kwargs)
+```
+
+Configures a detector. The JSON databases are loaded once, here, so one
+instance can analyze many files.
+
+| Parameter | Type | Description |
+|---|---|---|
+| `peid` | `bool` | Match PEiD signatures. |
+| `peid_large_db` | `bool` | Use the large embedded PEiD database instead of the short one. |
+| `peid_ep_only` | `bool` | Match PEiD signatures at the entry point only. `False` scans the whole file. |
+| `peid_db_path` | `str` or `None` | Path to a custom PEiD database. It takes precedence over `peid_large_db`. |
+| `bad_ep_sections` | `bool` | Check the section(s) holding the entry point. |
+| `low_imports` | `bool` | Check the number of imports. |
+| `packer_sections` | `bool` | Check for known packer section names. |
+| `unknown_sections` | `bool` | Check the number of non-standard and invalid section names. |
+| `import_threshold` | `int` | Report a detection when there are this many imports or fewer. |
+| `unknown_sections_threshold` | `int` | Report a detection from this many non-standard section names. |
+| `bad_sections_threshold` | `int` | Report a detection from this many section names with invalid characters. |
+| `logger` | `logging.Logger` | Optional logger for debug output. |
+| `packers_json` | `str` or `None` | Path to a custom packer section names JSON. `None` uses the embedded one. |
+| `sections_json` | `str` or `None` | Path to a custom known section names JSON. `None` uses the embedded one. |
+| `**kwargs` | | Ignored. It lets you pass `vars(args)` from an argument parser. |
+
+See [Databases](internals.md#databases) for the JSON formats.
+
+### `detect`
+
+```python
+PyPackerDetect.detect(pe)
+```
+
+Analyzes one executable.
+
+| Parameter | Type | Description |
+|---|---|---|
+| `pe` | `str` or `pefile.PE` | Path to the executable, or an already parsed `pefile.PE` object. |
+
+**Returns** a dictionary with two lists of messages:
+
+```python
+{'detections': [...], 'suspicions': [...]}
+```
+
+See [Findings](internals.md#findings) for the possible messages.
+
+**Raises** `pefile.PEFormatError` if the file is not a valid PE file.
+
+!!! note "Passing a `pefile.PE` object"
+    PEiD and `objdump` work on files. If the `PE` object has a `path`
+    attribute, that file is used. Otherwise (for instance with
+    `PE(data=...)`), the data is copied to a temporary file for the duration
+    of the analysis.
+
+### `report`
+
+```python
+PyPackerDetect.report(pe, findings)   # static method
+```
+
+Prints the findings returned by [`detect`](#detect) in the format of the
+original project (see [Output](cli/pypackerdetect.md#output)). `pe` is the
+path or the `pefile.PE` object, and is only used for the report header.
+
+## Example
+
+```python
+import logging
+from pefile import PE
+from pypackerdetect import PyPackerDetect
+
+detector = PyPackerDetect(peid_large_db=True, import_threshold=5,
+                          logger=logging.getLogger("pypackerdetect"))
+
+for path in ["a.exe", "b.exe"]:
+    findings = detector.detect(path)
+    if findings["detections"]:
+        print(f"{path} looks packed")
+    PyPackerDetect.report(path, findings)
+
+# with a PE object parsed from memory
+with open("c.exe", "rb") as f:
+    findings = detector.detect(PE(data=f.read()))
+```

pypackerdetect-1.2.1/docs/pages/cli/pypackerdetect.md

@@ -0,0 +1,100 @@
+# `pypackerdetect`: detect packing
+
+`pypackerdetect` runs all the [detection mechanisms](../internals.md) on a PE
+file and reports the findings, either as **detections** or **suspicions**.
+
+```console
+$ pypackerdetect [path arguments] [optional arguments] [threshold arguments] [-b] [-v] [-h] path
+```
+
+## Arguments
+
+| Argument | Description |
+|---|---|
+| `path` | The PE file to analyze. It must exist. |
+
+### Path arguments
+
+| Option | Default | Description |
+|---|---|---|
+| `--packers-json JSON` | embedded | JSON file mapping known packer section names to packer descriptions (see [Databases](../internals.md#databases)). |
+| `--peid-db-path TXT_DB` | embedded | PEiD signatures database to use instead of the embedded one. It takes precedence over `--peid-large-db`. |
+| `--sections-json JSON` | embedded | JSON file with the lists of known section names. |
+
+### Optional arguments
+
+All checks are **enabled by default**. Each of these flags **turns one off**.
+
+| Option | Turns off |
+|---|---|
+| `--bad-ep-sections` | The checks on the entry point section: irregular section, overlapping sections, no section, null entry point. |
+| `--low-imports` | The check for a low number of imports. |
+| `--packer-sections` | The check for known packer section names. |
+| `--peid` | Detection with PEiD signatures. |
+| `--peid-ep-only` | Entry point-only PEiD matching. PEiD then scans **every offset of the file** with the signatures that are not tied to the entry point, which is much slower. |
+| `--unknown-sections` | The checks on the number of non-standard and invalid section names. |
+
+| Option | Default | Description |
+|---|---|---|
+| `--peid-large-db` | off | Use the large embedded PEiD database (4,287 signatures) instead of the short one (1,832 signatures). |
+
+!!! warning "Flags turn checks off"
+    The option names describe the checks, not the action. For instance,
+    `pypackerdetect program.exe --low-imports` **disables** the low imports
+    check.
+
+### Threshold arguments
+
+| Option | Default | Description |
+|---|---|---|
+| `--bad-sections-threshold N` | 2 | Report a detection when at least `N` section names contain invalid characters. |
+| `--imports-threshold N` | 10 | Report a detection when the executable has `N` imports or fewer. |
+| `--unknown-sections-threshold N` | 3 | Report a detection when at least `N` section names are non-standard. |
+
+`N` must be a non-negative integer.
+
+### Extra arguments
+
+| Option | Description |
+|---|---|
+| `-b`, `--benchmark` | Print the elapsed time, in seconds, after the report. |
+| `-v`, `--verbose` | Show debug information, including each finding as it is found. |
+| `-h`, `--help` | Show the help message and exit. |
+
+## Output
+
+```console
+$ pypackerdetect calc_packed.exe
+Packer report for: calc_packed.exe
+	Detections: 3
+	Suspicions: 0
+	Log:
+		[DETECTION] Entry point 0xc4f80 in irregular section(s): 'UPX1'
+		[DETECTION] Section name 'UPX0' matches known packer: [UPX packer]
+		[DETECTION] Section name 'UPX1' matches known packer: [UPX packer]
+```
+
+The `Log:` section only appears when there is at least one finding. See
+[Findings](../internals.md#findings) for the list of messages.
+
+## Exit code
+
+| Code | Meaning |
+|---|---|
+| `0` | The analysis ran, whatever the findings. |
+| `1` | The analysis failed, for instance because the input is not a valid PE file. |
+| `2` | Invalid arguments. |
+
+!!! note
+    The exit code does not tell whether the file is packed. Parse the report,
+    or use the [Python API](../api.md), for that.
+
+## Examples
+
+```console
+$ pypackerdetect program.exe
+$ pypackerdetect program.exe -b
+$ pypackerdetect program.exe --low-imports --unknown-sections   # disable these two checks
+$ pypackerdetect program.exe --imports-threshold 5 --bad-sections-threshold 5
+$ pypackerdetect program.exe --peid-db-path user_db.txt --sections-json sections.json
+```

pypackerdetect-1.2.1/docs/pages/contributing.md

@@ -0,0 +1,56 @@
+# Contributing
+
+Contributions are welcome, especially **new packer section names** and
+**false positive reports**.
+
+## Reporting issues
+
+Open an issue on [GitHub](https://github.com/packing-box/pypackerdetect/issues)
+and include:
+
+- the PyPackerDetect version (`pypackerdetect --help` prints it);
+- the exact command and its output, ideally with `-v`;
+- if possible, a link to (or the hash of) the sample.
+
+## Adding section names
+
+- **Packer section names** go in `src/pypackerdetect/db/packers.json`, with a
+  short description of the packer. Avoid names that are also standard
+  section names: they are ignored anyway (see [Section names](internals.md#section-names)).
+- **Standard section names** that cause false positives go in the `known` list
+  of `src/pypackerdetect/db/sections.json`.
+
+Test your change on packed and unpacked samples. The
+[packed PE dataset](https://github.com/packing-box/dataset-packed-pe) and
+[Docker Packing Box](https://github.com/packing-box/docker-packing-box) help
+with this. Section names are compared after stripping null bytes and spaces,
+so do not add trailing spaces to the names.
+
+## Development setup
+
+```console
+$ git clone https://github.com/packing-box/pypackerdetect
+$ cd pypackerdetect
+$ pip install -e .
+$ pip install flake8
+$ flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+```
+
+CI (`.github/workflows/python-package.yml`) installs the package on Python
+3.9 to 3.13 and runs this flake8 check.
+
+## Releasing
+
+The version is stored in `src/pypackerdetect/VERSION.txt`. When a push to
+`main` changes that file, CI builds the package and publishes it to PyPI.
+
+## Building this documentation
+
+```console
+$ pip install -r docs/requirements.txt
+$ mkdocs serve -f docs/mkdocs.yml          # live preview at http://127.0.0.1:8000
+$ mkdocs build --strict -f docs/mkdocs.yml # what Read the Docs runs
+```
+
+Read the Docs builds the site from `.readthedocs.yaml` at the root of the
+repository.

pypackerdetect-1.2.1/docs/pages/index.md

@@ -0,0 +1,72 @@
+# PyPackerDetect
+
+<p align="center"><img src="img/logo.png" alt="PyPackerDetect logo" width="200"></p>
+
+**PyPackerDetect** detects whether a Windows PE executable is packed. It
+combines several heuristics on the PE structure with
+[PEiD](https://github.com/packing-box/peid) signatures. It is a complete
+refactoring of [Cylance's PyPackerDetect](https://github.com/cylance/PyPackerDetect)
+into a Python package with a console script.
+
+[![PyPi](https://img.shields.io/pypi/v/pypackerdetect.svg)](https://pypi.python.org/pypi/pypackerdetect/)
+[![Python Versions](https://img.shields.io/pypi/pyversions/pypackerdetect.svg)](https://pypi.python.org/pypi/pypackerdetect/)
+[![Build Status](https://github.com/packing-box/pypackerdetect/actions/workflows/python-package.yml/badge.svg)](https://github.com/packing-box/pypackerdetect/actions/workflows/python-package.yml)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.5732866.svg)](https://doi.org/10.5281/zenodo.5732866)
+[![License](https://img.shields.io/pypi/l/pypackerdetect.svg)](https://pypi.python.org/pypi/pypackerdetect)
+
+## Features
+
+- **Six detection mechanisms** (see [How it works](internals.md)):
+    - PEiD signatures;
+    - known packer section names;
+    - entry point in a non-standard section;
+    - entry point in overlapping sections, or outside every section;
+    - too many non-standard (or invalid) section names;
+    - a low number of imports.
+- **Two levels of findings**: *detections* are strong indicators of packing,
+  *suspicions* are weaker hints.
+- **Configurable**: each check can be turned off, thresholds can be tuned and
+  the section-name and signature databases can be replaced with your own.
+- **A command-line tool**, [`pypackerdetect`](cli/pypackerdetect.md), and a
+  [Python API](api.md).
+
+## Quick start
+
+```console
+$ pip install pypackerdetect
+$ pypackerdetect calc_packed.exe
+Packer report for: calc_packed.exe
+	Detections: 3
+	Suspicions: 0
+	Log:
+		[DETECTION] Entry point 0xc4f80 in irregular section(s): 'UPX1'
+		[DETECTION] Section name 'UPX0' matches known packer: [UPX packer]
+		[DETECTION] Section name 'UPX1' matches known packer: [UPX packer]
+```
+
+From Python:
+
+```python
+from pypackerdetect import PyPackerDetect
+
+findings = PyPackerDetect().detect("program.exe")
+print(findings["detections"], findings["suspicions"])
+```
+
+## Related projects
+
+PyPackerDetect is part of the [packing-box](https://github.com/packing-box) ecosystem:
+
+- [Awesome Executable Packing](https://github.com/packing-box/awesome-executable-packing): a curated list of resources about executable packing.
+- [Bintropy](https://github.com/packing-box/bintropy): estimates how likely a binary is to contain compressed or encrypted bytes.
+- [Dataset of packed ELF files](https://github.com/packing-box/dataset-packed-elf) and [Dataset of packed PE files](https://github.com/packing-box/dataset-packed-pe).
+- [Docker Packing Box](https://github.com/packing-box/docker-packing-box): a Docker image of packers and tools for building datasets of packed executables.
+- [DSFF](https://github.com/packing-box/python-dsff): the DataSet File Format library.
+- [PEiD](https://github.com/packing-box/peid): a Python implementation of the Packed Executable iDentifier, used here for signature matching.
+- [REMINDer](https://github.com/packing-box/reminder): a packing detector based on a simple entropy heuristic.
+
+## License
+
+PyPackerDetect is released under the
+[GNU General Public License v3 or later](https://www.gnu.org/licenses/gpl-3.0.html).
+The original project is © Cylance (author: Nick Cano).

pypackerdetect-1.2.1/docs/pages/installation.md

@@ -0,0 +1,56 @@
+# Installation
+
+## Requirements
+
+- Python **3.9 or later**. CI runs on 3.9 to 3.13.
+- [`pefile`](https://pypi.org/project/pefile/) and
+  [`peid`](https://pypi.org/project/peid/) (2.4.0 or later), which pip
+  installs automatically.
+- *Optional:* `objdump` (from GNU binutils). It is only used to resolve long
+  section names; see [Long section names](internals.md#long-section-names).
+
+## From PyPI
+
+```console
+$ pip install pypackerdetect
+```
+
+This installs the `pypackerdetect` Python package and the `pypackerdetect`
+console script.
+
+## From source
+
+```console
+$ git clone https://github.com/packing-box/pypackerdetect
+$ cd pypackerdetect
+$ pip install .
+```
+
+For development, use an editable install:
+
+```console
+$ pip install -e .
+```
+
+## Checking the installation
+
+```console
+$ pypackerdetect --help
+[...]
+PyPackerDetect 1.2.1
+[...]
+```
+
+```python
+>>> from pypackerdetect.__info__ import __version__
+>>> __version__
+'1.2.1'
+```
+
+## Write access to the database folder
+
+The first time a PEiD database is used, [peid](https://github.com/packing-box/peid)
+writes a JSON cache of its search tree **next to the database file**. For the
+embedded databases, that folder is the installed `pypackerdetect/db/` package
+directory. If the folder is read-only, detection still works without the
+cache, but each run rebuilds the search tree, which is slower.

pypackerdetect-1.2.1/docs/pages/internals.md

@@ -0,0 +1,141 @@
+# How it works
+
+## Overview
+
+```text
+                      ┌── section names ──▶ known/unknown/packer/invalid names
+ program.exe ─pefile─▶├── entry point ────▶ section(s) holding it
+                      ├── import table ───▶ number of imports
+                      └── file ──── peid ─▶ PEiD signatures
+                                              │
+                                              ▼
+                              {'detections': [...], 'suspicions': [...]}
+```
+
+[pefile](https://github.com/erocarrera/pefile) parses the executable, then
+each enabled check adds its findings. A **detection** is a strong indicator
+of packing. A **suspicion** is a weaker hint.
+
+## Section names
+
+Each section name is decoded as ASCII, then stripped of padding (null bytes
+and spaces). A name that is not valid ASCII is an **invalid name**: it
+produces a suspicion, and it is counted for the
+[bad sections threshold](cli/pypackerdetect.md#threshold-arguments).
+
+A valid name is then:
+
+- **unknown** if it is not in the `known` list of `sections.json`;
+- a **packer section** if it is a key of `packers.json` and is not a known
+  name, so that a packer entry cannot clash with a standard name.
+
+### Long section names
+
+Section names are limited to 8 bytes. Some compilers (for instance MinGW with
+debug information) write longer names in the COFF string table and store
+`/N` in the section header, where `N` is the offset of the name in that
+table. For such names, PyPackerDetect runs `objdump -h` on the file and uses
+the real names. If `objdump` is not installed, fails or times out (after 10
+seconds), the `/N` name is kept.
+
+## Entry point section
+
+The entry point is the `AddressOfEntryPoint` field of the optional header. A
+section holds it when:
+
+```text
+VirtualAddress <= entry point < VirtualAddress + VirtualSize
+```
+
+`SizeOfRawData` is used instead of `VirtualSize` when the latter is zero, as
+the Windows loader does.
+
+Then:
+
+1. A null entry point is a suspicion.
+2. If no section holds the entry point, it is a detection.
+3. If several sections hold it, they overlap, which is a detection.
+4. If none of the entry point sections is `acceptable` (`.text`), the
+   section is irregular, unless one of these exceptions applies:
+    - the file has a Delphi BSS section (`delphi-bss`) and the entry point is
+      in a Delphi code section (`delphi`: `.itext`, `CODE`);
+    - the file has no `.text` section at all and the entry point is in an
+      `alternative` section (`.code`, `text`, `.text0`, ...);
+    - the file has a `.text` section and the entry point is in a `driver`
+      section (`INIT`).
+
+## Imports
+
+The imports of all the DLLs in the import table are counted. A count lower
+than or equal to the threshold (10 by default) is a detection, as packers
+usually only import what they need to unpack the payload (often
+`LoadLibrary` and `GetProcAddress`). A file with no import table counts as 0
+imports.
+
+.NET assemblies (importing `mscoree.dll`) are skipped, as they always have
+very few native imports.
+
+## PEiD signatures
+
+The file is passed to
+[`peid.identify_packer`](https://peid.readthedocs.io/en/latest/api.html#identify_packer)
+with the chosen database. By default, only the signatures that apply at the
+entry point are matched, using the short embedded database. Every match is
+reported in one detection.
+
+## Findings
+
+| Level | Message | Check |
+|---|---|---|
+| Suspicion | `Section name with invalid characters` | `unknown_sections` (always collected) |
+| Suspicion | `Null entry point` | `bad_ep_sections` |
+| Detection | `Entry point 0x… does not fall in any section` | `bad_ep_sections` |
+| Detection | `Entry point 0x… falls in overlapping sections: '…'` | `bad_ep_sections` |
+| Detection | `Entry point 0x… in irregular section(s): '…'` | `bad_ep_sections` |
+| Detection | `Too few imports (total: N)` | `low_imports` |
+| Detection | `Detected N non-standard sections: '…'` | `unknown_sections` |
+| Detection | `Detected N sections with invalid names` | `unknown_sections` |
+| Detection | `Section name '…' matches known packer: […]` | `packer_sections` |
+| Detection | `Found PEID signature: …` | `peid` |
+
+## Databases
+
+The embedded databases are in the `pypackerdetect/db/` package folder.
+
+### `packers.json`
+
+Known packer section names, mostly from
+[Hexacorn](https://www.hexacorn.com/blog/2016/12/15/pe-section-names-re-visited/).
+The `packers` key maps each section name to a description:
+
+```json
+{
+    "source": "https://www.hexacorn.com/blog/2016/12/15/pe-section-names-re-visited/",
+    "last_update": "Jul 11 2021",
+    "packers": {
+        "UPX0": "UPX packer",
+        ".MPRESS1": "Mpress Packer"
+    }
+}
+```
+
+### `sections.json`
+
+Lists of standard section names, under the `sections` key:
+
+| Key | Use |
+|---|---|
+| `known` | Standard section names; any other name is non-standard. |
+| `acceptable` | Regular entry point sections. |
+| `alternative` | Entry point sections accepted when there is no `.text` section. |
+| `delphi-bss` | BSS sections that identify a Delphi executable. |
+| `delphi` | Entry point sections accepted for Delphi executables. |
+| `driver` | Entry point sections accepted for drivers. |
+
+A custom file passed with `--sections-json` must contain all of these keys.
+
+### PEiD databases
+
+`sigs_short.txt` (1,832 signatures, the default) and `sigs_long.txt` (4,287
+signatures, `--peid-large-db`) are in the PEiD text format, described in the
+[PEiD documentation](https://peid.readthedocs.io/en/latest/signatures.html).

pypackerdetect-1.2.1/docs/requirements.txt

@@ -0,0 +1,6 @@
+jinja2
+mkdocs>=1.3.0
+mkdocs-bootswatch
+mkdocs-material
+mkdocs-rtd-dropdown
+pymdown-extensions

{pypackerdetect-1.2.0 → pypackerdetect-1.2.1}/pyproject.toml

@@ -19,7 +19,7 @@ authors = [
 description = "Re-implementation and packaging of PyPackerDetect"
 license = {file = "LICENSE"}
 keywords = ["packing", "packed-executable", "packing-detection", "pe"]
-requires-python = ">=3.9,<4"
+requires-python = ">=3.10,<4"
 classifiers = [
   "Development Status :: 5 - Production/Stable",
   "Environment :: Console",
@@ -33,7 +33,7 @@ classifiers = [
 ]
 dependencies = [
   "pefile",
-  "peid>=2.3.1",
+  "peid>=2.4.0",
 ]
 dynamic = ["version"]
 

pypackerdetect-1.2.1/requirements.txt

@@ -0,0 +1,2 @@
+pefile
+peid>=2.4.0

pypackerdetect-1.2.1/src/pypackerdetect/VERSION.txt

@@ -0,0 +1 @@
+1.2.1

{pypackerdetect-1.2.0 → pypackerdetect-1.2.1}/src/pypackerdetect/__info__.py

@@ -13,7 +13,7 @@ __credits__   = "Alexandre D'Hondt"
 __copyright__ = "© {} Cylance".format([__y, __s + "-" + __y][__y != __s])
 __license__   = "GPLv3 (https://www.gnu.org/licenses/gpl-3.0.fr.html)"
 __reference__ = "https://github.com/cylance/PyPackerDetect"
-__source__    = "https://github.com/dhondta/PyPackerDetect"
+__source__    = "https://github.com/packing-box/pypackerdetect"
 
 with open(os.path.join(os.path.dirname(__file__), "VERSION.txt")) as f:
     __version__ = f.read().strip()

{pypackerdetect-1.2.0 → pypackerdetect-1.2.1}/src/pypackerdetect/__init__.py

@@ -13,10 +13,17 @@ __all__ = ["PyPackerDetect"]
 @lru_cache
 def _real_section_names(path, logger=None, timeout=10):
     from subprocess import Popen, PIPE, TimeoutExpired
-    p, names = Popen(["objdump", "-h", path], stdout=PIPE, stderr=PIPE), []
+    try:
+        p, names = Popen(["objdump", "-h", path], stdout=PIPE, stderr=PIPE), []
+    except OSError as e:  # e.g. objdump is not installed
+        if logger:
+            logger.debug(f"objdump: {e}")
+        return
     try:
         out, err = p.communicate(timeout=timeout)
     except TimeoutExpired:
+        p.kill()
+        p.communicate()
         if logger:
             logger.debug(f"objdump: timeout ({timeout} seconds)")
         return
@@ -47,10 +54,12 @@ class PyPackerDetect:
             'packers_json': os.path.abspath(packers_json) if packers_json is not None else _p("packers.json"),
             'sections_json': os.path.abspath(sections_json) if sections_json is not None else _p("sections.json"),
         }
-        logger.debug(f"JSON with common packer section names: {self.__config['packers_json']}")
+        if logger:
+            logger.debug(f"JSON with common packer section names: {self.__config['packers_json']}")
         with open(self.__config['packers_json']) as f:
             self.__config['packers_json'] = json.load(f)['packers']
-        logger.debug(f"JSON with known section names: {self.__config['sections_json']}")
+        if logger:
+            logger.debug(f"JSON with known section names: {self.__config['sections_json']}")
         with open(self.__config['sections_json']) as f:
             self.__config['sections_json'] = json.load(f)['sections']
         self.__thresholds = {
@@ -61,30 +70,44 @@ class PyPackerDetect:
         self.logger = logger
     
     def detect(self, pe):
-        """ Analyze the input PE file for suspicions and/or detections of packers. """
-        r, s = {'detections': [], 'suspicions': []}, .0
-        def __add(i, msg):
-            if self.logger:
-                self.logger.debug("%s:%s" % (['DETECTION', 'SUSPICION'][i], msg))
-            r[['detections', 'suspicions'][i]].append(msg)
+        """ Analyze the input PE file (path or pefile.PE instance) for suspicions and/or detections of packers. """
         if not isinstance(pe, PE):
             path = pe
             pe = PE(path)
             pe.path = path
+        if getattr(pe, "path", None) is None:
+            # a PE instance parsed from a buffer has no path while PEiD and objdump work on files ; use a temporary copy
+            from tempfile import NamedTemporaryFile
+            with NamedTemporaryFile(suffix=".exe", delete=False) as f:
+                f.write(pe.__data__)
+            pe.path = f.name
+            try:
+                return self.detect(pe)
+            finally:
+                pe.path = None
+                os.remove(f.name)
+        r = {'detections': [], 'suspicions': []}
+        def __add(i, msg):
+            if self.logger:
+                self.logger.debug("%s:%s" % (['DETECTION', 'SUSPICION'][i], msg))
+            r[['detections', 'suspicions'][i]].append(msg)
         ep = pe.OPTIONAL_HEADER.AddressOfEntryPoint
         # parse the input PE file
         d = {'all': [], 'bad': [], 'ep': [], 'packer': [], 'unknown': []}
         for i, s in enumerate(pe.sections):
             try:
-                n = s.Name.decode("ascii").strip().rstrip("\0")
+                n = s.Name.decode("ascii").strip().rstrip("\0 ")
                 # special case: for some executables compiled with debug information, some sections may be named with
                 #                the format "/[N]" where N is the offset in the string table ; in this case, we compute
                 #                the real section names and map them to the shortened ones to compare the relevant names
                 #                with the database of known section names
-                if re.match(r"^\/\d+$", n) and (rsn := _real_section_names(pe.path, logger=self.logger)):
+                if re.match(r"^\/\d+$", n) and (rsn := _real_section_names(pe.path, logger=self.logger)) and \
+                   i < len(rsn):
                     n = rsn[i]
                 d['all'].append(n)
-                if ep >= s.VirtualAddress and ep <= (s.VirtualAddress + s.Misc_VirtualSize):
+                # the loader uses SizeOfRawData when VirtualSize is null ; the upper bound is excluded as it is the start
+                #  of the next section for contiguous sections
+                if s.VirtualAddress <= ep < s.VirtualAddress + (s.Misc_VirtualSize or s.SizeOfRawData):
                     d['ep'].append(n)
                 if n not in self.__config['sections_json']['known']:
                     d['unknown'].append(n)
@@ -93,7 +116,7 @@ class PyPackerDetect:
                 if p is not None and n not in self.__config['sections_json']['known']:
                     d['packer'].append(n)
             except UnicodeDecodeError:
-                d['bad'].append(s.Name.decode('latin1').strip().rstrip("\0"))
+                d['bad'].append(s.Name.decode('latin1').strip().rstrip("\0 "))
                 __add(1, "Section name with invalid characters")
         # bad entry point sections
         __intersect = lambda l1, l2: len(set(l1).intersection(l2)) > 0
@@ -103,7 +126,7 @@ class PyPackerDetect:
                 __add(1, "Null entry point")
             l_ep = len(d['ep'])
             if l_ep == 0:
-                __add(0, "Entry point 0x%x falls in overlapping sections: '%s'" % (ep, ep_sec_list))
+                __add(0, "Entry point 0x%x does not fall in any section" % ep)
             else:
                 if l_ep > 1:
                     __add(0, "Entry point 0x%x falls in overlapping sections: '%s'" % (ep, ep_sec_list))
@@ -126,11 +149,11 @@ class PyPackerDetect:
             c = 0
             try:
                 for lib in pe.DIRECTORY_ENTRY_IMPORT:
-                    if lib.dll.decode("ascii").strip().rstrip("\0").lower() == "mscoree.dll":
+                    if lib.dll.decode("latin-1").strip().rstrip("\0").lower() == "mscoree.dll":
                         c = -1  # .NET assembly, counting imports is misleading as they will have a low number
                         break
                     c += len(lib.imports)
-            except AttributeError as e:
+            except AttributeError:  # no import directory
                 pass
             if (0 <= c <= self.__thresholds['imports']):
                 __add(0, "Too few imports (total: %d)" % c)

{pypackerdetect-1.2.0 → pypackerdetect-1.2.1}/src/pypackerdetect/__main__.py

@@ -1,8 +1,9 @@
 # -*- coding: UTF-8 -*-
 import logging
-from argparse import ArgumentParser, RawTextHelpFormatter
+from argparse import ArgumentParser, ArgumentTypeError, RawTextHelpFormatter
 from ast import literal_eval
-from os.path import exists
+from os.path import isfile
+from pefile import PEFormatError
 from time import perf_counter
 
 from .__info__ import __author__, __credits__, __copyright__, __license__, __reference__, __source__, __version__
@@ -10,8 +11,8 @@ from .__init__ import PyPackerDetect
 
 
 def valid_file(path):
-    if not exists(path):
-        raise ValueError("input file does not exist")
+    if not isfile(path):
+        raise ArgumentTypeError(f"input file does not exist: {path}")
     return path
 
 
@@ -19,10 +20,10 @@ class PositiveInt:
     def __call__(self, string):
         try:
             n = literal_eval(string)
-        except ValueError:
-            raise ValueError(string)
+        except (SyntaxError, ValueError):
+            raise ArgumentTypeError(f"invalid positive int value: '{string}'")
         if not isinstance(n, int) or n < 0:
-            raise ValueError(string)
+            raise ArgumentTypeError(f"invalid positive int value: '{string}'")
         return int(n)
     
     def __repr__(self):
@@ -38,7 +39,7 @@ def main():
     examples = "usage examples:\n- " + "\n- ".join([
         "pypackerdetect program.exe",
         "pypackerdetect program.exe -b",
-        "pypackerdetect program.exe --low-imports --unknown-sections",
+        "pypackerdetect program.exe --low-imports --unknown-sections   # disable these two checks",
         "pypackerdetect program.exe --imports-threshold 5 --bad-sections-threshold 5",
         "pypackerdetect program.exe --peid-db-path user_db.txt --sections-json sections.json",
     ])
@@ -53,20 +54,24 @@ def main():
                                                               "(default: package-embedded JSON)")
     opt = parser.add_argument_group("optional arguments")
     opt.add_argument("--bad-ep-sections", action="store_false",
-                     help="check for bad entry point sections (default: True)")
-    opt.add_argument("--low-imports", action="store_false", help="check for the number of imports (default: True)")
-    opt.add_argument("--packer-sections", action="store_false", help="check for packer sections (default: True)")
-    opt.add_argument("--peid", action="store_false", help="detect with PEiD (default: True)")
-    opt.add_argument("--peid-ep-only", action="store_false", help="check only entry point signatures (default: True)")
+                     help="disable the check for bad entry point sections (enabled by default)")
+    opt.add_argument("--low-imports", action="store_false",
+                     help="disable the check for a low number of imports (enabled by default)")
+    opt.add_argument("--packer-sections", action="store_false",
+                     help="disable the check for known packer section names (enabled by default)")
+    opt.add_argument("--peid", action="store_false", help="disable detection with PEiD (enabled by default)")
+    opt.add_argument("--peid-ep-only", action="store_false",
+                     help="disable entry point-only matching for PEiD, scanning the whole file instead (default: False)")
     opt.add_argument("--peid-large-db", action="store_true", help="use the large database for PEiD (default: False)")
-    opt.add_argument("--unknown-sections", action="store_false", help="check for unknown sections (default: True)")
+    opt.add_argument("--unknown-sections", action="store_false",
+                     help="disable the check for unknown section names (enabled by default)")
     thrs = parser.add_argument_group("threshold arguments")
-    thrs.add_argument("--bad-sections-threshold", dest="bst", type=PositiveInt, default=2,
-                      help="threshold for the number of bad sections (default: 2)")
-    thrs.add_argument("--imports-threshold", dest="it", type=PositiveInt, default=10,
+    thrs.add_argument("--bad-sections-threshold", dest="bad_sections_threshold", metavar="N", type=PositiveInt(),
+                      default=2, help="threshold for the number of bad sections (default: 2)")
+    thrs.add_argument("--imports-threshold", dest="import_threshold", metavar="N", type=PositiveInt(), default=10,
                       help="threshold for the minimum number of imports (default: 10)")
-    thrs.add_argument("--unknown-sections-threshold", dest="ust", type=PositiveInt, default=3,
-                      help="threshold for the number of unknown sections (default: 3)")
+    thrs.add_argument("--unknown-sections-threshold", dest="unknown_sections_threshold", metavar="N",
+                      type=PositiveInt(), default=3, help="threshold for the number of unknown sections (default: 3)")
     extra = parser.add_argument_group("extra arguments")
     extra.add_argument("-b", "--benchmark", action="store_true",
                        help="enable benchmarking, output in seconds (default: False)")
@@ -86,11 +91,11 @@ def main():
         PyPackerDetect.report(args.path, r)
         if dt != "":
             print(dt)
+    except PEFormatError as e:
+        args.logger.error(f"Not a valid PE file ({e.value})")
+        code = 1
     except Exception as e:
-        if "magic not found." in str(e):
-            e.value = "Not a PE file"
-        if str(e) != "no result":
-            args.logger.exception(e)
+        args.logger.exception(e)
         code = 1
     return code
 

{pypackerdetect-1.2.0 → pypackerdetect-1.2.1}/src/pypackerdetect/db/sections.json

@@ -50,7 +50,7 @@
             ".debug$P",
             ".debug$S",
             ".debug$T",
-            ".drectve ",
+            ".drectve",
             ".didat",
             ".didata",
             ".edata",
@@ -94,7 +94,7 @@
             ".vsdata",
             ".xdata",
             ".wixburn",
-            ".wpp_sf ",
+            ".wpp_sf",
             "BSS",
             "CODE",
             "DATA",

{pypackerdetect-1.2.0 → pypackerdetect-1.2.1/src/pypackerdetect.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pypackerdetect
-Version: 1.2.0
+Version: 1.2.1
 Summary: Re-implementation and packaging of PyPackerDetect
 Author-email: Alexandre D'Hondt <alexandre.dhondt@gmail.com>
 License:                     GNU GENERAL PUBLIC LICENSE
@@ -691,14 +691,14 @@ Classifier: License :: OSI Approved :: GNU General Public License v3 or later (G
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Security
-Requires-Python: <4,>=3.9
+Requires-Python: <4,>=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pefile
-Requires-Dist: peid>=2.3.1
+Requires-Dist: peid>=2.4.0
 Dynamic: license-file
 
-<p align="center"><img src="https://github.com/packing-box/PyPackerDetect/raw/main/docs/logo.png"></p>
+<p align="center"><img src="https://github.com/packing-box/PyPackerDetect/raw/main/docs/pages/img/logo.png"></p>
 <h1 align="center">PyPackerDetect <a href="https://twitter.com/intent/tweet?text=PyPackerDetect%20-%20Packer%20detector%20using%20multiple%20heuristics%20and%20PEiD's%20signatures%20database.%0D%0Ahttps%3a%2f%2fgithub%2ecom%2fpacking-box%2fPyPackerDetect%0D%0A&hashtags=python,pe,peid,packer,packingdetection"><img src="https://img.shields.io/badge/Tweet--lightgrey?logo=twitter&style=social" alt="Tweet" height="20"/></a></h1>
 <h3 align="center">Detect packers on PE files using heuristics and signatures.</h3>
 
@@ -723,8 +723,9 @@ $ pypackerdetect --help
 usage examples:
 - pypackerdetect program.exe
 - pypackerdetect program.exe -b
-- pypackerdetect program.exe --low-imports --unknown-sections
+- pypackerdetect program.exe --low-imports --unknown-sections   # disable these two checks
 - pypackerdetect program.exe --imports-threshold 5 --bad-sections-threshold 5
+- pypackerdetect program.exe --peid-db-path user_db.txt --sections-json sections.json
 ```
 
 ## Detection Mechanisms
@@ -732,7 +733,7 @@ usage examples:
 - PEID signatures
 - Known packer section names
 - Entrypoint in non-standard section
-- Threshhold of non-standard sections reached
+- Threshold of non-standard sections reached
 - Low number of imports
 - Overlapping entrypoint sections
 

{pypackerdetect-1.2.0 → pypackerdetect-1.2.1}/src/pypackerdetect.egg-info/SOURCES.txt

@@ -1,10 +1,20 @@
 .gitignore
+.readthedocs.yaml
 LICENSE
 README.md
 pyproject.toml
 requirements.txt
 .github/workflows/python-package.yml
-docs/logo.png
+docs/mkdocs.yml
+docs/requirements.txt
+docs/pages/api.md
+docs/pages/contributing.md
+docs/pages/index.md
+docs/pages/installation.md
+docs/pages/internals.md
+docs/pages/cli/pypackerdetect.md
+docs/pages/img/icon.png
+docs/pages/img/logo.png
 src/pypackerdetect/VERSION.txt
 src/pypackerdetect/__info__.py
 src/pypackerdetect/__init__.py

pypackerdetect-1.2.1/src/pypackerdetect.egg-info/requires.txt

@@ -0,0 +1,2 @@
+pefile
+peid>=2.4.0

pypackerdetect-1.2.0/requirements.txt

@@ -1,2 +0,0 @@
-pefile
-peid>=2.2.1

pypackerdetect-1.2.0/src/pypackerdetect/VERSION.txt

@@ -1 +0,0 @@
-1.2.0

pypackerdetect-1.2.0/src/pypackerdetect.egg-info/requires.txt

@@ -1,2 +0,0 @@
-pefile
-peid>=2.3.1