disasm2vec 0.1.0__tar.gz

disasm2vec-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Ahmad Nur Rohim
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

disasm2vec-0.1.0/MANIFEST.in

@@ -0,0 +1,4 @@
+include LICENSE
+include README.md
+include pyproject.toml
+recursive-include models *.json *.pkl

disasm2vec-0.1.0/PKG-INFO

@@ -0,0 +1,84 @@
+Metadata-Version: 2.4
+Name: disasm2vec
+Version: 0.1.0
+Summary: disasm2vec is a research framework designed to generate vector representations from disassembled C/C++ binaries. It provides a modular pipeline that handles compilation, disassembly, tokenization, and vectorization, enabling researchers and security analysts to transform raw code into machine-learning-ready features.
+Author-email: Ahmad Nur Rohim <ahmadnurrohim2812@gmail.com>
+Project-URL: Repository, https://github.com/Anro128/disasm2vec
+Project-URL: Issues, https://github.com/Anro128/disasm2vec/issues
+Project-URL: Changelog, https://github.com/Anro128/disasm2vec/blob/main/CHANGELOG.md
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: scikit-learn>=1.0.0
+Requires-Dist: numpy>=1.20.0
+Dynamic: license-file
+
+# disasm2vec
+
+**disasm2vec** is a research framework designed to generate vector representations from disassembled C/C++ binaries. It provides a modular pipeline that handles compilation, disassembly, tokenization, and vectorization, enabling researchers and security analysts to transform raw code into machine-learning-ready features.
+
+## Features
+
+- **Automated Compilation**: Seamlessly compiles C and C++ source files using GCC.
+- **Disassembly Wrapper**: Extracts assembly instructions using `objdump`, supporting both full and function-specific disassembly.
+- **Intelligent Tokenization**: Normalizes and cleans assembly instructions, with options to preserve or abstract register names.
+- **Vectorization**: Implements TF-IDF vectorization with a flexible factory pattern for easy model management.
+- **End-to-End Pipeline**: Orchestrates the entire process from source code to vector embedding.
+- **Extensible Architecture**: Built with abstract base classes to easily support new compilers, disassemblers, or vectorizers.
+
+## Prerequisites
+
+- **Python**: version 3.10 or higher.
+- **Operating System**: Linux or Windows Subsystem for Linux (WSL).
+- **GCC**: Required for compiling source files.
+- **Objdump**: Required for disassembling binaries.
+
+**Note**: The compilation (`gcc`) and disassembly (`objdump`) modules rely on system-level tools typically found in Linux environments. If you are on Windows, please use WSL.
+
+Ensure both `gcc` and `objdump` are installed and available in your system's PATH.
+
+## Installation
+
+Install directly from PyPI:
+
+```bash
+pip install disasm2vec
+```
+
+Or install from source:
+
+```bash
+git clone https://github.com/yourusername/disasm2vec.git
+cd disasm2vec
+pip install .
+```
+
+## Usage
+
+The core of the framework is the `PipelineRunner`, which processes a source file based on a configuration object.
+
+### Basic Example
+
+```python
+from disasm2vec.pipeline import PipelineConfig, run_pipeline
+
+# Configure the pipeline
+config = PipelineConfig(
+    source_file="examples/sample.c",
+    build_dir="build",
+    asm_dir="asm",
+    model_path="models/base_tfidf_asm.pkl"  # Path to pre-trained model or where to save a new one
+)
+
+# Run the pipeline
+# returns:
+#   vector: The vector representation of the source file
+#   vectorizer: The fitted vectorizer instance
+vector, vectorizer = run_pipeline(config)
+
+print(f"Generated Vector Shape: {vector.shape}")
+```
+
+## License
+
+This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details.

disasm2vec-0.1.0/README.md

@@ -0,0 +1,69 @@
+# disasm2vec
+
+**disasm2vec** is a research framework designed to generate vector representations from disassembled C/C++ binaries. It provides a modular pipeline that handles compilation, disassembly, tokenization, and vectorization, enabling researchers and security analysts to transform raw code into machine-learning-ready features.
+
+## Features
+
+- **Automated Compilation**: Seamlessly compiles C and C++ source files using GCC.
+- **Disassembly Wrapper**: Extracts assembly instructions using `objdump`, supporting both full and function-specific disassembly.
+- **Intelligent Tokenization**: Normalizes and cleans assembly instructions, with options to preserve or abstract register names.
+- **Vectorization**: Implements TF-IDF vectorization with a flexible factory pattern for easy model management.
+- **End-to-End Pipeline**: Orchestrates the entire process from source code to vector embedding.
+- **Extensible Architecture**: Built with abstract base classes to easily support new compilers, disassemblers, or vectorizers.
+
+## Prerequisites
+
+- **Python**: version 3.10 or higher.
+- **Operating System**: Linux or Windows Subsystem for Linux (WSL).
+- **GCC**: Required for compiling source files.
+- **Objdump**: Required for disassembling binaries.
+
+**Note**: The compilation (`gcc`) and disassembly (`objdump`) modules rely on system-level tools typically found in Linux environments. If you are on Windows, please use WSL.
+
+Ensure both `gcc` and `objdump` are installed and available in your system's PATH.
+
+## Installation
+
+Install directly from PyPI:
+
+```bash
+pip install disasm2vec
+```
+
+Or install from source:
+
+```bash
+git clone https://github.com/yourusername/disasm2vec.git
+cd disasm2vec
+pip install .
+```
+
+## Usage
+
+The core of the framework is the `PipelineRunner`, which processes a source file based on a configuration object.
+
+### Basic Example
+
+```python
+from disasm2vec.pipeline import PipelineConfig, run_pipeline
+
+# Configure the pipeline
+config = PipelineConfig(
+    source_file="examples/sample.c",
+    build_dir="build",
+    asm_dir="asm",
+    model_path="models/base_tfidf_asm.pkl"  # Path to pre-trained model or where to save a new one
+)
+
+# Run the pipeline
+# returns:
+#   vector: The vector representation of the source file
+#   vectorizer: The fitted vectorizer instance
+vector, vectorizer = run_pipeline(config)
+
+print(f"Generated Vector Shape: {vector.shape}")
+```
+
+## License
+
+This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details.

disasm2vec-0.1.0/models/base_tfidf_asm.json

@@ -0,0 +1,27 @@
+{
+  "model_type": "tfidf",
+  "vectorizer": {
+    "max_features": 5000,
+    "ngram_range": [
+      1,
+      2
+    ],
+    "min_df": 2,
+    "max_df": 1.0,
+    "norm": "l2",
+    "use_idf": true
+  },
+  "dataset": {
+    "num_documents": 653,
+    "avg_length": 153.9770290964778,
+    "hash": "a1ac3063bb6cd0ac601cb6e5421402411b722c90fca288a03db2c518425bb284"
+  },
+  "environment": {
+    "python": "3.12.3",
+    "platform": "Linux-6.6.87.2-microsoft-standard-WSL2-x86_64-with-glibc2.39"
+  },
+  "training": {
+    "timestamp": "2026-02-17T16:14:47.363169",
+    "source_dataset": "dataset_tokens.csv"
+  }
+}

disasm2vec-0.1.0/models/base_tfidf_asm_keepreg.json

@@ -0,0 +1,27 @@
+{
+  "model_type": "tfidf",
+  "vectorizer": {
+    "max_features": 5000,
+    "ngram_range": [
+      1,
+      2
+    ],
+    "min_df": 2,
+    "max_df": 1.0,
+    "norm": "l2",
+    "use_idf": true
+  },
+  "dataset": {
+    "num_documents": 653,
+    "avg_length": 153.9770290964778,
+    "hash": "2ad5da17cc369457374e2a4465e8bd5c753981f79fe9e8cd61cd2ad231be26fb"
+  },
+  "environment": {
+    "python": "3.12.3",
+    "platform": "Linux-6.6.87.2-microsoft-standard-WSL2-x86_64-with-glibc2.39"
+  },
+  "training": {
+    "timestamp": "2026-02-17T16:14:02.894795",
+    "source_dataset": "dataset_tokens_keep_register.csv"
+  }
+}

disasm2vec-0.1.0/pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "disasm2vec"
+version = "0.1.0"
+description = "disasm2vec is a research framework designed to generate vector representations from disassembled C/C++ binaries. It provides a modular pipeline that handles compilation, disassembly, tokenization, and vectorization, enabling researchers and security analysts to transform raw code into machine-learning-ready features."
+readme = "README.md"
+requires-python = ">=3.10"
+authors = [
+    { name = "Ahmad Nur Rohim", email = "ahmadnurrohim2812@gmail.com" }
+]
+dependencies = [
+    "scikit-learn>=1.0.0",
+    "numpy>=1.20.0",
+]
+
+[project.urls]
+Repository = "https://github.com/Anro128/disasm2vec"
+Issues = "https://github.com/Anro128/disasm2vec/issues"
+Changelog = "https://github.com/Anro128/disasm2vec/blob/main/CHANGELOG.md"

disasm2vec-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

disasm2vec-0.1.0/src/disasm2vec/__init__.py

@@ -0,0 +1,17 @@
+"""disasm2vec"""
+
+from . import compiler
+from . import disassembler
+from . import tokenizer
+from . import vectorizer
+from . import pipeline
+
+__all__ = [
+    "compiler",
+    "disassembler",
+    "tokenizer",
+    "vectorizer",
+    "pipeline",
+]
+
+__version__ = "0.1.0"

disasm2vec-0.1.0/src/disasm2vec/compiler/__init__.py

@@ -0,0 +1,3 @@
+from .gcc import compile_c, compile_cpp, compile_folder
+
+__all__ = ["compile_c", "compile_cpp", "compile_folder"]

disasm2vec-0.1.0/src/disasm2vec/compiler/errors.py

@@ -0,0 +1,3 @@
+class CompilationError(Exception):
+    """Raised when compilation fails."""
+    pass

disasm2vec-0.1.0/src/disasm2vec/compiler/gcc.py

@@ -0,0 +1,106 @@
+import subprocess
+from pathlib import Path
+from .errors import CompilationError
+
+def compile_c(
+    source: str,
+    output: str,
+    flags: list[str] | None = None
+):
+    """
+    Compile C source file using gcc.
+    """
+    _compile(
+        compiler="gcc",
+        source=source,
+        output=output,
+        flags=flags,
+    )
+
+
+def compile_cpp(
+    source: str,
+    output: str,
+    flags: list[str] | None = None
+):
+    """
+    Compile C++ source file using g++.
+    """
+    _compile(
+        compiler="g++",
+        source=source,
+        output=output,
+        flags=flags,
+    )
+
+
+def _compile(
+    compiler: str,
+    source: str,
+    output: str,
+    flags: list[str] | None = None,
+):
+    source = Path(source)
+    output = Path(output)
+
+    if not source.exists():
+        raise FileNotFoundError(source)
+
+    cmd = [
+        compiler,
+        str(source),
+        "-o",
+        str(output),
+    ]
+
+    if flags:
+        cmd.extend(flags)
+
+    try:
+        subprocess.run(
+            cmd,
+            check=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+        )
+    except subprocess.CalledProcessError as e:
+        raise CompilationError(
+            f"Compilation failed for {source}:\n{e.stderr}"
+        ) from e
+
+
+def compile_folder(
+    src_dir: str,
+    out_dir: str,
+    optimize: str = "-O0",
+    extra_flags: list[str] | None = None,
+):
+    """
+    Compile all .c and .cpp files in a folder (recursively).
+    """
+    src_dir = Path(src_dir)
+    out_dir = Path(out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    extra_flags = extra_flags or []
+
+    sources = list(src_dir.rglob("*.c")) + list(src_dir.rglob("*.cpp"))
+
+    if not sources:
+        raise ValueError(f"No C/C++ files found in {src_dir}")
+
+    for src in sources:
+        output = out_dir / src.stem
+        flags = [optimize, *extra_flags]
+
+        try:
+            if src.suffix == ".c":
+                compile_c(src, output, flags)
+            else:
+                compile_cpp(src, output, flags)
+
+        except CompilationError as e:
+            raise CompilationError(
+                f"Compilation failed for {src}:\n{e}"
+            ) from e

disasm2vec-0.1.0/src/disasm2vec/disassembler/__init__.py

@@ -0,0 +1,6 @@
+from .objdump import disassemble, disassemble_folder
+
+__all__ = [
+    "disassemble",
+    "disassemble_folder",
+]

disasm2vec-0.1.0/src/disasm2vec/disassembler/errors.py

@@ -0,0 +1,2 @@
+class DisassemblyError(RuntimeError):
+    """Raised when objdump disassembly fails."""

disasm2vec-0.1.0/src/disasm2vec/disassembler/objdump.py

@@ -0,0 +1,127 @@
+import subprocess
+from pathlib import Path
+from .errors import DisassemblyError
+
+
+def disassemble(
+    binary: str,
+    output: str,
+    arch: str | None = None,
+    full: bool = False,
+):
+    """
+    Disassemble a single binary using objdump.
+
+    Parameters
+    ----------
+    binary : str
+        Path to compiled binary
+    output : str
+        Output .asm file
+    arch : str | None
+        Optional architecture (e.g. i386:x86-64)
+    full : bool
+        If True, disassemble all functions.
+        If False, exclude builtin / PLT functions.
+    """
+    binary = Path(binary)
+    output = Path(output)
+
+    if not binary.exists():
+        raise FileNotFoundError(binary)
+
+    output.parent.mkdir(parents=True, exist_ok=True)
+
+    # Base command
+    cmd = ["objdump", "-d", "--section=.text", str(binary)]
+
+    if arch:
+        cmd.extend(["-m", arch])
+
+    try:
+        result = subprocess.run(
+            cmd,
+            check=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+        )
+    except subprocess.CalledProcessError as e:
+        raise DisassemblyError(
+            f"objdump failed for {binary}:\n{e.stderr}"
+        ) from e
+
+    asm = result.stdout
+
+    if not full:
+        asm = _filter_builtin_functions(asm)
+
+    output.write_text(asm)
+
+
+def disassemble_folder(
+    bin_dir: str,
+    out_dir: str,
+    full: bool = False,
+):
+    """
+    Disassemble all binaries in a folder.
+
+    Parameters
+    ----------
+    bin_dir : str
+        Folder containing compiled binaries
+    out_dir : str
+        Folder to store .asm outputs
+    full : bool
+        If True, disassemble all functions.
+        If False, exclude builtin / PLT functions.
+    """
+    bin_dir = Path(bin_dir)
+    out_dir = Path(out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    binaries = [p for p in bin_dir.iterdir() if p.is_file()]
+
+    if not binaries:
+        raise ValueError(f"No binaries found in {bin_dir}")
+
+    for binary in binaries:
+        asm_out = out_dir / f"{binary.name}.asm"
+
+        try:
+            disassemble(binary, asm_out, full=full)
+        except DisassemblyError as e:
+            raise DisassemblyError(
+                f"Disassembly failed for {binary}:\n{e}"
+            ) from e
+
+
+def _filter_builtin_functions(asm: str) -> str:
+    """
+    Remove builtin / PLT / runtime functions from objdump output.
+    """
+    filtered_lines = []
+
+    skip = False
+    for line in asm.splitlines():
+        if "<" in line and ">" in line and line.strip().endswith(":"):
+            name = line.split("<")[1].split(">")[0]
+
+            if (
+                name.endswith("@plt")
+                or name.startswith("_start")
+                or name.startswith("frame_dummy")
+                or name.startswith("register_tm_clones")
+                or name.startswith("deregister_tm_clones")
+                or name.startswith("__")
+            ):
+                skip = True
+                continue
+            else:
+                skip = False
+
+        if not skip:
+            filtered_lines.append(line)
+
+    return "\n".join(filtered_lines)

disasm2vec-0.1.0/src/disasm2vec/pipeline/__init__.py

@@ -0,0 +1,7 @@
+from .config import PipelineConfig
+from .runner import run_pipeline
+
+__all__ = [
+    "run_pipeline",
+    "PipelineConfig"
+]

disasm2vec-0.1.0/src/disasm2vec/pipeline/config.py

@@ -0,0 +1,32 @@
+from dataclasses import dataclass
+from typing import Optional, Tuple
+
+
+@dataclass
+class PipelineConfig:
+    source_file: str
+
+    build_dir: str
+    asm_dir: str
+
+    # compiler
+    optimize: str = "-O0"
+    extra_flags: Optional[list[str]] = None
+
+    # disassembler
+    arch: Optional[str] = None
+    full_disasm: bool = False
+
+    # tokenizer
+    entry: str = "main"
+    keep_register: bool = False
+
+    # vectorizer
+    model_path: Optional[str] = None
+    max_features: Optional[int] = None
+    ngram_range: Tuple[int, int] = (1, 2)
+    min_df: int = 1
+
+    # switches
+    do_compile: bool = True
+    do_disassemble: bool = True

disasm2vec-0.1.0/src/disasm2vec/pipeline/runner.py

@@ -0,0 +1,78 @@
+from pathlib import Path
+
+from disasm2vec.compiler import compile_c, compile_cpp
+from disasm2vec.disassembler import disassemble
+from disasm2vec.tokenizer import tokenize
+from disasm2vec.vectorizer import Tfidf
+
+from .config import PipelineConfig
+
+
+def run_pipeline(config: PipelineConfig):
+    """
+    Run pipeline for single source file.
+
+    Flow:
+        source -> compile -> disassemble -> tokenizer -> vectorize
+    """
+
+    source = Path(config.source_file)
+
+    if not source.exists():
+        raise FileNotFoundError(source)
+
+    stem = source.stem
+
+    binary_path = Path(config.build_dir) / stem
+    asm_path = Path(config.asm_dir) / f"{stem}.asm"
+
+    binary_path.parent.mkdir(parents=True, exist_ok=True)
+    asm_path.parent.mkdir(parents=True, exist_ok=True)
+
+    # COMPILE
+    if config.do_compile:
+        flags = [config.optimize]
+        if config.extra_flags:
+            flags.extend(config.extra_flags)
+
+        if source.suffix == ".c":
+            compile_c(source, binary_path, flags)
+
+        elif source.suffix == ".cpp":
+            compile_cpp(source, binary_path, flags)
+
+        else:
+            raise ValueError(
+                f"Unsupported source type: {source.suffix}"
+            )
+
+    # DISASSEMBLE
+    if config.do_disassemble:
+        disassemble(
+            binary=binary_path,
+            output=asm_path,
+            arch=config.arch,
+            full=config.full_disasm,
+        )
+
+    # TOKENIZER
+    corpus = tokenize(
+        path=asm_path,
+        entry=config.entry,
+        keep_register=config.keep_register,
+    )
+
+    # VECTORIZE
+    if not config.model_path:
+        raise ValueError("model_path is required for pipeline")
+
+    vectorizer = Tfidf(
+        max_features=config.max_features,
+        ngram_range=config.ngram_range,
+        min_df=config.min_df,
+    )
+    vectorizer.load(config.model_path)
+    
+    X = vectorizer.transform_one(corpus)
+
+    return X, vectorizer

disasm2vec-0.1.0/src/disasm2vec/tokenizer/__init__.py

@@ -0,0 +1,4 @@
+from .core import tokenize, tokenize_batch
+
+__all__ = ["tokenize", 
+           "tokenize_batch",]

disasm2vec-0.1.0/src/disasm2vec/tokenizer/cleaner.py

@@ -0,0 +1,10 @@
+import re
+
+INSTRUCTION_PATTERN = re.compile(r"\s*[0-9a-fA-F]+:\s+")
+
+
+def is_instruction_line(line: str) -> bool:
+    """
+    Check whether a line is a valid objdump instruction line.
+    """
+    return bool(INSTRUCTION_PATTERN.match(line))