karta-ng 2.3.0__tar.gz

karta_ng-2.3.0/MANIFEST.in

@@ -0,0 +1,9 @@
+# Cython speedups must ship in sdists so a source install can build them.
+recursive-include src/karta_ng/speedups *.pyx
+recursive-include src/karta_ng/speedups *.pxd
+
+# Never ship build output or VCS metadata.
+prune build
+prune dist
+prune *.egg-info
+global-exclude *.o *.a *.so *.dylib *.pyc __pycache__

karta_ng-2.3.0/PKG-INFO

@@ -0,0 +1,23 @@
+Metadata-Version: 2.4
+Name: karta-ng
+Version: 2.3.0
+Summary: karta-ng (Karta next-gen): identify and match open-source libraries inside binaries. Structural function fingerprinting and location-driven matching for malware and firmware analysis.
+Author-email: Mahmoud Abdelkader <mabdelkader@gmail.com>
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: elementals
+Requires-Dist: click
+Requires-Dist: scikit-learn>=1.5.0
+Provides-Extra: speedups
+Requires-Dist: Cython>=3.0.0; extra == "speedups"
+Provides-Extra: domain
+Requires-Dist: ida-domain>=0.5.1; extra == "domain"
+Provides-Extra: dev
+Requires-Dist: setuptools>=77.0.1; extra == "dev"
+Requires-Dist: libcst>=1.0.0; extra == "dev"
+Requires-Dist: import-linter>=2.0; extra == "dev"
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: coverage[toml]>=7.0.0; extra == "dev"
+Requires-Dist: ruff>=0.12; extra == "dev"
+Requires-Dist: pydocstyle; extra == "dev"

karta_ng-2.3.0/pyproject.toml

@@ -0,0 +1,76 @@
+[build-system]
+requires = ["setuptools>=77.0.1", "wheel", "Cython>=3.0.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "karta-ng"
+requires-python = ">= 3.10"
+dynamic = ["version"]
+authors = [
+    { name = "Mahmoud Abdelkader", email = "mabdelkader@gmail.com" },
+]
+description = "karta-ng (Karta next-gen): identify and match open-source libraries inside binaries. Structural function fingerprinting and location-driven matching for malware and firmware analysis."
+readme = "README.md"
+dependencies = [
+    "elementals",
+    "click",
+    "scikit-learn>=1.5.0",
+]
+
+[project.optional-dependencies]
+speedups = ["Cython>=3.0.0"]
+domain = ["ida-domain>=0.5.1"]  # IDA Domain API (requires IDA 9.1+/idapro)
+dev = [
+    "setuptools>=77.0.1",
+    "libcst>=1.0.0",
+    "import-linter>=2.0",
+    "pytest>=7.0.0",
+    "pytest-cov>=4.0.0",
+    "coverage[toml]>=7.0.0",
+    "ruff>=0.12",
+    "pydocstyle",
+]
+
+[project.scripts]
+install-speedups = "karta_ng.speedups.install:main"
+karta-analyze-src = "karta_ng.karta_analyze_src:main"
+
+[tool.setuptools]
+package-dir = { "" = "src" }
+
+[tool.setuptools.dynamic]
+version = { attr = "karta_ng.__version__" }
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+# pytest configuration
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = ["-m", "not requires_ida and not slow"]
+markers = [
+    "requires_ida: Tests that require IDA Pro to run",
+    "runtime: IDA runtime invariant/API/stability tests",
+    "e2e: End-to-end pipeline tests with golden output comparison",
+    "slow: Slow tests (>10s) - typically long-running matching",
+    "integration: mark test as integration test",
+]
+
+[tool.coverage.run]
+source = ["src/karta_ng"]
+omit = [
+    "*/tests/*",
+    "*/test_*",
+    "*/__pycache__/*",
+    # Seekers and the ML classifier are fixture-driven: without a real
+    # library corpus they are only imported, not exercised.  Exclude them
+    # from the coverage gate so it measures the testable core instead of
+    # being dragged down by an untestable surface.
+    "src/karta_ng/libs/*",
+    "src/karta_ng/thumbs_up/*",
+]
+
+[tool.coverage.report]

karta_ng-2.3.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

karta_ng-2.3.0/setup.py

@@ -0,0 +1,74 @@
+"""Optional Cython extension builder for karta-ng.
+
+By default, karta-ng installs as a pure Python package.
+To build with Cython speedups:
+
+    KARTA_NG_BUILD_SPEEDUPS=1 pip install -e .[speedups]
+
+This setup.py only handles ext_modules; all other config is in pyproject.toml.
+Any ``.pyx`` files placed under ``src/karta_ng/speedups/`` are compiled when the
+env var is set; otherwise the pure-Python implementations are used.
+"""
+
+import os
+import pathlib
+
+from setuptools import setup
+
+BUILD_SPEEDUPS = os.environ.get("KARTA_NG_BUILD_SPEEDUPS", "0") == "1"
+DEBUG = os.environ.get("DEBUG") == "1"
+
+
+def get_ext_modules():
+    """Build Cython extensions if KARTA_NG_BUILD_SPEEDUPS=1, else return []."""
+    want_speedups = os.environ.get("KARTA_NG_BUILD_SPEEDUPS", "0") == "1"
+    if not want_speedups:
+        return []
+
+    try:
+        from Cython.Build import cythonize
+        from setuptools import Extension
+    except ImportError:
+        raise ImportError(
+            "Cython is required to build speedups. "
+            "Install with: pip install 'karta-ng[speedups]'"
+        )
+
+    speedups_root = pathlib.Path(__file__).parent / "src" / "karta_ng" / "speedups"
+    pyx_files = sorted(str(p) for p in speedups_root.rglob("*.pyx"))
+    if not pyx_files:
+        print("[speedups] no .pyx files found under src/karta_ng/speedups; nothing to build")
+        return []
+
+    macros = []
+    if DEBUG:
+        macros.extend([("CYTHON_TRACE", "1"), ("CYTHON_CLINE_IN_TRACEBACK", "1")])
+
+    return cythonize(
+        [
+            Extension(
+                pathlib.Path(p).with_suffix("").as_posix().replace("/", "."),
+                [p],
+                language="c++",
+                define_macros=macros,
+                extra_compile_args=(
+                    ["-g", "-O0"] if DEBUG else []
+                ),
+            )
+            for p in pyx_files
+        ],
+        compiler_directives={
+            "language_level": "3",
+            "binding": True,
+            "embedsignature": True,
+            "boundscheck": False,
+            "wraparound": False,
+            "profile": DEBUG,
+            "linetrace": DEBUG,
+        },
+        annotate=DEBUG,
+    )
+
+
+# Minimal setup() - everything else comes from pyproject.toml
+setup(ext_modules=get_ext_modules())

karta_ng-2.3.0/src/karta_ng/__init__.py

@@ -0,0 +1,7 @@
+"""karta-ng (Karta next-gen).
+
+Identify and match open-source libraries inside binaries using structural
+function fingerprinting and location-driven matching.
+"""
+
+__version__ = "2.3.0"

karta_ng-2.3.0/src/karta_ng/analyze_src_file.py

@@ -0,0 +1,63 @@
+from karta_ng.config.utils import (
+    get_disas,
+    set_windows_mode,
+    functions_to_file,
+    LIBRARY_NAME,
+    construct_init_log_path,
+    init_utils,
+    register_contexts,
+    construct_log_path,
+)
+from karta_ng.disassembler.factory import create_disassembler_handler
+from karta_ng.function_context import SourceContext, BinaryContext, IslandContext
+from elementals import Logger
+import logging
+import traceback
+
+
+def analyze_file():
+    """Analyzes all of the (source) functions for a single compiled file."""
+    disas = get_disas()
+    logger.info("Started the Script")
+    contexts = []
+    # check for windows binary
+    if disas.input_file().endswith(".obj"):
+        logger.debug("Activating Windows mode")
+        set_windows_mode()
+    # build the list of exported (non-static) functions
+    exported = disas.exports()
+    for segment_idx in range(disas.num_segments()):
+        # Some executables use "_text" and some use ".text", so just look for "text"
+        # in section name. Also look for other sections marked executable.
+        if "text" not in disas.segment_name(segment_idx) and "X" not in disas.segment_permissions(segment_idx):
+            continue
+        for function_ea in disas.segment_functions(segment_idx):
+            src_ctx = disas.analyze_function(function_ea, True)
+            # check if static or not
+            if src_ctx.name not in exported:
+                src_ctx.mark_static()
+            contexts.append(src_ctx)
+    functions_to_file(disas.input_file(), contexts)
+    logger.info("Finished Successfully")
+
+
+# Create a basic logger for the init phase
+init_logger = Logger(LIBRARY_NAME)
+init_logger.linkHandler(logging.FileHandler(construct_init_log_path(), "w"))
+disas = create_disassembler_handler(init_logger)
+# In case of a dependency issue, disas will be None
+if disas is None:
+    exit()
+# Always init the utils before we start (now with the real logger too)
+logger = Logger(LIBRARY_NAME, use_stdout=False)
+init_utils(logger, disas)
+# Register our contexts
+register_contexts(SourceContext, BinaryContext, IslandContext)
+# Start to analyze the file
+try:
+    logger.linkHandler(logging.FileHandler(construct_log_path(), "w"))
+    analyze_file()
+except Exception:
+    logger.error(traceback.format_exc())
+# Exit the disassembler
+get_disas().exit()

karta_ng-2.3.0/src/karta_ng/ar_parser.py

@@ -0,0 +1,52 @@
+def get_archive_files(ar_path):
+    """Return an ordered list of the files within the .ar archive.
+
+    Args:
+        ar_path (str): path to the compiled *.ar file
+
+    Return Value:
+        Ordered list of file names
+    """
+    import shutil
+    ar_bin = shutil.which("ar")
+    names = []
+
+    if ar_bin:
+        import subprocess
+        proc = subprocess.run([ar_bin, "t", ar_path], capture_output=True, text=True)
+        if proc.returncode == 0:
+            for line in proc.stdout.split("\n"):
+                if line:
+                    names.append(line)
+
+    if not names:
+        ar_fd = open(ar_path, "rb")
+        is_windows = ar_path.endswith(".lib")
+
+        # check the signature
+        if ar_fd.read(8) != b"!<arch>\n":
+            raise ValueError("Invalid archive signature")
+
+        # split the content to parts
+        ar_content = ar_fd.read()
+        for ar_part in ar_content.split(b"\x60\x0A")[:-1]:
+            # .ar file format (unix) seems simpler
+            if not is_windows:
+                # sanity check
+                if len(ar_part) < 58:
+                    continue
+                # now read the metadata of the record
+                name = ar_part[-58:].split(b'/')[0]
+                if not name.endswith(b".o"):
+                    continue
+            # .lib file format is more complex
+            else:
+                if ar_part.find(b".obj") == -1:
+                    continue
+                name = ar_part.split(b".obj")[-2].split(b"\x00")[-1].split(b"\\")[-1] + b".obj"
+                name = name.strip()
+            # append the new record
+            if name not in names and len(name) > 0:
+                names.append(name.decode("utf-8"))
+        ar_fd.close()
+    return names

karta_ng-2.3.0/src/karta_ng/config/__init__.py

@@ -0,0 +1,318 @@
+from .utils import (
+    DISASSEMBLER_PATH,
+    GUI_CMD_IMPORT_MATCHED,
+    GUI_CMD_IMPORT_SELECTED,
+    GUI_COLOR_DARK_GREEN,
+    GUI_COLOR_DARK_RED,
+    GUI_COLOR_GRAY,
+    GUI_COLOR_GREEN,
+    GUI_COLOR_RED,
+    GUI_MATCH_REASONS,
+    JSON_TAG_ANCHORS,
+    JSON_TAG_FILES,
+    JSON_TAG_LIBRARY,
+    JSON_TAG_MANUAL_ANCHORS,
+    JSON_TAG_MANUAL_VERSIONS,
+    KNOWLEDGE_FILE_SUFFIX,
+    KartaException,
+    LIBRARY_NAME,
+    LOG_FILE_SUFFIX,
+    REASON_AGENT,
+    REASON_ANCHOR,
+    REASON_CALL_ORDER,
+    REASON_COLLISION,
+    REASON_DISABLED,
+    REASON_FILE_HINT,
+    REASON_FILE_SINGLETON,
+    REASON_LIBRARY_UNUSED,
+    REASON_MANUAL_ANCHOR,
+    REASON_NEIGHBOUR,
+    REASON_SCORE,
+    REASON_SINGLE_CALL,
+    REASON_SINGLE_XREF,
+    REASON_STATIC_UNUSED,
+    REASON_STRUCTURAL_SIGNATURE,
+    REASON_SWALLOW,
+    REASON_TRAPPED_COUPLE,
+    SCRIPT_PATH,
+    STATE_FILE_SUFFIX,
+    accumulated_knowledge_path,
+    are_neighbours_safe,
+    bin_func_class,
+    bin_instr_count,
+    binary_context,
+    construct_config_path,
+    construct_init_log_path,
+    construct_log_path,
+    count_set_bits,
+    disas_layer,
+    functions_to_file,
+    get_contexts_stats,
+    get_disas,
+    get_disas_path,
+    get_neighbour_score,
+    get_source_functions,
+    global_logger,
+    init_utils,
+    island_context,
+    island_func_class,
+    is_matching,
+    isWindows,
+    libraryName,
+    load_knowledge,
+    matched_library_name,
+    matching_mode,
+    measure_bits_entropy,
+    measure_bits_variance,
+    num_instr_samples,
+    num_neighbours_matched,
+    num_neighbours_mismatched,
+    parse_file_stats,
+    rank_const,
+    record_instr_ratio,
+    record_neighbour_match,
+    register_contexts,
+    reset_ratio,
+    reset_scoring,
+    set_disassembler_path,
+    set_matching_mode,
+    set_windows_mode,
+    source_context,
+    src_file_mappings,
+    src_func_class,
+    src_functions_ctx,
+    src_functions_list,
+    src_instr_count,
+    src_seen_consts,
+    src_seen_strings,
+    store_knowledge,
+    windows_config,
+)
+
+from .score_config import (
+    AGENT_BOOST_SCORE,
+    ARTIFACT_MATCH_SCORE,
+    BLOCK_MATCH_SCORE,
+    BLOCK_MISMATCH_SCORE,
+    CALL_COUNT_SCORE,
+    CONST_BOOST_BIT_FLAG,
+    CONST_BOOST_SMALL_FUNCS,
+    CONST_BOOST_SPECIAL,
+    CONST_SPECIAL_VALUES,
+    EXISTENCE_BOOST_SCORE,
+    EXTERNAL_COUNT_SCORE,
+    EXT_FUNC_MATCH_SCORE,
+    FRAME_SAFETY_GAP,
+    FRAME_SIZE_THRESHOLD,
+    FUNC_FRAME_SCORE,
+    FUNC_HINT_SCORE,
+    INSTR_COUNT_SCORE,
+    INSTR_COUNT_THRESHOLD,
+    INSTR_RATIO_COUNT_THRESHOLD,
+    LIBC_COMP_FUNC_MATCH_SCORE,
+    LIBC_FUNC_MATCH_SCORE,
+    LOCATION_BOOST_LOW_THRESHOLD,
+    LOCATION_BOOST_SCORE,
+    LOCATION_BOOST_TOP_THRESHOLD,
+    MATCHED_CALL_SCORE,
+    MINIMAL_BLOCKS_BOOST,
+    MINIMAL_ISLAND_SCORE,
+    MINIMAL_MATCH_SCORE,
+    MINIMAL_NEIGHBOUR_THRESHOLD,
+    MIN_STR_SIZE,
+    NUM_BITS_IN_CONST,
+    SAFTEY_GAP_SCORE,
+    STATIC_VIOLATION_PENALTY,
+    STRING_MATCH_SCORE,
+    STRING_MISMATCH_SCORE,
+    STRING_NAME_SCORE,
+)
+
+from .libc_config import (
+    gcc_skip_functions,
+    gcc_skip_math_functions,
+    libc_comp_function_names,
+    libc_function_names,
+    libc_functions,
+    skip_function_names,
+    windows_skip_functions,
+    windows_skip_math_functions,
+)
+
+from .anchor_config import (
+    CONST_COMPLEX_GROUP,
+    CONST_COMPLEX_LIMIT,
+    CONST_MEDIUM_GROUP,
+    CONST_MEDIUM_LIMIT,
+    MAXIMAL_CONST_SEARCH_RATE,
+    MAXIMAL_CONST_SEARCH_TIME,
+    STRING_HUGE_GROUP,
+    STRING_HUGE_LIMIT,
+    STRING_LONG_GROUP,
+    STRING_LONG_LIMIT,
+    STRING_MEDIUM_GROUP,
+    STRING_MEDIUM_LIMIT,
+    STRING_SHORT_GROUP,
+    STRING_SHORT_LIMIT,
+)
+
+from .anchor import (
+    is_agent,
+    isAnchor,
+)
+
+__all__ = [
+    # From utils
+    "DISASSEMBLER_PATH",
+    "GUI_CMD_IMPORT_MATCHED",
+    "GUI_CMD_IMPORT_SELECTED",
+    "GUI_COLOR_DARK_GREEN",
+    "GUI_COLOR_DARK_RED",
+    "GUI_COLOR_GRAY",
+    "GUI_COLOR_GREEN",
+    "GUI_COLOR_RED",
+    "GUI_MATCH_REASONS",
+    "JSON_TAG_ANCHORS",
+    "JSON_TAG_FILES",
+    "JSON_TAG_LIBRARY",
+    "JSON_TAG_MANUAL_ANCHORS",
+    "JSON_TAG_MANUAL_VERSIONS",
+    "KNOWLEDGE_FILE_SUFFIX",
+    "KartaException",
+    "LIBRARY_NAME",
+    "LOG_FILE_SUFFIX",
+    "REASON_AGENT",
+    "REASON_ANCHOR",
+    "REASON_CALL_ORDER",
+    "REASON_COLLISION",
+    "REASON_DISABLED",
+    "REASON_FILE_HINT",
+    "REASON_FILE_SINGLETON",
+    "REASON_LIBRARY_UNUSED",
+    "REASON_MANUAL_ANCHOR",
+    "REASON_NEIGHBOUR",
+    "REASON_SCORE",
+    "REASON_SINGLE_CALL",
+    "REASON_SINGLE_XREF",
+    "REASON_STATIC_UNUSED",
+    "REASON_STRUCTURAL_SIGNATURE",
+    "REASON_SWALLOW",
+    "REASON_TRAPPED_COUPLE",
+    "SCRIPT_PATH",
+    "STATE_FILE_SUFFIX",
+    "accumulatedKnowledgePath",
+    "areNeighboursSafe",
+    "bin_func_class",
+    "bin_instr_count",
+    "binaryContext",
+    "constructConfigPath",
+    "constructInitLogPath",
+    "constructLogPath",
+    "countSetBits",
+    "disas_layer",
+    "functionsToFile",
+    "getContextsStats",
+    "getDisas",
+    "getDisasPath",
+    "getNeighbourScore",
+    "getSourceFunctions",
+    "global_logger",
+    "initUtils",
+    "islandContext",
+    "island_func_class",
+    "isMatching",
+    "isWindows",
+    "libraryName",
+    "loadKnowledge",
+    "matched_library_name",
+    "matching_mode",
+    "measureBitsEntropy",
+    "measureBitsVariance",
+    "num_instr_samples",
+    "num_neighbours_matched",
+    "num_neighbours_mismatched",
+    "parseFileStats",
+    "rankConst",
+    "recordInstrRatio",
+    "recordNeighbourMatch",
+    "registerContexts",
+    "resetRatio",
+    "resetScoring",
+    "setDisassemblerPath",
+    "setMatchingMode",
+    "setWindowsMode",
+    "sourceContext",
+    "src_file_mappings",
+    "src_func_class",
+    "src_functions_ctx",
+    "src_functions_list",
+    "src_instr_count",
+    "src_seen_consts",
+    "src_seen_strings",
+    "storeKnowledge",
+    "windows_config",
+    # From score_config
+    "AGENT_BOOST_SCORE",
+    "ARTIFACT_MATCH_SCORE",
+    "BLOCK_MATCH_SCORE",
+    "BLOCK_MISMATCH_SCORE",
+    "CALL_COUNT_SCORE",
+    "CONST_BOOST_BIT_FLAG",
+    "CONST_BOOST_SMALL_FUNCS",
+    "CONST_BOOST_SPECIAL",
+    "CONST_SPECIAL_VALUES",
+    "EXISTENCE_BOOST_SCORE",
+    "EXTERNAL_COUNT_SCORE",
+    "EXT_FUNC_MATCH_SCORE",
+    "FRAME_SAFETY_GAP",
+    "FRAME_SIZE_THRESHOLD",
+    "FUNC_FRAME_SCORE",
+    "FUNC_HINT_SCORE",
+    "INSTR_COUNT_SCORE",
+    "INSTR_COUNT_THRESHOLD",
+    "INSTR_RATIO_COUNT_THRESHOLD",
+    "LIBC_COMP_FUNC_MATCH_SCORE",
+    "LIBC_FUNC_MATCH_SCORE",
+    "LOCATION_BOOST_LOW_THRESHOLD",
+    "LOCATION_BOOST_SCORE",
+    "LOCATION_BOOST_TOP_THRESHOLD",
+    "MATCHED_CALL_SCORE",
+    "MINIMAL_BLOCKS_BOOST",
+    "MINIMAL_ISLAND_SCORE",
+    "MINIMAL_MATCH_SCORE",
+    "MINIMAL_NEIGHBOUR_THRESHOLD",
+    "MIN_STR_SIZE",
+    "NUM_BITS_IN_CONST",
+    "SAFTEY_GAP_SCORE",
+    "STATIC_VIOLATION_PENALTY",
+    "STRING_MATCH_SCORE",
+    "STRING_MISMATCH_SCORE",
+    "STRING_NAME_SCORE",
+    # From libc_config
+    "gcc_skip_functions",
+    "gcc_skip_math_functions",
+    "libc_comp_function_names",
+    "libc_function_names",
+    "libc_functions",
+    "skip_function_names",
+    "windows_skip_functions",
+    "windows_skip_math_functions",
+    # From anchor_config
+    "CONST_COMPLEX_GROUP",
+    "CONST_COMPLEX_LIMIT",
+    "CONST_MEDIUM_GROUP",
+    "CONST_MEDIUM_LIMIT",
+    "MAXIMAL_CONST_SEARCH_RATE",
+    "MAXIMAL_CONST_SEARCH_TIME",
+    "STRING_HUGE_GROUP",
+    "STRING_HUGE_LIMIT",
+    "STRING_LONG_GROUP",
+    "STRING_LONG_LIMIT",
+    "STRING_MEDIUM_GROUP",
+    "STRING_MEDIUM_LIMIT",
+    "STRING_SHORT_GROUP",
+    "STRING_SHORT_LIMIT",
+    # From anchor
+    "isAgent",
+    "isAnchor",
+]