reait 0.0.13__py3-none-any.whl → 0.0.15__py3-none-any.whl

reait-0.0.13.data/scripts/reait → reait/__main__.py

@@ -1,27 +1,29 @@
-#!python
+#!/usr/bin/env python
 from __future__ import print_function
 from json import dumps
 from os import system
 from tqdm import tqdm 
 from hashlib import sha256
-from rich import print_json
+from rich import print_json, print as rich_print
 from sklearn.metrics.pairwise import cosine_similarity
 import os
-from argparse import ArgumentParser
+import re
+import argparse
 import requests
-from numpy import array, vstack
+from numpy import array, vstack, mean
 from pandas import DataFrame
 import json
 import tomli
 from os.path import isfile
 from sys import exit
-# from IPython import embed
+from IPython import embed
 
-__version__ = "0.0.13"
+__version__ = "0.0.15"
 
 re_conf = {
     'apikey' : 'l1br3', 
-    'host' : 'https://api.reveng.ai'
+    'host' : 'https://api.reveng.ai',
+    'model': 'binnet-0.1'
 }
 
 def reveng_req(r: requests.request, end_point: str, data=None, ex_headers: dict = None, params=None):
@@ -47,11 +49,14 @@ def RE_delete(fpath: str):
     return
 
 
-def RE_analyse(fpath: str):
+def RE_analyse(fpath: str, model: str = None):
     """
         Start analysis job for binary file
     """
-    res = reveng_req(requests.post, f"analyse", data=open(fpath, 'rb').read())
+    params={}
+    if model:
+        params['model'] = model
+    res = reveng_req(requests.post, f"analyse", data=open(fpath, 'rb').read(), params=params)
     if res.status_code == 200:
         print("[+] Successfully submitted binary for analysis.")
         print(f"[+] {fpath} - {binary_id(fpath)}")
@@ -65,6 +70,24 @@ def RE_analyse(fpath: str):
     res.raise_for_status()
 
 
+def RE_upload(fpath: str):
+    """
+        Upload binary to Server
+    """
+    res = reveng_req(requests.post, f"upload", data=open(fpath, 'rb').read())
+    if res.status_code == 200:
+        print("[+] Successfully uploaded binary to your account.")
+        print(f"[+] {fpath} - {binary_id(fpath)}")
+        return res
+
+    if res.status_code == 400:
+        if 'already exists' in json.loads(res.text)['reason']:
+            print(f"[-] {fpath} already exists. Please check the results log file for {binary_id(fpath)}")
+            return True
+
+    res.raise_for_status()
+
+
 def RE_embeddings(fpath: str):
     """
         Fetch symbol embeddings
@@ -94,11 +117,33 @@ def RE_logs(fpath: str):
     res.raise_for_status()
 
 
-def RE_compute_distance(embedding: list, fpath_source: str, nns: int = 5):
+def RE_cves(fpath: str):
+    """
+        Check for known CVEs in Binary 
+    """
+    bin_id = binary_id(fpath)
+    res = reveng_req(requests.get, f"/cves/{bin_id}")
+    if res.status_code == 200:
+        cves = json.loads(res.text)
+        rich_print(f"[bold blue]Checking for known CVEs embedded inside [/bold blue] [bold bright_green]{fpath}[/bold bright_green]:")
+        if len(cves) == 0:
+            rich_print(f"[bold bright_green]0 CVEs found.[/bold bright_green]")
+        else:
+            rich_print(f"[bold red]Warning CVEs found![/bold red]")
+            print_json(data=cves)
+        return
+    elif res.status_code == 404:
+        print(f"[!] Error, binary analysis for {bin_id} not found.")
+        return
+
+    res.raise_for_status()
+
+
+#def RE_compute_distance(embedding: list, fpath_source: str, nns: int = 5):
+def RE_compute_distance(embedding: list, embeddings: list, nns: int = 5):
     """
-        Comput ecosine distance between source embedding and embeddinsg from binary
+        Compute the cosine distance between source embedding and embeddinsg from binary
     """
-    embeddings = RE_embeddings(fpath_source)
     df = DataFrame(data=embeddings)
     np_embedding = array(embedding).reshape(1, -1)
     source_embeddings = vstack(df['embedding'].values)
@@ -112,15 +157,18 @@ def RE_compute_distance(embedding: list, fpath_source: str, nns: int = 5):
     return json_sims
 
 
-def RE_nearest_symbols(embedding: list, nns: int = 5):
+def RE_nearest_symbols(embedding: list, nns: int = 5, collections : list = None):
     """
         Get function name suggestions for an embedding
+        :param embedding: embedding vector as python list
         :param nns: Number of nearest neighbors
-        :param source: Binary file to search embeddings from
+        :param collections: str RegEx to search through RevEng.AI collections
     """
     params={'nns': nns}
-    if source:
-        params['source'] = source
+
+    if collections:
+        params['collections'] = collections
+
     res = reveng_req(requests.post, "ann", data=json.dumps(embedding), params=params)
     res.raise_for_status()
     f_suggestions = res.json()
@@ -152,44 +200,71 @@ def version():
     """
         Display program version
     """
-    print(f"reait v{__version__}")
+    rich_print(f"[bold red]reait[/bold red] [bold bright_green]v{__version__}[/bold bright_green]")
     print_json(data=re_conf)
 
-if __name__ == '__main__':
+
+def main() -> None:
+    """
+    Tool entry
+    """
     parse_config()
-    parser = ArgumentParser()
-    parser.add_argument("-b", "--binary", default="", help="Path on binary to analyse")
-    parser.add_argument("-a", "--analyse", action='store_true', help="Analyse new binary")
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument("-b", "--binary", default="", help="Path of binary to analyse")
+    parser.add_argument("-a", "--analyse", action='store_true', help="Perform a full analysis and generate embeddings for every symbol")
+    parser.add_argument("--no-embeddings", action='store_true', help="Only perform binary analysis. Do not generate embeddings for symbols")
+    parser.add_argument("--base-address", help="Image base of the executable image to map for remote analysis")
+    parser.add_argument("-A", action='store_true', help="Upload and Analyse a new binary")
+    parser.add_argument("-u", "--upload", action='store_true', help="Upload a new binary to remote server")
     parser.add_argument("-n", "--ann", action='store_true', help="Fetch Approximate Nearest Neighbours (ANNs) for embedding")
     parser.add_argument("--embedding", help="Path of JSON file containing a BinNet embedding")
     parser.add_argument("--nns", default="5", help="Number of approximate nearest neighbors to fetch")
+    parser.add_argument("--collections", default=None, help="Regex string to select RevEng.AI collections for filtering e.g., libc")
     parser.add_argument("--found-in", help="ANN flag to limit to embeddings returned to those found in specific binary")
-    # parser.add_argument("-m", "--model", default="BinNet", help="AI model used to generate embeddings")
+    parser.add_argument("--from-file", help="ANN flag to limit to embeddings returned to those found in JSON embeddings file")
+    parser.add_argument("-c", "--cves", action="store_true", help="Check for CVEs found inside binary")
+    parser.add_argument("-C", "--sca", action="store_true", help="Perform Software Composition Anaysis to identify common libraries embedded in binary")
+    parser.add_argument("-m", "--model", default="binnet-0.1", help="AI model used to generate embeddings")
     parser.add_argument("-x", "--extract", action='store_true', help="Fetch embeddings for binary")
+    parser.add_argument("--start-address", help="Start vaddr of the function to extract embeddings")
+    parser.add_argument("--end-address", help="End vaddr of the function to extract embeddings")
+    parser.add_argument("-s", "--summary", action='store_true', help="Average symbol embeddings in binary")
+    parser.add_argument("-S", "--signature", action='store_true', help="Generate a RevEng.AI binary signature")
     parser.add_argument("-l", "--logs", action='store_true', help="Fetch analysis log file for binary")
-    parser.add_argument("-d", "--delete", action='store_true', help="Securely delete all analyses and metadata associated with binary")
+    parser.add_argument("-d", "--delete", action='store_true', help="Delete all metadata associated with binary")
     parser.add_argument("-k", "--apikey", help="RevEng.AI API key")
-    parser.add_argument("-s", "--host", help="Analysis Host. Defaults to https://api.reveng.ai")
+    parser.add_argument("-h", "--host", help="Analysis Host (https://api.reveng.ai)")
     parser.add_argument("-v", "--version", action="store_true", help="Display version information")
+    parser.add_argument("--help", action="help", default=argparse.SUPPRESS, help=argparse._('Show this help message and exit'))
     args = parser.parse_args()
 
     if args.apikey:
         re_conf['apikey'] = args.apikey
     if args.host:
         re_conf['host'] = args.host
+    if args.model:
+        re_conf['model'] = args.model
 
     # display version and exit
     if args.version:
         version()
         exit(0)
 
-    if args.analyse or args.extract or args.logs or args.delete:
+    if args.A or args.analyse or args.extract or args.logs or args.delete or args.summary or args.upload:
         # verify binary is a file
         if not os.path.isfile(args.binary):
             print("[!] Error, please supply a valid binary file using '-b'.")
             parser.print_help()
             exit(-1)
 
+    if args.upload:
+        # upload binary first, them carry out actions
+        print(f"[!] RE:upload not implemented. Use analyse.")
+        exit(-1)
+
+    if args.A:
+        RE_analyse(args.binary)
+
     if args.analyse:
         RE_analyse(args.binary)
 
@@ -197,6 +272,12 @@ if __name__ == '__main__':
         embeddings = RE_embeddings(args.binary)
         print_json(data=embeddings)
 
+    elif args.summary:
+        # Arithetic mean of symbol embeddings
+        embeddings = RE_embeddings(args.binary)
+        b_embed = mean(vstack(list(map(lambda x: array(x['embedding']), embeddings))), axis=0)
+        print_json(data=b_embed.tolist())
+
     elif args.ann:
         source = None
         # parse embedding json file
@@ -207,15 +288,32 @@ if __name__ == '__main__':
 
         embedding = json.loads(open(args.embedding, 'r').read())
 
+        # check for valid regex
+        if args.collections:
+            try:
+                re.compile(args.collections)
+            except re.error as e:
+                print(f"[!] Error, invalid regex for collections - {args.collections}")
+                exit(-1)
+
         if args.found_in:
             if not os.path.isfile(args.found_in):
                 print("[!] Error, --found-in flag requires a path to a binary to search from")
                 exit(-1)
             print(f"[+] Searching for symbols similar to embedding in binary {args.found_in}")
-            res = RE_compute_distance(embedding, args.found_in, int(args.nns))
+            embeddings = RE_embeddings(args.found_in)
+            res = RE_compute_distance(embedding, embeddings, int(args.nns))
+            print_json(data=res)
+        elif args.from_file:
+            if not os.path.isfile(args.from_file):
+                print("[!] Error, --from-file flag requires a path to a JSON embeddings file")
+                exit(-1)
+            print(f"[+] Searching for symbols similar to embedding in binary {args.from_file}")
+            res = RE_compute_distance(embedding, json.load(open(args.from_file, "r")), int(args.nns))
             print_json(data=res)
         else:
-            RE_nearest_symbols(embedding, int(args.nns))
+            print(f"[+] Searching for similar symbols to embedding in {'all' if not args.collections else args.collections} collections.")
+            RE_nearest_symbols(embedding, int(args.nns), collections=args.collections)
 
     elif args.logs:
         RE_logs(args.binary)
@@ -223,7 +321,16 @@ if __name__ == '__main__':
     elif args.delete:
         RE_delete(args.binary)
 
+    elif args.cves:
+        RE_cves(args.binary)
+    elif args.signature:
+        print(f"[!] Error, feature not available yet")
+        exit(-1)
+
     else:
-        print("[!] Error, please suply an action command")
+        print("[!] Error, please supply an action command")
         parser.print_help()
 
+
+if __name__ == '__main__':
+    main()

{reait-0.0.13.dist-info → reait-0.0.15.dist-info}/METADATA

@@ -1,7 +1,8 @@
 Metadata-Version: 2.1
 Name: reait
-Version: 0.0.13
+Version: 0.0.15
 Home-page: https://github.com/RevEng-AI/reait
+Author: James Patrick-Evans
 Project-URL: Homepage, https://github.com/RevEng-AI/reait
 Project-URL: Bug Tracker, https://github.com/RevEng-AI/reait/issues
 Project-URL: Organisation Homepage, https://reveng.ai
@@ -22,9 +23,10 @@ Requires-Dist: pandas
 Requires-Dist: numpy
 
 # reait
-RevEng.AI Toolkit
 
-Analyse compiled executable binaries using the RevEng.AI API. This tool allows you to search for similar components across different compiled executable programs. More details about the API can be found at [docs.reveng.ai](https://docs.reveng.ai).
+## <ins>R</ins>ev<ins>E</ins>ng.<ins>AI</ins> <ins>T</ins>oolkit
+
+Analyse compiled executable binaries using the RevEng.AI API. This tool allows you to search for similar components across different compiled executable programs, identify known vulnerabilities in stripped executables, and generate "YARA-like" AI signatures for entire binary files. More details about the API can be found at [docs.reveng.ai](https://docs.reveng.ai).
 
 NB: We are in Alpha. We support GNU/Linux ELF and Windows PE executables for x86_64, and focus our support for x86_64 Linux ELF executables. 
 
@@ -57,20 +59,52 @@ Once an analysis is complete, you may access RevEng.AI's BinNet embeddings for a
 `reait -b /usr/bin/true -x | jq ".[] | select(.vaddr==$((0x19f0))).embedding" > embedding.json`
 
 
-### Search for similar symbols based on JSON embedding file
-To query our database of similar symbols based on an embedding, use `-n` to search using Approximate Nearest Neighbours. The `--nns` allows you to specify the number of results returned. A list of symbol names and the distance between each vector is returned. 
+### Search for similar symbols using an embedding
+To query our database of similar symbols based on an embedding, use `-n` to search using Approximate Nearest Neighbours. The `--nns` allows you to specify the number of results returned. A list of symbols with their names, distance (similarity), RevEng.AI collection set, source code filename, source code line number, and file creation timestamp is returned. 
 
 `reait -e embedding.json -n`
 
 NB: A smaller distance indicates a higher degree of similarity.
 
-#### Limited Search
-To search for the most similar symbols found in a binary to a specific embedding, use the `--found-in` option with a path to the executable.
+#### Specific Search
+To search for the most similar symbols found in a specific binary, use the `--found-in` option with a path to the executable to search from.
 
 `reait -n --embedding /tmp/sha256_init.json --found-in ~/malware.exe --nns 5` 
 
 This downloads embeddings from `malware.exe` and computes the cosine similarity between all symbols and `sha256_init.json`. The returned results lists the most similar symbol locations by cosine similarity score (1.0 most similar, -1.0 dissimilar).
 
+The `--from-file` option may also be used to limit the search to a custom file containing a JSON list of embeddings.
+
+
+#### Limited Search
+To search for most similar symbols from a set of RevEng.AI collections, use the `--collections` options with a RegEx to match collection names. For example:
+
+`reait -n --embedding my_func.json --collections "(libc.*|lib.*crypt.*)"`
+
+RevEng.AI collections are sets of pre-analysed executable objects. To create custom collection sets e.g., malware collections, please create a RevEng.AI account.
+
+### RevEng.AI embedding models
+To use specific RevEng.AI AI models, or for training custom models, use `-m` to specify the model. The default option is to use the latest development model. Available models are `binnet-0.1` and `dexter`.
+
+`reait -b /usr/bin/true -m dexter -a`
+
+### Software Composition Analysis
+To identify known open source software components embedded inside a binary, use the `-C` flag.
+
+#### Stripped Binary CVE Checker
+To check for known vulnerabilities found with embedded software components, use `-c` or `--cves`.
+
+
+### RevEng.AI Binary Signature
+To generate an AI functional description of an entire binary file, use the `-S` flag. NB: Under development.
+
+
+### Binary embedding
+Produce a dumb fingerprint for the whole binary by calculating the arithmetic mean of all symbol embeddings.
+
+`reait -b /usr/bin/true -s`
+
+
 
 ## Configuration
 
@@ -79,6 +113,7 @@ This downloads embeddings from `malware.exe` and computes the cosine similarity
 ```
 apikey = "l1br3"
 host = "https://api.reveng.ai"
+model = "binnet-0.1"
 ```
 
 ## Contact

reait-0.0.15.dist-info/RECORD

@@ -0,0 +1,9 @@
+reait/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reait/__main__.py,sha256=HeD5Pg7_fWaHH7JZ241t3bod_8Gs0CfihuwU3-qOloc,12768
+tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reait-0.0.15.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+reait-0.0.15.dist-info/METADATA,sha256=7rwUG5WAvUFhEBPrOgVul7ci0kVWaPWd82npjcVknOQ,5261
+reait-0.0.15.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+reait-0.0.15.dist-info/entry_points.txt,sha256=OgGvIbjDbTIqotTSK2Gi1D_CJIVWzOjs1Hl3T-za-uE,46
+reait-0.0.15.dist-info/top_level.txt,sha256=AXxC4J_UsjYtyxWQSPi9r4ygoKIXz6u94cW2bdgw714,12
+reait-0.0.15.dist-info/RECORD,,

{reait-0.0.13.dist-info → reait-0.0.15.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.38.4)
+Generator: bdist_wheel (0.40.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

reait-0.0.15.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+reait = reait.__main__:main

reait-0.0.15.dist-info/top_level.txt

@@ -0,0 +1,2 @@
+reait
+tests

reait-0.0.13.dist-info/RECORD

@@ -1,6 +0,0 @@
-reait-0.0.13.data/scripts/reait,sha256=1LLJ0T4oeUFJtcSy-Ocxifuy3KSPbKH8YwLdWoZVC8c,7830
-reait-0.0.13.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-reait-0.0.13.dist-info/METADATA,sha256=e8S2y484-mC0nEXphurKRqoN1Yj1K9_xJyu0MPhZbQk,3575
-reait-0.0.13.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
-reait-0.0.13.dist-info/top_level.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-reait-0.0.13.dist-info/RECORD,,

reait-0.0.13.dist-info/top_level.txt

@@ -1 +0,0 @@
-