pyroclastic 3.1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyroclastic-3.1.4.dist-info/METADATA +156 -0
- pyroclastic-3.1.4.dist-info/RECORD +29 -0
- pyroclastic-3.1.4.dist-info/WHEEL +5 -0
- pyroclastic-3.1.4.dist-info/licenses/LICENSE +9 -0
- pyroclastic-3.1.4.dist-info/top_level.txt +1 -0
- scripts/add_queries.sh +231 -0
- scripts/arg_parse.sh +183 -0
- scripts/bug_mining.py +252 -0
- scripts/database_types.py +745 -0
- scripts/docker-shell.sh +66 -0
- scripts/find_bug_injection.py +1041 -0
- scripts/fninstr.py +310 -0
- scripts/funcs.sh +116 -0
- scripts/generate_schema.py +358 -0
- scripts/get_c_files.py +61 -0
- scripts/inject.py +204 -0
- scripts/lava.py +905 -0
- scripts/lava.sh +258 -0
- scripts/lava.sql +246 -0
- scripts/preprocess.py +187 -0
- scripts/process_compile_commands.py +42 -0
- scripts/reset_db.sh +46 -0
- scripts/setup_postgres.sh +46 -0
- scripts/shell.sh +41 -0
- scripts/sw-btrace +40 -0
- scripts/sw-btrace-to-compiledb +169 -0
- scripts/test_crash.py +78 -0
- scripts/vars.py +130 -0
- scripts/vars.sh +90 -0
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pyroclastic
|
|
3
|
+
Version: 3.1.4
|
|
4
|
+
Summary: Python Interface for LAVA
|
|
5
|
+
Author-email: Andrew Quijano <andrew.quijano@nyu.edu>
|
|
6
|
+
License-Expression: GPL-2.0-only
|
|
7
|
+
Project-URL: Homepage, https://github.com/panda-re/lava
|
|
8
|
+
Project-URL: Source, https://github.com/panda-re/lava
|
|
9
|
+
Project-URL: Tracker, https://github.com/panda-re/lava/issues
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Intended Audience :: Developers
|
|
17
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
18
|
+
Requires-Python: >=3.10
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Requires-Dist: dotenv
|
|
22
|
+
Requires-Dist: colorama
|
|
23
|
+
Requires-Dist: db
|
|
24
|
+
Requires-Dist: ipython
|
|
25
|
+
Requires-Dist: lockfile
|
|
26
|
+
Requires-Dist: numpy
|
|
27
|
+
Requires-Dist: pandare==1.8.78
|
|
28
|
+
Requires-Dist: PyYAML
|
|
29
|
+
Requires-Dist: SQLAlchemy
|
|
30
|
+
Requires-Dist: tabulate
|
|
31
|
+
Requires-Dist: psycopg2-binary
|
|
32
|
+
Requires-Dist: ijson
|
|
33
|
+
Dynamic: license-file
|
|
34
|
+
|
|
35
|
+
# LAVA: Large Scale Automated Vulnerability Addition
|
|
36
|
+
|
|
37
|
+
[](https://github.com/panda-re/lava/actions/workflows/publish_docker.yml)
|
|
38
|
+
|
|
39
|
+
[](https://github.com/panda-re/lava/actions/workflows/parallel_tests.yml)
|
|
40
|
+
|
|
41
|
+
Evaluating and improving bug-finding tools is currently difficult due to
|
|
42
|
+
a shortage of ground truth corpora (i.e., software that has known bugs
|
|
43
|
+
with triggering inputs). LAVA attempts to solve this problem by
|
|
44
|
+
automatically injecting bugs into software. Every LAVA bug is
|
|
45
|
+
accompanied by an input that triggers it whereas normal inputs are
|
|
46
|
+
extremely unlikely to do so. These vulnerabilities are synthetic but, we
|
|
47
|
+
argue, still realistic, in the sense that they are embedded deep within
|
|
48
|
+
programs and are triggered by real inputs. Our work forms the basis of
|
|
49
|
+
an approach for generating large ground-truth vulnerability corpora on
|
|
50
|
+
demand, enabling rigorous tool evaluation and providing a high-quality
|
|
51
|
+
target for tool developers.
|
|
52
|
+
|
|
53
|
+
LAVA is the product of a collaboration between MIT Lincoln Laboratory,
|
|
54
|
+
NYU, and Northeastern University.
|
|
55
|
+
|
|
56
|
+
# Quick Start
|
|
57
|
+
|
|
58
|
+
## Docker
|
|
59
|
+
The latest version of LAVA's `master` branch is automatically built as a docker images based on Ubuntu 22.04 and published to [Docker Hub](https://hub.docker.com/r/pandare/lava). Most users will want to use the `lava` container which has PANDA and LAVA installed along with their runtime dependencies, but no build artifacts or source code to reduce the size of the container.
|
|
60
|
+
|
|
61
|
+
To use the `lava` container you can pull it from Docker Hub:
|
|
62
|
+
```
|
|
63
|
+
$ docker pull pandare/lava
|
|
64
|
+
```
|
|
65
|
+
Or build from this repository:
|
|
66
|
+
```
|
|
67
|
+
$ DOCKER_BUILDKIT=1 docker build lava .
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
## Ubuntu, Debian
|
|
71
|
+
On a system running Ubuntu 22.04, you should be able to just run `bash install.sh`. Note that this [install script](./install.sh) will install packages and make changes to your system.
|
|
72
|
+
|
|
73
|
+
## Final steps
|
|
74
|
+
|
|
75
|
+
### Utilizing host.json
|
|
76
|
+
Next, run `init-host.py` to generate a `host.json`.
|
|
77
|
+
This file is used by LAVA to store settings specific
|
|
78
|
+
to your machine. You can edit these settings as necessary, but the default
|
|
79
|
+
values should work, see [vars.sh](scripts/vars.sh).
|
|
80
|
+
|
|
81
|
+
A few values to keep in mind are the following:
|
|
82
|
+
* **buildhost** This is the location of where LAVA is being executed from. Currently, it defaults to `localhost`
|
|
83
|
+
* **docker** is the name of the docker image to use that has the LAVA binaries. Currently it defaults to `lava32`, but you can switch this to `pandare/lava`
|
|
84
|
+
* **pguser** This is the name of database user, currently defaults to `postgres`
|
|
85
|
+
* **pgpass** This is the password of the database user, currently defaults to `postgrespostgres`
|
|
86
|
+
* **host** is the name of the Postgres SQL database with all the LAVA bugs. Currently it defaults to `database`, although if you installed LAVA locally, you likely should change this to `localhost`
|
|
87
|
+
|
|
88
|
+
### Project configurations
|
|
89
|
+
Project configurations are located in the `target_configs` directory, where
|
|
90
|
+
every configuration is located at `target_configs/projectname/projectname.json`.
|
|
91
|
+
Paths specified within these configuration files are relative to values set
|
|
92
|
+
in your `host.json` file.
|
|
93
|
+
|
|
94
|
+
### Setting up postgres SQL database
|
|
95
|
+
As alluded to, you should create a Postgres SQL user. You can use a script to [use default credentials](scripts/setup_postgres.sh) for the following:
|
|
96
|
+
* Create the user with default password
|
|
97
|
+
* Update Postgres SQL database on host to accept traffic from external sources (e. g. LAVA Docker container)
|
|
98
|
+
* Switch password encryption to md5 (Do we need this?)
|
|
99
|
+
|
|
100
|
+
# Usage
|
|
101
|
+
|
|
102
|
+
Finally, you can run `./scripts/lava.sh` to actually inject bugs into a program. Just provide the name of a project that is in the `target_configs` directory, for example:
|
|
103
|
+
|
|
104
|
+
```
|
|
105
|
+
./scripts/lava.sh toy
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
You should now have a buggy copy of toy!
|
|
109
|
+
|
|
110
|
+
If you want to inject bugs into a new target, you will likely need to make some
|
|
111
|
+
modifications. Check out [How-to-Lava](docs/how-to-lava.md) for guidance.
|
|
112
|
+
|
|
113
|
+
# Documentation
|
|
114
|
+
Check out the [docs](docs/) folder to get started.
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
# Current Status
|
|
118
|
+
## Version 2.0.0
|
|
119
|
+
|
|
120
|
+
Expected results from test suite:
|
|
121
|
+
```
|
|
122
|
+
Project RESET CLEAN ADD MAKE TAINT INJECT COMP
|
|
123
|
+
blecho PASS PASS PASS PASS PASS PASS PASS
|
|
124
|
+
libyaml PASS PASS PASS PASS PASS PASS PASS
|
|
125
|
+
file PASS PASS PASS PASS PASS PASS PASS
|
|
126
|
+
toy PASS PASS PASS PASS PASS PASS PASS
|
|
127
|
+
pcre2 PASS PASS PASS PASS PASS PASS PASS
|
|
128
|
+
jq PASS PASS PASS PASS PASS PASS PASS
|
|
129
|
+
grep PASS PASS PASS PASS PASS FAIL
|
|
130
|
+
libjpeg PASS PASS PASS PASS FAIL
|
|
131
|
+
tinyexpr PASS PASS PASS PASS FAIL
|
|
132
|
+
duktape PASS PASS PASS FAIL
|
|
133
|
+
tweetNaCl PASS PASS FAIL
|
|
134
|
+
gzip FAIL
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
# Authors
|
|
138
|
+
|
|
139
|
+
LAVA is the result of several years of development by many people; a
|
|
140
|
+
partial (alphabetical) list of contributors is below:
|
|
141
|
+
|
|
142
|
+
* Andy Davis
|
|
143
|
+
* Brendan Dolan-Gavitt
|
|
144
|
+
* Andrew Fasano
|
|
145
|
+
* Zhenghao Hu
|
|
146
|
+
* Patrick Hulin
|
|
147
|
+
* Amy Jiang
|
|
148
|
+
* Engin Kirda
|
|
149
|
+
* Tim Leek
|
|
150
|
+
* Andrea Mambretti
|
|
151
|
+
* Andrew Quijano
|
|
152
|
+
* Wil Robertson
|
|
153
|
+
* Aaron Sedlacek
|
|
154
|
+
* Rahul Sridhar
|
|
155
|
+
* Frederick Ulrich
|
|
156
|
+
* Ryan Whelan
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
pyroclastic-3.1.4.dist-info/licenses/LICENSE,sha256=6wat89yFXtw45OVk6X18KOl3VfZa2HuertwM-pczDM4,773
|
|
2
|
+
scripts/add_queries.sh,sha256=agMzIbTCOOXrVIwHjvqJFdgQeJrUDBPFaSiS9Dvxh0c,7707
|
|
3
|
+
scripts/arg_parse.sh,sha256=AeLnwKPay71p9CYWOl8cITWR70udBjXPEC_HnVq50GA,6313
|
|
4
|
+
scripts/bug_mining.py,sha256=7lRo4MbSPkX9259DQs_1jYZRPD7xchxpDSOl6-l38dc,7205
|
|
5
|
+
scripts/database_types.py,sha256=rzf_6-jdzFcE1d7kRukXzQ25CwAV0ccZ99KSOyqB8X8,26994
|
|
6
|
+
scripts/docker-shell.sh,sha256=qWWV8rE35lp3JZQOXlYizwzA490hMLtIHN177b0J7N8,1966
|
|
7
|
+
scripts/find_bug_injection.py,sha256=fkumA6omN5bX3E1nfWLTRmuPP_iK-rOF1x9UfZKZYq8,37980
|
|
8
|
+
scripts/fninstr.py,sha256=HVJK-s5d_miRZ14FBJZm3Hy4MQayMCIzar_aB8CNN_U,9730
|
|
9
|
+
scripts/funcs.sh,sha256=tJ3afatYss3JNjQ6NkFt-JkjdEUmDZXfJXqa9iLMlFI,3320
|
|
10
|
+
scripts/generate_schema.py,sha256=tNtLHe9rU9oVI-cE6reoAq1TnLALgEm346fa-lrwRtc,16731
|
|
11
|
+
scripts/get_c_files.py,sha256=RASd8RxdVT-IaqqCaDSosgkabrdCRuTBrpyD9wfJJtI,1792
|
|
12
|
+
scripts/inject.py,sha256=6R0JzkYlaQCIrhZvA3l9J_UIt6n3bAGOsOutyy4Fjq8,8040
|
|
13
|
+
scripts/lava.py,sha256=BZQ0H08rpiI60zf4JHb1stQw860_zpowO9ygmOnRTfI,35270
|
|
14
|
+
scripts/lava.sh,sha256=31yIKF4AC6C_ecXZuBLZ4866_coMWJeqJXmdcqDkNs4,8765
|
|
15
|
+
scripts/lava.sql,sha256=UkZw_GoJPJdsT9NPPKtaYxaxqAtEnGpUbEhTkJ6kujM,5966
|
|
16
|
+
scripts/preprocess.py,sha256=KZejoxkMqGOWbeJC3Xy_YkM7nmyUE9l-ByMt9oOV5uI,6992
|
|
17
|
+
scripts/process_compile_commands.py,sha256=I8dh1yorLyuPzKgzOiadNB-_HqX1Z-FeiTwAQfLTP5M,1312
|
|
18
|
+
scripts/reset_db.sh,sha256=88zw5fBJsGFXqKMUs-gl2c-ERYlhk4jmmZdbDNxIyHw,1046
|
|
19
|
+
scripts/setup_postgres.sh,sha256=C6002AQqPCmT0iVYRSOH1l6WZnUsdmp3SGOWyu0swNk,1436
|
|
20
|
+
scripts/shell.sh,sha256=5r-3zTssC4-GOIPYmSxdWPT4gyAf-QfUaBt_jSK_Dp8,1297
|
|
21
|
+
scripts/sw-btrace,sha256=CyLyA_kTgLqV3Y-nyWN1EzgmMniaeBEp_FGUxClJsTQ,1493
|
|
22
|
+
scripts/sw-btrace-to-compiledb,sha256=-u6-7PDAbGlFOL5CyJd1IwT3Mtumv_iYelb4X-KVitE,5121
|
|
23
|
+
scripts/test_crash.py,sha256=or2BPwXaSv8EGNDi0TE2tfF8HXckpHBk60P-tR6zlsQ,2221
|
|
24
|
+
scripts/vars.py,sha256=qWCIT-xKBUmrda-eM3EHkjLBLa3CErce-LxI7lREFr4,4581
|
|
25
|
+
scripts/vars.sh,sha256=7sm2l2iZDOsGJddxaRwAQidMB2BpYSqc3I2kO3Tlhxo,3024
|
|
26
|
+
pyroclastic-3.1.4.dist-info/METADATA,sha256=vXs5Km6672cMi2OfleNsDpg4Bz_TiMdzM_LR5gfJUIA,6553
|
|
27
|
+
pyroclastic-3.1.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
28
|
+
pyroclastic-3.1.4.dist-info/top_level.txt,sha256=rmzd5mewlrJy4sT608KPib7sM7edoY75AeqJeY3SPB4,8
|
|
29
|
+
pyroclastic-3.1.4.dist-info/RECORD,,
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
SPDX-License-Identifier: GPL-2.0
|
|
2
|
+
|
|
3
|
+
Copyright (C) 2016, MASSACHUSETTS INSTITUTE OF TECHNOLOGY
|
|
4
|
+
|
|
5
|
+
This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version.
|
|
6
|
+
|
|
7
|
+
This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
|
|
8
|
+
|
|
9
|
+
You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
scripts
|
scripts/add_queries.sh
ADDED
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# Query insertion script.
|
|
3
|
+
#
|
|
4
|
+
# Takes one argument, the project name
|
|
5
|
+
# That json file must contain all of the following
|
|
6
|
+
#
|
|
7
|
+
# name name for project, usually the name of the software (binutils-2.25, openssh-2.1, etc)
|
|
8
|
+
# directory directory in which src-2-src query injection will occur -- should be somewhere on the nas
|
|
9
|
+
# tarfile path to software tar file
|
|
10
|
+
# configure how to configure the software (./configure plus arguments) (will just use /bin/true if not present)
|
|
11
|
+
# make how to make the software (make might have args or might have FOO=bar required precursors)
|
|
12
|
+
# install how to install the software (note that configure will be run with --prefix ...lava-install)
|
|
13
|
+
#
|
|
14
|
+
# script proceeds to untar the software, run btrace on it to extract a compile_commands.json file,
|
|
15
|
+
# which contains all information needed to compile every file in the project.
|
|
16
|
+
# then, the script runs lavaTool using that compile_commands.json file, on every source file,
|
|
17
|
+
# adding extra source code to perform taint queries. At the time of this writing, the taint
|
|
18
|
+
# queries were for every argument of every fn call, injected both before and after the call.
|
|
19
|
+
# Also, the return value of the fn is queried. Oh, and lavaTool also injects "queries" that
|
|
20
|
+
# indicate when a potential attack point has been encountered. At the time of this writing,
|
|
21
|
+
# that includes calls to memcpy and malloc.
|
|
22
|
+
#
|
|
23
|
+
# After lavaTool has transformed this source, it exits. You should now try to make the project
|
|
24
|
+
# and deal with any complaints (often src-to-src breaks the code a little). Once you have a working
|
|
25
|
+
# version of the compiled exec with queries you will need to log on to a 64-bit machine
|
|
26
|
+
# and run the bug_mining.py script (which uses PANDA to trace taint).
|
|
27
|
+
#
|
|
28
|
+
|
|
29
|
+
tick
|
|
30
|
+
|
|
31
|
+
USAGE() {
|
|
32
|
+
echo "Usage: $1 [ATP_Type] JSONfile"
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
set -e # Exit on error
|
|
36
|
+
# set -x # Debug mode
|
|
37
|
+
|
|
38
|
+
if [ $# -lt 1 ]; then
|
|
39
|
+
USAGE $0
|
|
40
|
+
elif [ $# -lt 2 ]; then
|
|
41
|
+
echo "No ATP_Type specified. Defaulting to all."
|
|
42
|
+
ATP_TYPE=""
|
|
43
|
+
json="$(readlink -f $1)"
|
|
44
|
+
elif [ $# -eq 2 ]; then
|
|
45
|
+
ATP_TYPE="-$1"
|
|
46
|
+
json="$(readlink -f $2)"
|
|
47
|
+
else
|
|
48
|
+
USAGE $0
|
|
49
|
+
exit 1
|
|
50
|
+
fi
|
|
51
|
+
|
|
52
|
+
absolute_path=$(readlink -f "$0")
|
|
53
|
+
scripts_path=$(dirname "$absolute_path")
|
|
54
|
+
lava=$(dirname "$scripts_path")
|
|
55
|
+
project_name="$1"
|
|
56
|
+
|
|
57
|
+
# Load lava-functions and vars
|
|
58
|
+
source "$scripts_path/funcs.sh"
|
|
59
|
+
source "$scripts_path/vars.sh"
|
|
60
|
+
|
|
61
|
+
progress "queries" 0 "Entering $directory/$name."
|
|
62
|
+
mkdir -p "$directory/$name"
|
|
63
|
+
cd "$directory/$name"
|
|
64
|
+
|
|
65
|
+
progress "queries" 0 "Untarring $tarfile..."
|
|
66
|
+
source=$(tar tf "$tarfile" | head -n 1 | cut -d / -f 1)
|
|
67
|
+
|
|
68
|
+
if [ -e "$source" ]; then
|
|
69
|
+
progress "queries" 0 "Deleting $directory/$name/$source..."
|
|
70
|
+
rm -rf "$directory/$name/$source"
|
|
71
|
+
fi
|
|
72
|
+
tar xf "$tarfile"
|
|
73
|
+
|
|
74
|
+
progress "queries" 0 "Entering $source."
|
|
75
|
+
cd "$source"
|
|
76
|
+
|
|
77
|
+
progress "queries" 0 "Creating git repo."
|
|
78
|
+
rm -rf .git || true #Remove any existing git repo
|
|
79
|
+
git init
|
|
80
|
+
git config user.name LAVA
|
|
81
|
+
git config user.email "nobody@nowhere"
|
|
82
|
+
git add -A .
|
|
83
|
+
git commit -m 'Unmodified source.'
|
|
84
|
+
|
|
85
|
+
progress "queries" 0 "Configuring..."
|
|
86
|
+
mkdir -p lava-install
|
|
87
|
+
configure_file=${configure_cmd%% *}
|
|
88
|
+
if [ -e "$configure_file" ]; then
|
|
89
|
+
CC=$llvm/bin/clang \
|
|
90
|
+
CXX=$llvm/bin/clang++ \
|
|
91
|
+
CFLAGS="-O0 -DHAVE_CONFIG_H -g -gdwarf-2 -fno-stack-protector -D_FORTIFY_SOURCE=0 -I. -I.. -I../include -I./src/" \
|
|
92
|
+
$configure_cmd --prefix=$(pwd)/lava-install
|
|
93
|
+
fi
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
progress "queries" 0 "Making with btrace..."
|
|
97
|
+
# Delete any pre-existing btrace.log (could be in archive by mistake)
|
|
98
|
+
rm -f btrace.log
|
|
99
|
+
ORIGIN_IFS=$IFS
|
|
100
|
+
IFS='&&'
|
|
101
|
+
read -ra MAKES <<< "$makecmd"
|
|
102
|
+
for i in "${MAKES[@]}"; do
|
|
103
|
+
IFS=' '
|
|
104
|
+
read -ra ARGS <<< "$i"
|
|
105
|
+
echo "$scripts_path/sw-btrace ${ARGS[@]}"
|
|
106
|
+
CC=$llvm/bin/clang \
|
|
107
|
+
CXX=$llvm/bin/clang++ \
|
|
108
|
+
CFLAGS="-O0 -DHAVE_CONFIG_H -g -gdwarf-2 -fno-stack-protector -D_FORTIFY_SOURCE=0 -I. -I.. -I../include -I./src/" \
|
|
109
|
+
"$scripts_path/sw-btrace" "${ARGS[@]}"
|
|
110
|
+
IFS='&&'
|
|
111
|
+
done
|
|
112
|
+
IFS=$ORIGIN_IFS
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
progress "queries" 0 "Installing..."
|
|
116
|
+
bash -c $install
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
progress "queries" 0 "Creating compile_commands.json..."
|
|
120
|
+
# Delete any pre-existing compile commands.json (could be in archive by mistake)
|
|
121
|
+
rm -f compile_commands.json
|
|
122
|
+
"$scripts_path/sw-btrace-to-compiledb" $llvm/lib/clang/11/include
|
|
123
|
+
if [ -e "$directory/$name/extra_compile_commands.json" ]; then
|
|
124
|
+
sed -i '$d' compile_commands.json
|
|
125
|
+
echo "," >> compile_commands.json
|
|
126
|
+
tail -n +$((2)) "$directory/$name/extra_compile_commands.json" >> compile_commands.json
|
|
127
|
+
fi
|
|
128
|
+
git add compile_commands.json
|
|
129
|
+
git commit -m 'Add compile_commands.json.'
|
|
130
|
+
|
|
131
|
+
cd ..
|
|
132
|
+
|
|
133
|
+
# Switching IFS to '\n' to support paths with spaces in them.
|
|
134
|
+
c_files=$($python "$scripts_path/get_c_files.py" "$source")
|
|
135
|
+
IFS=$'\n'
|
|
136
|
+
c_dirs=$(for i in $c_files; do dirname "$i"; done | sort | uniq)
|
|
137
|
+
|
|
138
|
+
progress "queries" 0 "Copying include files..."
|
|
139
|
+
for i in $c_dirs; do
|
|
140
|
+
echo " $i"
|
|
141
|
+
if [ -d "$i" ]; then
|
|
142
|
+
cp "$lava"/tools/include/*.h "$i"/
|
|
143
|
+
fi
|
|
144
|
+
done
|
|
145
|
+
|
|
146
|
+
# Run another clang tool that provides information about functions,
|
|
147
|
+
# i.e., which have only prototypes, which have bodies.
|
|
148
|
+
progress "queries" 0 "Figure out functions"
|
|
149
|
+
for this_c_file in $c_files; do
|
|
150
|
+
"$lava/tools/install/bin/lavaFnTool" "$this_c_file"
|
|
151
|
+
done
|
|
152
|
+
|
|
153
|
+
#progress "queries" 0 "Initialize variables..."
|
|
154
|
+
#for i in $c_files; do
|
|
155
|
+
# /src_clang/build/lavaTool -action=init \
|
|
156
|
+
# -p="$source/compile_commands.json" \
|
|
157
|
+
# -src-prefix=$(readlink -f "$source") \
|
|
158
|
+
# $i
|
|
159
|
+
#done
|
|
160
|
+
|
|
161
|
+
# TODO: This should probably be just for dataflow
|
|
162
|
+
# but we still need it for non-dataflow targets, otherwise we inject into
|
|
163
|
+
# va_args functions and everything breask
|
|
164
|
+
|
|
165
|
+
# Analyze that output and figure out
|
|
166
|
+
fnfiles=$(echo $c_files | sed 's/\.c/\.c\.fn/g')
|
|
167
|
+
fninstr=$directory/$name/fninstr
|
|
168
|
+
|
|
169
|
+
echo "Creating fninstr [$fninstr]"
|
|
170
|
+
echo -e "\twith command: \"python $lava/scripts/fninstr.py -d -o $fninstr $fnfiles\""
|
|
171
|
+
$python "$scripts_path/fninstr.py" -d -o $fninstr $fnfiles
|
|
172
|
+
|
|
173
|
+
if [[ ! -z "$df_fn_blacklist" ]]; then
|
|
174
|
+
cmd=$(echo "sed -i /${df_fn_blacklist}/d $fninstr")
|
|
175
|
+
echo "Removing blacklisted functions with regex: $df_fn_blacklist"
|
|
176
|
+
$cmd
|
|
177
|
+
fi
|
|
178
|
+
|
|
179
|
+
if [ "$dataflow" = "true" ]; then
|
|
180
|
+
# Insert queries with DF - could merge this with the else if logic below instead of duplicating
|
|
181
|
+
# TODO: Just make lavaTool load dataflow from project.json instead of passing as CLI arg.
|
|
182
|
+
# Since it's okay to pass the whitelist either way
|
|
183
|
+
progress "queries" 0 "Inserting queries for dataflow"
|
|
184
|
+
for i in $c_files; do
|
|
185
|
+
"$lava/tools/install/bin/lavaTool" -action=query \
|
|
186
|
+
-lava-db="$directory/$name/lavadb" \
|
|
187
|
+
-p="$directory/$name/$source/compile_commands.json" \
|
|
188
|
+
-arg_dataflow \
|
|
189
|
+
-lava-wl="$fninstr" \
|
|
190
|
+
-src-prefix=$(readlink -f "$source") \
|
|
191
|
+
$ATP_TYPE \
|
|
192
|
+
-db="$db" \
|
|
193
|
+
$i
|
|
194
|
+
done
|
|
195
|
+
else
|
|
196
|
+
progress "queries" 0 "Inserting queries..."
|
|
197
|
+
# TODO: remove lava-wl here, unless we're using it to limit where we inject
|
|
198
|
+
for i in $c_files; do
|
|
199
|
+
"$lava/tools/install/bin/lavaTool" -action=query \
|
|
200
|
+
-lava-db="$directory/$name/lavadb" \
|
|
201
|
+
-lava-wl="$fninstr" \
|
|
202
|
+
-p="$source/compile_commands.json" \
|
|
203
|
+
-src-prefix=$(readlink -f "$source") \
|
|
204
|
+
$ATP_TYPE \
|
|
205
|
+
-db="$db" \
|
|
206
|
+
$i
|
|
207
|
+
done
|
|
208
|
+
fi
|
|
209
|
+
|
|
210
|
+
for i in $c_dirs; do
|
|
211
|
+
echo "Applying replacements to $i"
|
|
212
|
+
pushd $i
|
|
213
|
+
"$llvm/bin/clang-apply-replacements" .
|
|
214
|
+
popd
|
|
215
|
+
done
|
|
216
|
+
|
|
217
|
+
# Ensure every c file was modified
|
|
218
|
+
# Alternatively, we could just check that at least one file was modified
|
|
219
|
+
for this_c_file in $c_files; do
|
|
220
|
+
if ! grep -q "pirate_mark_lava.h" $this_c_file; then
|
|
221
|
+
echo "FATAL ERROR: LAVA queries missing from source files!"
|
|
222
|
+
exit 1
|
|
223
|
+
fi
|
|
224
|
+
done
|
|
225
|
+
unset IFS
|
|
226
|
+
|
|
227
|
+
progress "queries" 0 "Done inserting queries. Time to make and run actuate.py on a 64-BIT machine!"
|
|
228
|
+
|
|
229
|
+
tock
|
|
230
|
+
echo "add queries complete $time_diff seconds"
|
|
231
|
+
|
scripts/arg_parse.sh
ADDED
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
#!/bin/sh
|
|
2
|
+
. `dirname $0`/funcs.sh
|
|
3
|
+
digit_re='^[0-9]+$'
|
|
4
|
+
|
|
5
|
+
function parse_args {
|
|
6
|
+
echo
|
|
7
|
+
progress "everything" 0 "Parsing args"
|
|
8
|
+
if [ "$#" -eq 1 ]; then # With no arguments run everything, else try matching args
|
|
9
|
+
# If the single argument is a flag, e.g., --help, handle that instead
|
|
10
|
+
if ! [[ $1 == *"-"* ]]; then
|
|
11
|
+
reset=1
|
|
12
|
+
reset_db=1
|
|
13
|
+
add_queries=1
|
|
14
|
+
make=1
|
|
15
|
+
taint=1
|
|
16
|
+
inject=1
|
|
17
|
+
num_trials=3
|
|
18
|
+
progress "everything" 0 "All steps will be executed"
|
|
19
|
+
project_name=$1
|
|
20
|
+
return 0
|
|
21
|
+
fi
|
|
22
|
+
fi
|
|
23
|
+
while :; do
|
|
24
|
+
case $1 in
|
|
25
|
+
-h|-\?|--help)
|
|
26
|
+
USAGE
|
|
27
|
+
exit
|
|
28
|
+
;;
|
|
29
|
+
-a|--all)
|
|
30
|
+
reset=1
|
|
31
|
+
reset_db=1
|
|
32
|
+
add_queries=1
|
|
33
|
+
make=1
|
|
34
|
+
taint=1
|
|
35
|
+
inject=1
|
|
36
|
+
num_trials=3
|
|
37
|
+
progress "everything" 0 "All steps will be executed"
|
|
38
|
+
;;
|
|
39
|
+
-k|--force)
|
|
40
|
+
ok=1
|
|
41
|
+
progress "everything" 0 "--force: Forcing through deletes"
|
|
42
|
+
;;
|
|
43
|
+
|
|
44
|
+
-ak) # Backwards compatability with everyone's favorite lava1 option
|
|
45
|
+
reset=1
|
|
46
|
+
reset_db=1
|
|
47
|
+
add_queries=1
|
|
48
|
+
make=1
|
|
49
|
+
taint=1
|
|
50
|
+
inject=1
|
|
51
|
+
num_trials=3
|
|
52
|
+
progress "everything" 0 "All steps will be executed"
|
|
53
|
+
ok=1
|
|
54
|
+
progress "everything" 0 "--force: Forcing through deletes"
|
|
55
|
+
;;
|
|
56
|
+
|
|
57
|
+
# Individual steps
|
|
58
|
+
-r|--reset)
|
|
59
|
+
reset=1
|
|
60
|
+
progress "everything" 0 "Reset step will be executed"
|
|
61
|
+
;;
|
|
62
|
+
-c|--clean)
|
|
63
|
+
reset_db=1
|
|
64
|
+
progress "everything" 0 "Reset (clean) just databse step will be executed"
|
|
65
|
+
;;
|
|
66
|
+
-q|--add-queries)
|
|
67
|
+
add_queries=1
|
|
68
|
+
progress "everything" 0 "Add queries step will be executed"
|
|
69
|
+
;;
|
|
70
|
+
-m|--make)
|
|
71
|
+
make=1 # TODO: what does this mean? Make queries?
|
|
72
|
+
progress "everything" 0 "Make step will be executed"
|
|
73
|
+
;;
|
|
74
|
+
-t|--taint)
|
|
75
|
+
taint=1
|
|
76
|
+
progress "everything" 0 "Taint step will be executed"
|
|
77
|
+
;;
|
|
78
|
+
|
|
79
|
+
# Expert only options- Dev/testing flags that may be broken
|
|
80
|
+
--demo)
|
|
81
|
+
demo=1
|
|
82
|
+
progress "everything" 0 "-d: demo mode"
|
|
83
|
+
;;
|
|
84
|
+
--test-data-flow) # For testing- inject 0 bugs, but add data_flow
|
|
85
|
+
inject=1
|
|
86
|
+
many=0
|
|
87
|
+
num_trials=1
|
|
88
|
+
progress "everything" 0 "[TESTING] Inject data_flow only, 0 bugs"
|
|
89
|
+
;;
|
|
90
|
+
|
|
91
|
+
# Arguments that take options
|
|
92
|
+
-i|--inject)
|
|
93
|
+
if [ "$2" ]; then
|
|
94
|
+
inject=1
|
|
95
|
+
num_trials=$2
|
|
96
|
+
shift
|
|
97
|
+
progress "everything" 0 "Inject step will be executed: num_trials = $num_trials"
|
|
98
|
+
else
|
|
99
|
+
die "ERROR: --inject requires [num_trials], got $@"
|
|
100
|
+
fi
|
|
101
|
+
;;
|
|
102
|
+
-n|--count)
|
|
103
|
+
if [ "$2" ]; then
|
|
104
|
+
many="$2"
|
|
105
|
+
if ! [[ $many =~ $re ]] ; then
|
|
106
|
+
die 'ERROR: --many requires numeric argument'
|
|
107
|
+
fi
|
|
108
|
+
progress "everything" 0 "Number of injected bug at the same time: $many"
|
|
109
|
+
shift
|
|
110
|
+
else
|
|
111
|
+
die 'ERROR: --many requires argument'
|
|
112
|
+
fi
|
|
113
|
+
;;
|
|
114
|
+
|
|
115
|
+
--enable-knob-trigger)
|
|
116
|
+
if [ "$2" ]; then
|
|
117
|
+
knob=$2
|
|
118
|
+
kt="--knobTrigger $knob"
|
|
119
|
+
progress "everything" 0 "Inject step will be executed with knob trigger: knob = $knob"
|
|
120
|
+
shift
|
|
121
|
+
else
|
|
122
|
+
die "--knobTrigger requires knob argument"
|
|
123
|
+
fi
|
|
124
|
+
;;
|
|
125
|
+
--curtail)
|
|
126
|
+
if [ "$2" ] && [[ $2 =~ ^[0-9]+$ ]]; then
|
|
127
|
+
curtail="$2"
|
|
128
|
+
shift
|
|
129
|
+
else
|
|
130
|
+
# default curtail=1000
|
|
131
|
+
curtail=1000
|
|
132
|
+
fi
|
|
133
|
+
progress "everything" 0 "Curtailing FBI after $curtail"
|
|
134
|
+
;;
|
|
135
|
+
-y|--bug-types)
|
|
136
|
+
if [ "$2" ]; then
|
|
137
|
+
bugtypes="$2" # TODO: a single arguments must be passed as 'arg1,'
|
|
138
|
+
progress "everything" 0 "Injecting bugs of type(s): $bugtypes"
|
|
139
|
+
shift
|
|
140
|
+
else
|
|
141
|
+
die 'ERROR: --bug-types requires comma-seperated list of bug types'
|
|
142
|
+
fi
|
|
143
|
+
;;
|
|
144
|
+
-b|--atp-type )
|
|
145
|
+
# -b [bugType] : use this to specify attact point type: [mem_write|mem_read|fn_arg]
|
|
146
|
+
# TODO: should allow combinations of atp types
|
|
147
|
+
if [ "$2" ]; then
|
|
148
|
+
ATP_TYPE="$2"
|
|
149
|
+
else
|
|
150
|
+
die 'ERROR: --atp-types requires a single atp-type'
|
|
151
|
+
fi
|
|
152
|
+
|
|
153
|
+
if [ "$ATP_TYPE" != "mem_read" -a "$ATP_TYPE" != "fn_arg" -a "$ATP_TYPE" != "mem_write" ]; then
|
|
154
|
+
echo "ATP Type ($ATP_TYPE) is not valid must specify:"
|
|
155
|
+
echo " --atp-type [mem_write|mem_read|fn_arg]"
|
|
156
|
+
echo "Exiting . . ."
|
|
157
|
+
exit 1
|
|
158
|
+
fi
|
|
159
|
+
progress "everything" 0 "Query step will be executed with bug type: atp = $ATP_TYPE"
|
|
160
|
+
;;
|
|
161
|
+
|
|
162
|
+
#TODO: enable --inject=1 instead of just --inject 1 with something like:
|
|
163
|
+
#--file=?*)
|
|
164
|
+
# file=${1#*=} # Delete everything up to "=" and assign the remainder.
|
|
165
|
+
# ;;
|
|
166
|
+
--) # End of all options.
|
|
167
|
+
shift
|
|
168
|
+
break
|
|
169
|
+
;;
|
|
170
|
+
-?*)
|
|
171
|
+
printf 'ERROR: Unknown option: %s\n' "$1" >&2
|
|
172
|
+
USAGE
|
|
173
|
+
;;
|
|
174
|
+
*) # Default case: No more options, so break out of the loop.
|
|
175
|
+
break
|
|
176
|
+
esac
|
|
177
|
+
|
|
178
|
+
shift
|
|
179
|
+
done
|
|
180
|
+
|
|
181
|
+
project_name=$1
|
|
182
|
+
}
|
|
183
|
+
|