evalidate 2.0.4__tar.gz → 2.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {evalidate-2.0.4 → evalidate-2.1.0}/.gitignore +1 -0
- evalidate-2.0.4/README.md → evalidate-2.1.0/PKG-INFO +98 -6
- evalidate-2.0.4/PKG-INFO → evalidate-2.1.0/README.md +82 -20
- {evalidate-2.0.4 → evalidate-2.1.0}/evalidate/__init__.py +16 -8
- evalidate-2.1.0/evalidate/cli/genfakeproducts.py +93 -0
- evalidate-2.1.0/evalidate/cli/jg.py +105 -0
- {evalidate-2.0.4 → evalidate-2.1.0}/pyproject.toml +5 -1
- evalidate-2.1.0/tests/__init__.py +0 -0
- evalidate-2.1.0/tests/test_locals_globals.py +70 -0
- evalidate-2.1.0/tests/test_no_mutation.py +25 -0
- evalidate-2.0.4/NOTES +0 -4
- evalidate-2.0.4/ev.py +0 -60
- evalidate-2.0.4/examples/products.py +0 -31
- evalidate-2.0.4/x.py +0 -47
- evalidate-2.0.4/xx.py +0 -26
- evalidate-2.0.4/xx2.py +0 -27
- evalidate-2.0.4/xx3.py +0 -29
- evalidate-2.0.4/y.py +0 -9
- {evalidate-2.0.4 → evalidate-2.1.0}/LICENSE +0 -0
- {evalidate-2.0.4 → evalidate-2.1.0}/benchmark/benchmark.py +0 -0
- {evalidate-2.0.4 → evalidate-2.1.0}/evalidate/credits.txt +0 -0
- {evalidate-2.0.4 → evalidate-2.1.0}/evalidate/security.py +0 -0
- {evalidate-2.0.4 → evalidate-2.1.0}/examples/basic.py +0 -0
- {evalidate-2.0.4 → evalidate-2.1.0}/setup.py +0 -0
- {evalidate-2.0.4 → evalidate-2.1.0}/tests/test_exc.py +0 -0
- {evalidate-2.0.4 → evalidate-2.1.0}/tests/test_expr.py +0 -0
- {evalidate-2.0.4 → evalidate-2.1.0}/tests/test_feature.py +0 -0
- {evalidate-2.0.4 → evalidate-2.1.0}/tests/test_jailbreak.py +0 -0
|
@@ -1,10 +1,26 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: evalidate
|
|
3
|
+
Version: 2.1.0
|
|
4
|
+
Summary: Validation and secure evaluation of untrusted python expressions
|
|
5
|
+
Project-URL: Homepage, https://github.com/yaroslaff/evalidate
|
|
6
|
+
Project-URL: Issues, https://github.com/yaroslaff/evalidate/issues
|
|
7
|
+
Author-email: Yaroslav Polyakov <yaroslaff@gmail.com>
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Keywords: eval,evaluate,evaluation,expression,filter,grep,jg,jq,json,python,safe,sandbox,secure,security,validate
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Requires-Python: >=3.8
|
|
14
|
+
Requires-Dist: faker
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
1
17
|
# Evalidate
|
|
2
18
|
Evalidate is simple python module for safe and very fast eval()'uating user-supplied (possible malicious) python expressions.
|
|
3
19
|
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
20
|
+
Evalidate comes with a very convenient and fast CLI tool, `jg`, for filtering (grep‑ing) items in JSON lists:
|
|
21
|
+
```shell
|
|
22
|
+
jg 'attributes["storage"] == "512GB" and price<300 and rating>4' products.json
|
|
23
|
+
```
|
|
8
24
|
|
|
9
25
|
## Purpose
|
|
10
26
|
Originally it's developed for filtering complex data structures e.g.
|
|
@@ -111,7 +127,61 @@ m.imported_functions["one"] = one
|
|
|
111
127
|
Expr('one()', model=m).eval()
|
|
112
128
|
~~~
|
|
113
129
|
|
|
114
|
-
|
|
130
|
+
|
|
131
|
+
## CLI tools
|
|
132
|
+
### genfakeproducts
|
|
133
|
+
`genfakeproducts` generates a JSON list of N fake products for testing. For example:
|
|
134
|
+
|
|
135
|
+
```shell
|
|
136
|
+
# generates 1 million fake JSON records for testing (takes about 1 minute on my computer)
|
|
137
|
+
genfakeproducts -n 1000000 -o products.json
|
|
138
|
+
```
|
|
139
|
+
By default, N is 10000 and the output file is products.json (takes 1 second).
|
|
140
|
+
|
|
141
|
+
Example record:
|
|
142
|
+
```json
|
|
143
|
+
{
|
|
144
|
+
"id": "d5af9f93-c164-4872-a535-4fb56cf62e6e",
|
|
145
|
+
"sku": "SKU-D5AF9F93",
|
|
146
|
+
"title": "Google ET-3843 Pro",
|
|
147
|
+
"brand": "Google",
|
|
148
|
+
"category": "phones",
|
|
149
|
+
"description": "Foot trial city avoid he wish real college investment tend include draw window either over somebody history detail risk support glass. Google phones designed for modern users.",
|
|
150
|
+
"price": 340.94,
|
|
151
|
+
"currency": "USD",
|
|
152
|
+
"stock": 24,
|
|
153
|
+
"rating": 4.23,
|
|
154
|
+
"reviews_count": 222,
|
|
155
|
+
"attributes": {
|
|
156
|
+
"storage": "512GB",
|
|
157
|
+
"color": "red"
|
|
158
|
+
},
|
|
159
|
+
"tags": [
|
|
160
|
+
"limited",
|
|
161
|
+
"new"
|
|
162
|
+
]
|
|
163
|
+
}
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
### jg (grep)
|
|
167
|
+
`jg` is a very fast JSON grep CLI tool. It accepts a Python expression and a filename, or reads from stdin:
|
|
168
|
+
|
|
169
|
+
```shell
|
|
170
|
+
jg 'category=="phones"' products.json
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
`jg` implements only a small subset of jq's functionality, but it is twice as fast and much simpler to use.
|
|
174
|
+
|
|
175
|
+
**Options**
|
|
176
|
+
- `-b` - benchmark. Will not print output JSON. Measures filtering time.
|
|
177
|
+
- `-v` - verbose mode.
|
|
178
|
+
- `-l` - reads input as JSONL (one JSON object per line) instead of a JSON array
|
|
179
|
+
- `-k` - keypath to list e.g. "shop::products::onstock" if the input data is a nested dictionary
|
|
180
|
+
- `-f` - custom output format for non-JSON output e.g. '{sku} {price} ({stock}) {title!r}'
|
|
181
|
+
|
|
182
|
+
## Advanced topics
|
|
183
|
+
|
|
184
|
+
### Improve speed by using native eval() with validated code
|
|
115
185
|
Evalidate is very fast, but it's still takes CPU cycles... If you want to achieve maximal possible speed, you can use python native [eval](https://docs.python.org/3/library/functions.html#eval) with this kind of code:
|
|
116
186
|
|
|
117
187
|
~~~python
|
|
@@ -126,7 +196,29 @@ This is as secure as expr.eval(), because `expr.code` is already validated to be
|
|
|
126
196
|
|
|
127
197
|
Difference is very little: execution of `expr.code` can throw any exception, while `expr.eval()` can throw only ExecutionException. Also, if you want to export your functions to eval, you should do this manually.
|
|
128
198
|
|
|
129
|
-
|
|
199
|
+
### Custom mapping classes
|
|
200
|
+
|
|
201
|
+
You can use custom mapping classes instead of `dict`. As with native python eval, custom mapping class may be used only as locals context.
|
|
202
|
+
|
|
203
|
+
Example:
|
|
204
|
+
|
|
205
|
+
~~~python
|
|
206
|
+
from collections import UserDict
|
|
207
|
+
from evalidate import Expr
|
|
208
|
+
|
|
209
|
+
class LazyDict(UserDict):
|
|
210
|
+
def __missing__(self, key):
|
|
211
|
+
return 42
|
|
212
|
+
|
|
213
|
+
ctx = LazyDict(a=100)
|
|
214
|
+
expr = Expr("a+b")
|
|
215
|
+
|
|
216
|
+
res = expr.eval(ctx_locals=ctx)
|
|
217
|
+
print(res) # 142
|
|
218
|
+
~~~
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
### Limitations
|
|
130
222
|
|
|
131
223
|
evalidate uses [ast.parse()](https://docs.python.org/3/library/ast.html#ast.parse) to get [AST node](https://docs.python.org/3/library/ast.html#node-classes) to validate it.
|
|
132
224
|
|
|
@@ -1,24 +1,10 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: evalidate
|
|
3
|
-
Version: 2.0.4
|
|
4
|
-
Summary: Validation and secure evaluation of untrusted python expressions
|
|
5
|
-
Project-URL: Homepage, https://github.com/yaroslaff/evalidate
|
|
6
|
-
Project-URL: Issues, https://github.com/yaroslaff/evalidate/issues
|
|
7
|
-
Author-email: Yaroslav Polyakov <yaroslaff@gmail.com>
|
|
8
|
-
License-File: LICENSE
|
|
9
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
-
Classifier: Operating System :: OS Independent
|
|
11
|
-
Classifier: Programming Language :: Python :: 3
|
|
12
|
-
Requires-Python: >=3.8
|
|
13
|
-
Description-Content-Type: text/markdown
|
|
14
|
-
|
|
15
1
|
# Evalidate
|
|
16
2
|
Evalidate is simple python module for safe and very fast eval()'uating user-supplied (possible malicious) python expressions.
|
|
17
3
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
4
|
+
Evalidate comes with a very convenient and fast CLI tool, `jg`, for filtering (grep‑ing) items in JSON lists:
|
|
5
|
+
```shell
|
|
6
|
+
jg 'attributes["storage"] == "512GB" and price<300 and rating>4' products.json
|
|
7
|
+
```
|
|
22
8
|
|
|
23
9
|
## Purpose
|
|
24
10
|
Originally it's developed for filtering complex data structures e.g.
|
|
@@ -125,7 +111,61 @@ m.imported_functions["one"] = one
|
|
|
125
111
|
Expr('one()', model=m).eval()
|
|
126
112
|
~~~
|
|
127
113
|
|
|
128
|
-
|
|
114
|
+
|
|
115
|
+
## CLI tools
|
|
116
|
+
### genfakeproducts
|
|
117
|
+
`genfakeproducts` generates a JSON list of N fake products for testing. For example:
|
|
118
|
+
|
|
119
|
+
```shell
|
|
120
|
+
# generates 1 million fake JSON records for testing (takes about 1 minute on my computer)
|
|
121
|
+
genfakeproducts -n 1000000 -o products.json
|
|
122
|
+
```
|
|
123
|
+
By default, N is 10000 and the output file is products.json (takes 1 second).
|
|
124
|
+
|
|
125
|
+
Example record:
|
|
126
|
+
```json
|
|
127
|
+
{
|
|
128
|
+
"id": "d5af9f93-c164-4872-a535-4fb56cf62e6e",
|
|
129
|
+
"sku": "SKU-D5AF9F93",
|
|
130
|
+
"title": "Google ET-3843 Pro",
|
|
131
|
+
"brand": "Google",
|
|
132
|
+
"category": "phones",
|
|
133
|
+
"description": "Foot trial city avoid he wish real college investment tend include draw window either over somebody history detail risk support glass. Google phones designed for modern users.",
|
|
134
|
+
"price": 340.94,
|
|
135
|
+
"currency": "USD",
|
|
136
|
+
"stock": 24,
|
|
137
|
+
"rating": 4.23,
|
|
138
|
+
"reviews_count": 222,
|
|
139
|
+
"attributes": {
|
|
140
|
+
"storage": "512GB",
|
|
141
|
+
"color": "red"
|
|
142
|
+
},
|
|
143
|
+
"tags": [
|
|
144
|
+
"limited",
|
|
145
|
+
"new"
|
|
146
|
+
]
|
|
147
|
+
}
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
### jg (grep)
|
|
151
|
+
`jg` is a very fast JSON grep CLI tool. It accepts a Python expression and a filename, or reads from stdin:
|
|
152
|
+
|
|
153
|
+
```shell
|
|
154
|
+
jg 'category=="phones"' products.json
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
`jg` implements only a small subset of jq's functionality, but it is twice as fast and much simpler to use.
|
|
158
|
+
|
|
159
|
+
**Options**
|
|
160
|
+
- `-b` - benchmark. Will not print output JSON. Measures filtering time.
|
|
161
|
+
- `-v` - verbose mode.
|
|
162
|
+
- `-l` - reads input as JSONL (one JSON object per line) instead of a JSON array
|
|
163
|
+
- `-k` - keypath to list e.g. "shop::products::onstock" if the input data is a nested dictionary
|
|
164
|
+
- `-f` - custom output format for non-JSON output e.g. '{sku} {price} ({stock}) {title!r}'
|
|
165
|
+
|
|
166
|
+
## Advanced topics
|
|
167
|
+
|
|
168
|
+
### Improve speed by using native eval() with validated code
|
|
129
169
|
Evalidate is very fast, but it's still takes CPU cycles... If you want to achieve maximal possible speed, you can use python native [eval](https://docs.python.org/3/library/functions.html#eval) with this kind of code:
|
|
130
170
|
|
|
131
171
|
~~~python
|
|
@@ -140,7 +180,29 @@ This is as secure as expr.eval(), because `expr.code` is already validated to be
|
|
|
140
180
|
|
|
141
181
|
Difference is very little: execution of `expr.code` can throw any exception, while `expr.eval()` can throw only ExecutionException. Also, if you want to export your functions to eval, you should do this manually.
|
|
142
182
|
|
|
143
|
-
|
|
183
|
+
### Custom mapping classes
|
|
184
|
+
|
|
185
|
+
You can use custom mapping classes instead of `dict`. As with native python eval, custom mapping class may be used only as locals context.
|
|
186
|
+
|
|
187
|
+
Example:
|
|
188
|
+
|
|
189
|
+
~~~python
|
|
190
|
+
from collections import UserDict
|
|
191
|
+
from evalidate import Expr
|
|
192
|
+
|
|
193
|
+
class LazyDict(UserDict):
|
|
194
|
+
def __missing__(self, key):
|
|
195
|
+
return 42
|
|
196
|
+
|
|
197
|
+
ctx = LazyDict(a=100)
|
|
198
|
+
expr = Expr("a+b")
|
|
199
|
+
|
|
200
|
+
res = expr.eval(ctx_locals=ctx)
|
|
201
|
+
print(res) # 142
|
|
202
|
+
~~~
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
### Limitations
|
|
144
206
|
|
|
145
207
|
evalidate uses [ast.parse()](https://docs.python.org/3/library/ast.html#ast.parse) to get [AST node](https://docs.python.org/3/library/ast.html#node-classes) to validate it.
|
|
146
208
|
|
|
@@ -6,7 +6,7 @@ import ast
|
|
|
6
6
|
import dataclasses
|
|
7
7
|
from typing import Callable
|
|
8
8
|
|
|
9
|
-
__version__ = '2.0
|
|
9
|
+
__version__ = '2.1.0'
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
class EvalException(Exception):
|
|
@@ -123,21 +123,29 @@ class Expr():
|
|
|
123
123
|
|
|
124
124
|
self.code = compile(self.node, filename or '<usercode>', 'eval')
|
|
125
125
|
|
|
126
|
-
def eval(self,
|
|
126
|
+
def eval(self, ctx_globals=None, ctx_locals=None):
|
|
127
127
|
|
|
128
|
-
|
|
129
|
-
|
|
128
|
+
#
|
|
129
|
+
# PEP709 (python 3.12 and above): list comprehensions works even with data from locals
|
|
130
|
+
# before 3.12, list comprehensions works only with data in globals
|
|
131
|
+
#
|
|
132
|
+
|
|
133
|
+
ctx_locals = dict() if ctx_locals is None else ctx_locals
|
|
134
|
+
ctx_globals = dict() if ctx_globals is None else ctx_globals
|
|
135
|
+
|
|
136
|
+
if ctx_globals:
|
|
137
|
+
ctx_globals = {
|
|
130
138
|
**self.model.imported_functions,
|
|
131
|
-
**
|
|
139
|
+
**ctx_globals
|
|
132
140
|
}
|
|
133
141
|
else:
|
|
134
|
-
|
|
142
|
+
ctx_globals = self.model.imported_functions
|
|
135
143
|
|
|
136
144
|
try:
|
|
137
|
-
result = eval(self.code,
|
|
145
|
+
result = eval(self.code, ctx_globals, ctx_locals)
|
|
138
146
|
except Exception as e:
|
|
139
147
|
raise ExecutionException(e)
|
|
140
|
-
|
|
148
|
+
|
|
141
149
|
return result
|
|
142
150
|
|
|
143
151
|
def __str__(self):
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
gen_products.py
|
|
4
|
+
Generate fake e-commerce products (USD, no images) using Faker.
|
|
5
|
+
|
|
6
|
+
Default:
|
|
7
|
+
10,000 products → products.json
|
|
8
|
+
Example:
|
|
9
|
+
python3 gen_products.py --out /tmp/products.json --count 5000
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import argparse, json, random, uuid
|
|
13
|
+
from datetime import datetime, UTC
|
|
14
|
+
from faker import Faker
|
|
15
|
+
|
|
16
|
+
fake = Faker()
|
|
17
|
+
|
|
18
|
+
BRANDS = {
|
|
19
|
+
"phones": ["Samsung", "Apple", "Xiaomi", "OnePlus", "Google"],
|
|
20
|
+
"laptops": ["Dell", "HP", "Lenovo", "Asus", "Acer", "Apple"],
|
|
21
|
+
"audio": ["Sony", "JBL", "Bose", "Sennheiser", "Marshall"],
|
|
22
|
+
"tv": ["LG", "Samsung", "Philips", "TCL", "Panasonic"],
|
|
23
|
+
"camera": ["Canon", "Nikon", "Sony", "Fujifilm", "GoPro"],
|
|
24
|
+
"appliances": ["Bosch", "Philips", "Panasonic", "Midea", "Tefal"]
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
SUFFIXES = ["", " Pro", " Max", " Plus", " Ultra", " SE"]
|
|
28
|
+
|
|
29
|
+
def make_title(category, brand):
|
|
30
|
+
letters = ''.join(random.choices('ABCDEFGHIJKLMNOPQRSTUVWXYZ', k=2))
|
|
31
|
+
numbers = str(random.randint(100, 9999))
|
|
32
|
+
suffix = random.choice(SUFFIXES)
|
|
33
|
+
return f"{brand} {letters}-{numbers}{suffix}"
|
|
34
|
+
|
|
35
|
+
def make_price(category):
|
|
36
|
+
base = {
|
|
37
|
+
"phones": 300, "laptops": 800, "audio": 120,
|
|
38
|
+
"tv": 400, "camera": 600, "appliances": 200
|
|
39
|
+
}.get(category, 100)
|
|
40
|
+
p = base * random.uniform(0.7, 1.8)
|
|
41
|
+
return round(p - 0.01, 2)
|
|
42
|
+
|
|
43
|
+
def make_description(category, brand):
|
|
44
|
+
return fake.sentence(nb_words=random.randint(12, 24)) + f" {brand} {category} designed for modern users."
|
|
45
|
+
|
|
46
|
+
def make_attributes(category):
|
|
47
|
+
attrs = {}
|
|
48
|
+
if category in ("phones", "laptops"):
|
|
49
|
+
attrs["storage"] = random.choice(["64GB","128GB","256GB","512GB","1TB"])
|
|
50
|
+
if category in ("phones","audio","camera"):
|
|
51
|
+
attrs["color"] = random.choice(["black","white","blue","red","silver"])
|
|
52
|
+
if category in ("tv","appliances"):
|
|
53
|
+
attrs["power_w"] = random.choice([800,1200,1500,2000])
|
|
54
|
+
return attrs
|
|
55
|
+
|
|
56
|
+
def gen_product():
|
|
57
|
+
category = random.choice(list(BRANDS.keys()))
|
|
58
|
+
brand = random.choice(BRANDS[category])
|
|
59
|
+
uid = str(uuid.uuid4())
|
|
60
|
+
return {
|
|
61
|
+
"id": uid,
|
|
62
|
+
"sku": f"SKU-{uid[:8].upper()}",
|
|
63
|
+
"title": make_title(category, brand),
|
|
64
|
+
"brand": brand,
|
|
65
|
+
"category": category,
|
|
66
|
+
"description": make_description(category, brand),
|
|
67
|
+
"price": make_price(category),
|
|
68
|
+
"currency": "USD",
|
|
69
|
+
"stock": random.randint(0, 500),
|
|
70
|
+
"rating": round(random.uniform(3.0, 5.0), 2),
|
|
71
|
+
"reviews_count": random.randint(0, 1000),
|
|
72
|
+
"attributes": make_attributes(category),
|
|
73
|
+
"tags": random.sample(["new","bestseller","discount","limited","eco"], k=random.randint(0,3)),
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
def main():
|
|
77
|
+
ap = argparse.ArgumentParser(description="Generate fake e-commerce products in JSON format.")
|
|
78
|
+
ap.add_argument("-n", "--num", type=int, default=10000, help="Number of products to generate (default: 10000).")
|
|
79
|
+
ap.add_argument("-o", "--out", type=str, default="products.json", help="Output JSON file path (default: products.json).")
|
|
80
|
+
args = ap.parse_args()
|
|
81
|
+
|
|
82
|
+
with open(args.out, "w", encoding="utf-8") as f:
|
|
83
|
+
f.write("[\n")
|
|
84
|
+
for i in range(args.num):
|
|
85
|
+
json.dump(gen_product(), f, ensure_ascii=False)
|
|
86
|
+
if i != args.num - 1:
|
|
87
|
+
f.write(",\n")
|
|
88
|
+
f.write("\n]\n")
|
|
89
|
+
|
|
90
|
+
print(f"Wrote {args.num} products to {args.out}")
|
|
91
|
+
|
|
92
|
+
if __name__ == "__main__":
|
|
93
|
+
main()
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
|
|
3
|
+
import requests
|
|
4
|
+
from evalidate import Expr, ValidationException, CompilationException, ExecutionException
|
|
5
|
+
import json
|
|
6
|
+
import sys
|
|
7
|
+
import argparse
|
|
8
|
+
import time
|
|
9
|
+
|
|
10
|
+
def short_repr(data, limit=100):
|
|
11
|
+
try:
|
|
12
|
+
text = json.dumps(data, ensure_ascii=False)
|
|
13
|
+
except Exception:
|
|
14
|
+
text = repr(data)
|
|
15
|
+
if len(text) > limit:
|
|
16
|
+
text = text[:limit] + "..."
|
|
17
|
+
return text
|
|
18
|
+
|
|
19
|
+
def get_args():
|
|
20
|
+
parser = argparse.ArgumentParser('Grep JSON files by python expression')
|
|
21
|
+
parser.add_argument('-b', '--benchmark', default=False, action='store_true', help='Do not print, just search and measure search time')
|
|
22
|
+
parser.add_argument('-v', '--verbose', default=False, action='store_true', help='Be verbose')
|
|
23
|
+
parser.add_argument('-l', '--jsonl', default=False, action='store_true', help='input is JSONL (not JSON list)')
|
|
24
|
+
parser.add_argument('-k', '--keypath', type=str, help='key path to list separated by ::, e.g.: -k shop::products::onstock')
|
|
25
|
+
parser.add_argument('-f', '--format', type=str, help='format (if not JSON), e.g. -f \'{sku} {price} ({stock}) {title!r}\'')
|
|
26
|
+
parser.add_argument('expr', help='Python filter expression')
|
|
27
|
+
parser.add_argument('path', default=None, help='Path to JSON list (or stdin)', nargs='?')
|
|
28
|
+
|
|
29
|
+
return parser.parse_args()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def vprint(s: str):
|
|
34
|
+
if args.verbose:
|
|
35
|
+
print(s)
|
|
36
|
+
|
|
37
|
+
def main():
|
|
38
|
+
global args
|
|
39
|
+
|
|
40
|
+
args = get_args()
|
|
41
|
+
if args.benchmark:
|
|
42
|
+
args.verbose = True
|
|
43
|
+
out = list()
|
|
44
|
+
|
|
45
|
+
vprint(f'# loading data from {args.path or "stdin"}')
|
|
46
|
+
|
|
47
|
+
if args.path:
|
|
48
|
+
f = open(args.path, "r", encoding="utf-8")
|
|
49
|
+
else:
|
|
50
|
+
f = sys.stdin
|
|
51
|
+
|
|
52
|
+
with f:
|
|
53
|
+
if args.jsonl:
|
|
54
|
+
data = [json.loads(line) for line in f if line.strip()]
|
|
55
|
+
else:
|
|
56
|
+
data = json.load(f)
|
|
57
|
+
|
|
58
|
+
# follow keypath if needed
|
|
59
|
+
try:
|
|
60
|
+
if args.keypath:
|
|
61
|
+
for k in args.keypath.split('::'):
|
|
62
|
+
data = data[k]
|
|
63
|
+
except KeyError as e:
|
|
64
|
+
print(f"No such key: {e}", file=sys.stderr)
|
|
65
|
+
sys.exit(1)
|
|
66
|
+
|
|
67
|
+
vprint(f'# loaded {len(data)} records from {args.path}')
|
|
68
|
+
|
|
69
|
+
try:
|
|
70
|
+
expr = Expr(args.expr)
|
|
71
|
+
except (ValidationException, CompilationException) as e:
|
|
72
|
+
print(e)
|
|
73
|
+
sys.exit(1)
|
|
74
|
+
|
|
75
|
+
c=0
|
|
76
|
+
start = time.time()
|
|
77
|
+
|
|
78
|
+
if not isinstance(data, list):
|
|
79
|
+
print("data is not a list", file=sys.stderr)
|
|
80
|
+
print(short_repr(data))
|
|
81
|
+
sys.exit(1)
|
|
82
|
+
|
|
83
|
+
for p in data:
|
|
84
|
+
try:
|
|
85
|
+
# we use native eval with validated code to get better speed
|
|
86
|
+
r = eval(expr.code, dict(), p)
|
|
87
|
+
if r:
|
|
88
|
+
out.append(p)
|
|
89
|
+
c+=1
|
|
90
|
+
except KeyError as e:
|
|
91
|
+
pass
|
|
92
|
+
# print("Runtime exception:", e)
|
|
93
|
+
|
|
94
|
+
elapsed = round(time.time() - start, 2)
|
|
95
|
+
|
|
96
|
+
vprint(f"# {c}/{len(data)} elements found in {elapsed} seconds".format(c))
|
|
97
|
+
if not args.benchmark:
|
|
98
|
+
if args.format:
|
|
99
|
+
for el in out:
|
|
100
|
+
print(args.format.format(**el))
|
|
101
|
+
else:
|
|
102
|
+
print(json.dumps(out, indent=2))
|
|
103
|
+
|
|
104
|
+
if __name__ == '__main__':
|
|
105
|
+
main()
|
|
@@ -8,8 +8,9 @@ build-backend = "hatchling.build"
|
|
|
8
8
|
[project]
|
|
9
9
|
name="evalidate"
|
|
10
10
|
dynamic = [ "version" ]
|
|
11
|
+
keywords = ["python", "security", "json", "sandbox", "filter", "evaluation", "expression", "secure", "jq", "jg", "validate", "safe", "grep", "eval", "evaluate", "jg"]
|
|
11
12
|
|
|
12
|
-
dependencies = []
|
|
13
|
+
dependencies = ['faker']
|
|
13
14
|
authors = [
|
|
14
15
|
{ name="Yaroslav Polyakov", email="yaroslaff@gmail.com" },
|
|
15
16
|
]
|
|
@@ -27,6 +28,9 @@ Homepage = "https://github.com/yaroslaff/evalidate"
|
|
|
27
28
|
Issues = "https://github.com/yaroslaff/evalidate/issues"
|
|
28
29
|
|
|
29
30
|
|
|
31
|
+
[project.scripts]
|
|
32
|
+
genfakeproducts = "evalidate.cli.genfakeproducts:main"
|
|
33
|
+
jg = "evalidate.cli.jg:main"
|
|
30
34
|
|
|
31
35
|
[tool.hatch.version]
|
|
32
36
|
path = 'evalidate/__init__.py'
|
|
File without changes
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# from evalidate import ExecutionException, ValidationException, Expr, base_eval_model
|
|
2
|
+
from evalidate import Expr, base_eval_model, ValidationException, ExecutionException
|
|
3
|
+
import pytest
|
|
4
|
+
from collections import UserDict
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class LazyDict(UserDict):
|
|
8
|
+
def __missing__(self, key):
|
|
9
|
+
return 42
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class TestLocalsGlobals():
|
|
13
|
+
|
|
14
|
+
def test_locals(self):
|
|
15
|
+
expr = Expr('a + b')
|
|
16
|
+
ctx = dict(a=1, b=2)
|
|
17
|
+
result = expr.eval(None, ctx)
|
|
18
|
+
assert result == 3
|
|
19
|
+
|
|
20
|
+
def test_globals(self):
|
|
21
|
+
expr = Expr('a + b')
|
|
22
|
+
ctx = dict(a=1, b=2)
|
|
23
|
+
result = expr.eval(ctx)
|
|
24
|
+
assert result == 3
|
|
25
|
+
|
|
26
|
+
def test_list_comprehension(self):
|
|
27
|
+
|
|
28
|
+
# prepare model
|
|
29
|
+
my_model = base_eval_model.clone()
|
|
30
|
+
my_model.nodes.extend(
|
|
31
|
+
[
|
|
32
|
+
"Call",
|
|
33
|
+
"Attributes",
|
|
34
|
+
"ListComp",
|
|
35
|
+
"DictComp",
|
|
36
|
+
"comprehension",
|
|
37
|
+
"Store",
|
|
38
|
+
"ForOfStatement",
|
|
39
|
+
"Subscript",
|
|
40
|
+
"GeneratorExp",
|
|
41
|
+
"For",
|
|
42
|
+
]
|
|
43
|
+
)
|
|
44
|
+
my_model.allowed_functions.append("sum")
|
|
45
|
+
|
|
46
|
+
my_shelve = {
|
|
47
|
+
"height": 200,
|
|
48
|
+
"boxes": {
|
|
49
|
+
"box1": {"volume": 110},
|
|
50
|
+
"box2": {"volume": 90},
|
|
51
|
+
},
|
|
52
|
+
"column_width": [20, 480, 40],
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
exp_string = "sum([my_shelve['boxes'][box]['volume'] for box in my_shelve['boxes'] ])"
|
|
56
|
+
exp = Expr(exp_string, my_model)
|
|
57
|
+
res = exp.eval({"my_shelve": my_shelve})
|
|
58
|
+
assert res == 200
|
|
59
|
+
|
|
60
|
+
def test_userdict(self):
|
|
61
|
+
ctx = LazyDict(a=100)
|
|
62
|
+
expr = Expr("a+b")
|
|
63
|
+
|
|
64
|
+
# must raise exception
|
|
65
|
+
with pytest.raises(ExecutionException):
|
|
66
|
+
res = expr.eval(ctx_globals=ctx)
|
|
67
|
+
|
|
68
|
+
res = expr.eval(ctx_locals=ctx)
|
|
69
|
+
assert res == 142
|
|
70
|
+
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# from evalidate import ExecutionException, ValidationException, Expr, base_eval_model
|
|
2
|
+
from evalidate import Expr, base_eval_model, ValidationException, ExecutionException
|
|
3
|
+
from copy import deepcopy
|
|
4
|
+
from collections import UserDict
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class TestNoMutation():
|
|
8
|
+
|
|
9
|
+
def test_no_mutation_globals(self):
|
|
10
|
+
expr = Expr('a + b')
|
|
11
|
+
ctx = dict(a=1, b=2)
|
|
12
|
+
ctx_orig = deepcopy(ctx)
|
|
13
|
+
result = expr.eval(ctx_globals=ctx)
|
|
14
|
+
assert result == 3
|
|
15
|
+
assert ctx == ctx_orig
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def test_no_mutation_locals(self):
|
|
19
|
+
expr = Expr('a + b')
|
|
20
|
+
ctx = dict(a=1, b=2)
|
|
21
|
+
ctx_orig = deepcopy(ctx)
|
|
22
|
+
result = expr.eval(ctx_locals=ctx)
|
|
23
|
+
assert result == 3
|
|
24
|
+
assert ctx == ctx_orig
|
|
25
|
+
|
evalidate-2.0.4/NOTES
DELETED
evalidate-2.0.4/ev.py
DELETED
|
@@ -1,60 +0,0 @@
|
|
|
1
|
-
from evalidate import Expr, base_eval_model, ExecutionException
|
|
2
|
-
import sys
|
|
3
|
-
|
|
4
|
-
my_model = base_eval_model.clone()
|
|
5
|
-
my_model.nodes.extend(
|
|
6
|
-
[
|
|
7
|
-
"Call",
|
|
8
|
-
"Attributes",
|
|
9
|
-
"ListComp",
|
|
10
|
-
"DictComp",
|
|
11
|
-
"comprehension",
|
|
12
|
-
"Store",
|
|
13
|
-
"ForOfStatement",
|
|
14
|
-
"Subscript",
|
|
15
|
-
"GeneratorExp",
|
|
16
|
-
"For",
|
|
17
|
-
]
|
|
18
|
-
)
|
|
19
|
-
my_model.allowed_functions.append("sum")
|
|
20
|
-
my_model.allowed_functions.append("len")
|
|
21
|
-
|
|
22
|
-
my_shelve = {
|
|
23
|
-
"height": 200,
|
|
24
|
-
"boxes": {
|
|
25
|
-
"box1": {"volume": 110},
|
|
26
|
-
"box2": {"volume": 90},
|
|
27
|
-
},
|
|
28
|
-
"column_width": [20, 480, 40],
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
box_volumes = [my_shelve["boxes"][box]["volume"] for box in my_shelve["boxes"]]
|
|
32
|
-
total_volume = sum(box_volumes)
|
|
33
|
-
print(total_volume)
|
|
34
|
-
|
|
35
|
-
# works fine
|
|
36
|
-
exp_string = "sum( my_shelve['column_width'])"
|
|
37
|
-
exp = Expr(exp_string, my_model)
|
|
38
|
-
print("code:", exp.code)
|
|
39
|
-
res = exp.eval({"my_shelve": my_shelve})
|
|
40
|
-
print(res)
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
# throws an Exception:
|
|
44
|
-
# evalidate.ExecutionException: name 'my_shelve' is not defined
|
|
45
|
-
|
|
46
|
-
# exp_string = "sum([my_shelve['boxes'][box]['volume'] for box in my_shelve['boxes'] ])"
|
|
47
|
-
# exp_string = "sum([my_shelve['boxes'][box]['volume'] for box in my_shelve['boxes'] ])"
|
|
48
|
-
exp_string = "len(my_shelve)"
|
|
49
|
-
exp = Expr(exp_string, my_model)
|
|
50
|
-
|
|
51
|
-
#print("basic eval")
|
|
52
|
-
#res = eval(exp.code, dict(), {"my_shelve": my_shelve})
|
|
53
|
-
#print("basic res:",res)
|
|
54
|
-
# sys.exit(0)
|
|
55
|
-
try:
|
|
56
|
-
res = exp.eval({"my_shelve": my_shelve})
|
|
57
|
-
except ExecutionException as e:
|
|
58
|
-
print(e)
|
|
59
|
-
|
|
60
|
-
# print(res)
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
|
|
3
|
-
import requests
|
|
4
|
-
from evalidate import Expr, ValidationException, CompilationException, ExecutionException
|
|
5
|
-
import json
|
|
6
|
-
import sys
|
|
7
|
-
|
|
8
|
-
data = requests.get('https://dummyjson.com/products?limit=100').json()
|
|
9
|
-
|
|
10
|
-
try:
|
|
11
|
-
src = sys.argv[1]
|
|
12
|
-
except IndexError:
|
|
13
|
-
src = 'True'
|
|
14
|
-
|
|
15
|
-
try:
|
|
16
|
-
expr = Expr(src)
|
|
17
|
-
except (ValidationException, CompilationException) as e:
|
|
18
|
-
print(e)
|
|
19
|
-
sys.exit(1)
|
|
20
|
-
|
|
21
|
-
c=0
|
|
22
|
-
for p in data['products']:
|
|
23
|
-
# print(p)
|
|
24
|
-
try:
|
|
25
|
-
r = expr.eval(p)
|
|
26
|
-
if r:
|
|
27
|
-
print(json.dumps(p, indent=2))
|
|
28
|
-
c+=1
|
|
29
|
-
except ExecutionException as e:
|
|
30
|
-
print("Runtime exception:", e)
|
|
31
|
-
print("# {} products matches".format(c))
|
evalidate-2.0.4/x.py
DELETED
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
import evalidate
|
|
2
|
-
|
|
3
|
-
def eval_expression(input_string):
|
|
4
|
-
# Step 1
|
|
5
|
-
allowed_names = {"sum": sum, "int": int, "a":1, "b":2, 'startswith': str.startswith}
|
|
6
|
-
# Step 2
|
|
7
|
-
code = compile(input_string, "<string>", "eval")
|
|
8
|
-
# Step 3
|
|
9
|
-
print(code.co_names)
|
|
10
|
-
for name in code.co_names:
|
|
11
|
-
if name not in allowed_names:
|
|
12
|
-
# Step 4
|
|
13
|
-
raise NameError(f"Use of {name!r} not allowed")
|
|
14
|
-
return eval(code, {"__builtins__": {}}, allowed_names)
|
|
15
|
-
|
|
16
|
-
try:
|
|
17
|
-
src="""
|
|
18
|
-
(lambda fc=(
|
|
19
|
-
lambda n: [
|
|
20
|
-
c for c in
|
|
21
|
-
().__class__.__bases__[0].__subclasses__()
|
|
22
|
-
if c.__name__ == n
|
|
23
|
-
][0]
|
|
24
|
-
):
|
|
25
|
-
fc("function")(
|
|
26
|
-
fc("code")(
|
|
27
|
-
0,0,0,0,0,0,b"BOOM",(),(),(),"","",0,b""
|
|
28
|
-
),{}
|
|
29
|
-
)()
|
|
30
|
-
)()
|
|
31
|
-
"""
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
#src = "'asdf'.startswith('as')"
|
|
35
|
-
#src2 ="""__builtins__['eval']("print(1)")"""
|
|
36
|
-
|
|
37
|
-
#node = evalidate.evalidate(src,
|
|
38
|
-
# addnodes=['Call', 'Attribute', 'ListComp', 'comprehension', 'Store'],
|
|
39
|
-
# attrs=['startswith'])
|
|
40
|
-
#code = compile(node, '<test>', 'eval')
|
|
41
|
-
#r = eval(code)
|
|
42
|
-
#print("eval:", r)
|
|
43
|
-
|
|
44
|
-
r = eval_expression(src)
|
|
45
|
-
print(r)
|
|
46
|
-
except Exception as e:
|
|
47
|
-
print(e)
|
evalidate-2.0.4/xx.py
DELETED
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
import evalidate
|
|
2
|
-
from dataclasses import dataclass
|
|
3
|
-
from evalidate import Expr, EvalException, base_eval_model
|
|
4
|
-
|
|
5
|
-
@dataclass
|
|
6
|
-
class Person:
|
|
7
|
-
name: str
|
|
8
|
-
weight: float
|
|
9
|
-
|
|
10
|
-
john = Person(name="John", weight=100)
|
|
11
|
-
jack = Person(name="Jack", weight=60)
|
|
12
|
-
passengers = {"john": john, "jack": jack}
|
|
13
|
-
|
|
14
|
-
sum_expr = "john.weight + jack.weight"
|
|
15
|
-
|
|
16
|
-
mymodel = base_eval_model.clone()
|
|
17
|
-
mymodel.nodes.append('Attribute')
|
|
18
|
-
mymodel.attributes.append('weight')
|
|
19
|
-
|
|
20
|
-
validated_expr = evalidate.Expr(sum_expr, model=mymodel)
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
total_weight = eval(validated_expr.code, passengers, None)
|
|
24
|
-
|
|
25
|
-
print(total_weight)
|
|
26
|
-
|
evalidate-2.0.4/xx2.py
DELETED
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
from dataclasses import dataclass
|
|
2
|
-
from evalidate import Expr, EvalException, base_eval_model
|
|
3
|
-
|
|
4
|
-
@dataclass
|
|
5
|
-
class Person:
|
|
6
|
-
name: str
|
|
7
|
-
weight: float
|
|
8
|
-
|
|
9
|
-
def get_weight(self):
|
|
10
|
-
return self.weight
|
|
11
|
-
|
|
12
|
-
john = Person(name="John", weight=100)
|
|
13
|
-
jack = Person(name="Jack", weight=60)
|
|
14
|
-
passengers = {"john": john, "jack": jack}
|
|
15
|
-
|
|
16
|
-
sum_expr = "john.get_weight() + jack.get_weight()"
|
|
17
|
-
|
|
18
|
-
mymodel = base_eval_model.clone()
|
|
19
|
-
mymodel.nodes.extend(['Attribute', 'Call'])
|
|
20
|
-
mymodel.attributes.append('get_weight')
|
|
21
|
-
|
|
22
|
-
validated_expr = Expr(sum_expr, model=mymodel)
|
|
23
|
-
|
|
24
|
-
total_weight = eval(validated_expr.code, passengers, None)
|
|
25
|
-
|
|
26
|
-
print(total_weight)
|
|
27
|
-
|
evalidate-2.0.4/xx3.py
DELETED
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
import evalidate
|
|
2
|
-
from dataclasses import dataclass
|
|
3
|
-
from evalidate import Expr, EvalException, base_eval_model
|
|
4
|
-
|
|
5
|
-
@dataclass
|
|
6
|
-
class Person:
|
|
7
|
-
name: str
|
|
8
|
-
weight: float
|
|
9
|
-
|
|
10
|
-
def get_weight(self):
|
|
11
|
-
return self.weight
|
|
12
|
-
|
|
13
|
-
john = Person(name="John", weight=100)
|
|
14
|
-
jack = Person(name="Jack", weight=60)
|
|
15
|
-
passengers = {"john": john, "jack": jack}
|
|
16
|
-
|
|
17
|
-
sum_expr = "john.get_weight() + jack.get_weight()"
|
|
18
|
-
|
|
19
|
-
mymodel =
|
|
20
|
-
|
|
21
|
-
mymodel.nodes.extend(['Attribute', 'Call'])
|
|
22
|
-
mymodel.attributes.append('get_weight')
|
|
23
|
-
|
|
24
|
-
validated_expr = evalidate.Expr(sum_expr, model=mymodel)
|
|
25
|
-
|
|
26
|
-
total_weight = eval(validated_expr.code, {"john": john, "jack": jack}, None)
|
|
27
|
-
|
|
28
|
-
print(total_weight)
|
|
29
|
-
|
evalidate-2.0.4/y.py
DELETED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|