firegex 5.0.0__cp313-cp313-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- firegex/__init__.py +69 -0
- firegex/__main__.py +6 -0
- firegex/_llhttp.cp313-win_amd64.pyd +0 -0
- firegex/cli.py +219 -0
- firegex/pyfilters/__init__.py +58 -0
- firegex/pyfilters/internals/__init__.py +317 -0
- firegex/pyfilters/internals/data.py +226 -0
- firegex/pyfilters/internals/exceptions.py +16 -0
- firegex/pyfilters/internals/models.py +55 -0
- firegex/pyfilters/internals/zstd_compat.py +66 -0
- firegex/pyfilters/models/__init__.py +76 -0
- firegex/pyfilters/models/http.py +1387 -0
- firegex/pyfilters/models/tcp.py +87 -0
- firegex/pyfilters/proxysim/__init__.py +295 -0
- firegex/regex/__init__.py +342 -0
- firegex/regex/sim.py +104 -0
- firegex-5.0.0.dist-info/METADATA +69 -0
- firegex-5.0.0.dist-info/RECORD +21 -0
- firegex-5.0.0.dist-info/WHEEL +5 -0
- firegex-5.0.0.dist-info/entry_points.txt +2 -0
- firegex-5.0.0.dist-info/top_level.txt +1 -0
firegex/__init__.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
|
|
2
|
+
def _get_version():
|
|
3
|
+
v = "5.0.0" if "{" not in "5.0.0" else "0.0.0"
|
|
4
|
+
if v == "0.0.0":
|
|
5
|
+
import os
|
|
6
|
+
env_v = os.getenv("FIREGEX_VERSION")
|
|
7
|
+
if env_v:
|
|
8
|
+
return env_v
|
|
9
|
+
try:
|
|
10
|
+
import re
|
|
11
|
+
|
|
12
|
+
def parse_version(ver):
|
|
13
|
+
ver = ver.lstrip("v")
|
|
14
|
+
parts = re.split(r'[^0-9]+', ver)
|
|
15
|
+
return tuple(int(p) if p.isdigit() else 0 for p in parts if p)
|
|
16
|
+
|
|
17
|
+
def find_git_dir(start_path):
|
|
18
|
+
current = start_path
|
|
19
|
+
while current and current != "/" and current != "":
|
|
20
|
+
gd = os.path.join(current, ".git")
|
|
21
|
+
if os.path.isdir(gd):
|
|
22
|
+
return gd
|
|
23
|
+
parent = os.path.dirname(current)
|
|
24
|
+
if parent == current:
|
|
25
|
+
break
|
|
26
|
+
current = parent
|
|
27
|
+
return None
|
|
28
|
+
|
|
29
|
+
git_dir = find_git_dir(os.path.abspath(os.path.dirname(__file__)))
|
|
30
|
+
if not git_dir:
|
|
31
|
+
return v
|
|
32
|
+
|
|
33
|
+
branch = None
|
|
34
|
+
with open(os.path.join(git_dir, "HEAD"), "r") as f:
|
|
35
|
+
head = f.read().strip()
|
|
36
|
+
if head.startswith("ref: refs/heads/"):
|
|
37
|
+
branch = head.split("ref: refs/heads/")[1]
|
|
38
|
+
|
|
39
|
+
if branch != "main":
|
|
40
|
+
return v
|
|
41
|
+
|
|
42
|
+
tags = []
|
|
43
|
+
try:
|
|
44
|
+
with open(os.path.join(git_dir, "packed-refs"), "r") as f:
|
|
45
|
+
for line in f:
|
|
46
|
+
if " refs/tags/" in line:
|
|
47
|
+
tags.append(line.strip().split(" refs/tags/")[1])
|
|
48
|
+
except Exception:
|
|
49
|
+
pass
|
|
50
|
+
|
|
51
|
+
tags_dir = os.path.join(git_dir, "refs", "tags")
|
|
52
|
+
try:
|
|
53
|
+
for root, _, files in os.walk(tags_dir):
|
|
54
|
+
for file in files:
|
|
55
|
+
tags.append(os.path.relpath(os.path.join(root, file), tags_dir).replace(os.path.sep, "/"))
|
|
56
|
+
except Exception:
|
|
57
|
+
pass
|
|
58
|
+
|
|
59
|
+
version_tags = [t for t in set(tags) if any(c.isdigit() for c in t)]
|
|
60
|
+
if version_tags:
|
|
61
|
+
return max(version_tags, key=parse_version)
|
|
62
|
+
except Exception:
|
|
63
|
+
pass
|
|
64
|
+
return v
|
|
65
|
+
|
|
66
|
+
__version__ = _get_version()
|
|
67
|
+
|
|
68
|
+
#Exported functions
|
|
69
|
+
__all__ = []
|
firegex/__main__.py
ADDED
|
Binary file
|
firegex/cli.py
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
|
|
2
|
+
#!/usr/bin/env python3
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import socket
|
|
7
|
+
import sys
|
|
8
|
+
|
|
9
|
+
import typer
|
|
10
|
+
from rich import print
|
|
11
|
+
from rich.markup import escape
|
|
12
|
+
from typer import Exit
|
|
13
|
+
|
|
14
|
+
from firegex import __version__
|
|
15
|
+
from firegex.pyfilters.proxysim import run_proxy_simulation
|
|
16
|
+
|
|
17
|
+
app = typer.Typer(
|
|
18
|
+
no_args_is_help=True,
|
|
19
|
+
context_settings={"help_option_names": ["-h", "--help"]}
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
def close_cli(code:int=1):
|
|
23
|
+
raise Exit(code)
|
|
24
|
+
|
|
25
|
+
DEV_MODE = __version__ == "0.0.0"
|
|
26
|
+
|
|
27
|
+
def test_connection(host, port, use_ipv6=False):
|
|
28
|
+
family = socket.AF_INET6 if use_ipv6 else socket.AF_INET
|
|
29
|
+
sock = socket.socket(family, socket.SOCK_STREAM)
|
|
30
|
+
|
|
31
|
+
try:
|
|
32
|
+
sock.settimeout(3)
|
|
33
|
+
sock.connect((host, port))
|
|
34
|
+
return True
|
|
35
|
+
except Exception:
|
|
36
|
+
return False
|
|
37
|
+
finally:
|
|
38
|
+
sock.close()
|
|
39
|
+
|
|
40
|
+
@app.command("pyfilters", help="Run your Python filters against a real service, locally")
|
|
41
|
+
def pyfilters(
|
|
42
|
+
filter_file: str = typer.Argument(..., help="The path to the filter file"),
|
|
43
|
+
address: str = typer.Argument(..., help="The address of the target to proxy"),
|
|
44
|
+
port: int = typer.Argument(..., help="The port of the target to proxy"),
|
|
45
|
+
|
|
46
|
+
from_address: str = typer.Option(None, help="The address of the local server"),
|
|
47
|
+
from_port: int = typer.Option(7474, help="The port of the local server"),
|
|
48
|
+
ipv6: bool = typer.Option(False, "-6", help="Use IPv6 for the connection"),
|
|
49
|
+
):
|
|
50
|
+
if from_address is None:
|
|
51
|
+
from_address = "::1" if ipv6 else "127.0.0.1"
|
|
52
|
+
if not os.path.isfile(filter_file):
|
|
53
|
+
print(f"[bold red]'{escape(os.path.abspath(filter_file))}' not found[/]")
|
|
54
|
+
close_cli()
|
|
55
|
+
if not test_connection(address, port, ipv6):
|
|
56
|
+
print(f"[bold red]Can't connect to {escape(address)}:{port}[/]")
|
|
57
|
+
close_cli()
|
|
58
|
+
# No protocol to pass: the filter file says which one it speaks by what it asks for.
|
|
59
|
+
run_proxy_simulation(filter_file, address, port, from_address, from_port, ipv6)
|
|
60
|
+
|
|
61
|
+
regex_app = typer.Typer(
|
|
62
|
+
no_args_is_help=True,
|
|
63
|
+
help="Try a regex ruleset with the engine firegex matches with",
|
|
64
|
+
)
|
|
65
|
+
app.add_typer(regex_app, name="regex")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _load_ruleset(path: str):
|
|
69
|
+
"""Read a ruleset, with the errors an operator can act on.
|
|
70
|
+
|
|
71
|
+
The file is the same shape firegex itself uses, so a ruleset can be written here and
|
|
72
|
+
pasted there, or exported from there and replayed here.
|
|
73
|
+
"""
|
|
74
|
+
from firegex.regex import HyperscanMissing, Rule, available, validate
|
|
75
|
+
|
|
76
|
+
if not available():
|
|
77
|
+
print(f"[bold red]{escape(str(HyperscanMissing()))}[/]")
|
|
78
|
+
close_cli()
|
|
79
|
+
if not os.path.isfile(path):
|
|
80
|
+
print(f"[bold red]'{escape(os.path.abspath(path))}' not found[/]")
|
|
81
|
+
close_cli()
|
|
82
|
+
try:
|
|
83
|
+
with open(path) as f:
|
|
84
|
+
raw = json.load(f)
|
|
85
|
+
except json.JSONDecodeError as e:
|
|
86
|
+
print(f"[bold red]{escape(path)} is not valid JSON: {escape(str(e))}[/]")
|
|
87
|
+
close_cli()
|
|
88
|
+
if not isinstance(raw, list):
|
|
89
|
+
print("[bold red]A ruleset is a list of rules[/]")
|
|
90
|
+
close_cli()
|
|
91
|
+
|
|
92
|
+
rules = []
|
|
93
|
+
broken = False
|
|
94
|
+
for index, item in enumerate(raw):
|
|
95
|
+
try:
|
|
96
|
+
rule = Rule.from_dict(item)
|
|
97
|
+
except (KeyError, ValueError) as e:
|
|
98
|
+
print(f"[bold red]rule {index} is malformed: {escape(str(e))}[/]")
|
|
99
|
+
broken = True
|
|
100
|
+
continue
|
|
101
|
+
# Judged against the mode it would actually run in, so what is accepted here is
|
|
102
|
+
# what firegex will accept.
|
|
103
|
+
why = validate(rule)
|
|
104
|
+
if why:
|
|
105
|
+
print(f"[bold red]{escape(rule.id)}: {escape(why)}[/]")
|
|
106
|
+
broken = True
|
|
107
|
+
continue
|
|
108
|
+
rules.append(rule)
|
|
109
|
+
if broken:
|
|
110
|
+
close_cli()
|
|
111
|
+
if not rules:
|
|
112
|
+
print("[bold red]The ruleset is empty[/]")
|
|
113
|
+
close_cli()
|
|
114
|
+
return rules
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
@regex_app.command("check", help="Check that a ruleset compiles, and say what is in it")
|
|
118
|
+
def regex_check(
|
|
119
|
+
rules_file: str = typer.Argument(..., help="The path to the ruleset (JSON)"),
|
|
120
|
+
):
|
|
121
|
+
rules = _load_ruleset(rules_file)
|
|
122
|
+
print(f"[bold green]{len(rules)} rule(s), all valid[/]")
|
|
123
|
+
for rule in rules:
|
|
124
|
+
case = "" if rule.case_sensitive else ", any case"
|
|
125
|
+
print(f" [bold]{escape(rule.id)}[/] /{escape(rule.pattern)}/ "
|
|
126
|
+
f"[dim]{rule.direction.value}{case}, block[/]")
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@regex_app.command("test", help="Run a ruleset over a sample and show what it would do")
|
|
130
|
+
def regex_test(
|
|
131
|
+
rules_file: str = typer.Argument(..., help="The path to the ruleset (JSON)"),
|
|
132
|
+
sample_file: str = typer.Option(None, "--sample", "-s",
|
|
133
|
+
help="File to match against; stdin when omitted"),
|
|
134
|
+
from_service: bool = typer.Option(False, "--from-service",
|
|
135
|
+
help="Treat the sample as traffic from the service"),
|
|
136
|
+
):
|
|
137
|
+
from firegex.regex import Ruleset
|
|
138
|
+
|
|
139
|
+
rules = _load_ruleset(rules_file)
|
|
140
|
+
if sample_file:
|
|
141
|
+
if not os.path.isfile(sample_file):
|
|
142
|
+
print(f"[bold red]'{escape(os.path.abspath(sample_file))}' not found[/]")
|
|
143
|
+
close_cli()
|
|
144
|
+
with open(sample_file, "rb") as f:
|
|
145
|
+
sample = f.read()
|
|
146
|
+
else:
|
|
147
|
+
sample = sys.stdin.buffer.read()
|
|
148
|
+
if not sample:
|
|
149
|
+
print("[bold red]Nothing to match against[/]")
|
|
150
|
+
close_cli()
|
|
151
|
+
|
|
152
|
+
ruleset = Ruleset(rules)
|
|
153
|
+
is_input = not from_service
|
|
154
|
+
blocked_by = ruleset.apply(sample, is_input)
|
|
155
|
+
|
|
156
|
+
# Said before the verdict, because it changes what the verdict means: "passed
|
|
157
|
+
# through" from a run that could not try every rule is not the same answer.
|
|
158
|
+
for rule_id, why in ruleset.unscannable.items():
|
|
159
|
+
print(f"[bold yellow]not tried here[/] [bold]{escape(rule_id)}[/]: {escape(why)}. "
|
|
160
|
+
f"It is valid and will run on a real service — only matching it against a "
|
|
161
|
+
f"sample needs a mode hyperscan does not offer for it.")
|
|
162
|
+
|
|
163
|
+
if blocked_by is not None:
|
|
164
|
+
print(f"[bold red]blocked[/] by [bold]{escape(blocked_by)}[/]")
|
|
165
|
+
close_cli(0)
|
|
166
|
+
print("[bold green]passed through[/]"
|
|
167
|
+
+ (" by the rules that could be tried" if ruleset.unscannable else ""))
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
@regex_app.command("proxy", help="Run a local proxy applying a ruleset, like `fgex pyfilters`")
|
|
171
|
+
def regex_proxy(
|
|
172
|
+
rules_file: str = typer.Argument(..., help="The path to the ruleset (JSON)"),
|
|
173
|
+
address: str = typer.Argument(..., help="The address of the target to proxy"),
|
|
174
|
+
port: int = typer.Argument(..., help="The port of the target to proxy"),
|
|
175
|
+
from_address: str = typer.Option(None, help="The address of the local server"),
|
|
176
|
+
from_port: int = typer.Option(7474, help="The port of the local server"),
|
|
177
|
+
ipv6: bool = typer.Option(False, "-6", help="Use IPv6 for the connection"),
|
|
178
|
+
):
|
|
179
|
+
from firegex.regex import Ruleset
|
|
180
|
+
from firegex.regex.sim import run_regex_simulation
|
|
181
|
+
|
|
182
|
+
rules = _load_ruleset(rules_file)
|
|
183
|
+
if from_address is None:
|
|
184
|
+
from_address = "::1" if ipv6 else "127.0.0.1"
|
|
185
|
+
if not test_connection(address, port, ipv6):
|
|
186
|
+
print(f"[bold red]Can't connect to {escape(address)}:{port}[/]")
|
|
187
|
+
close_cli()
|
|
188
|
+
print("[dim]Blocking matches per chunk here, but across the whole stream on a real "
|
|
189
|
+
"service: a pattern split over two reads is caught there and not here.[/]")
|
|
190
|
+
ruleset = Ruleset(rules)
|
|
191
|
+
# Built before the first connection rather than on it, so the warning is on screen
|
|
192
|
+
# while the operator is still reading the banner instead of appearing mid-traffic.
|
|
193
|
+
for is_input in (True, False):
|
|
194
|
+
ruleset._db(is_input)
|
|
195
|
+
for rule_id, why in ruleset.unscannable.items():
|
|
196
|
+
print(f"[bold yellow]not applied here[/] [bold]{escape(rule_id)}[/]: {escape(why)}. "
|
|
197
|
+
f"It is valid and will run on a real service.")
|
|
198
|
+
run_regex_simulation(ruleset, address, port, from_address, from_port, ipv6)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def version_callback(verison: bool):
|
|
202
|
+
if verison:
|
|
203
|
+
print(__version__, "Development Mode" if DEV_MODE else "Release")
|
|
204
|
+
raise typer.Exit()
|
|
205
|
+
|
|
206
|
+
@app.callback()
|
|
207
|
+
def main(
|
|
208
|
+
verison: bool = typer.Option(False, "--version", "-v", help="Show the version of the client", callback=version_callback),
|
|
209
|
+
):
|
|
210
|
+
pass
|
|
211
|
+
|
|
212
|
+
def run():
|
|
213
|
+
try:
|
|
214
|
+
app()
|
|
215
|
+
except KeyboardInterrupt:
|
|
216
|
+
print("[bold yellow]Operation cancelled[/]")
|
|
217
|
+
|
|
218
|
+
if __name__ == "__main__":
|
|
219
|
+
run()
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
from firegex.pyfilters.models import (
|
|
2
|
+
RawPacket,
|
|
3
|
+
TCPInputStream,
|
|
4
|
+
TCPOutputStream,
|
|
5
|
+
TCPClientStream,
|
|
6
|
+
TCPServerStream,
|
|
7
|
+
HttpHistory,
|
|
8
|
+
HttpStreamHistory,
|
|
9
|
+
)
|
|
10
|
+
from firegex.pyfilters.internals.models import Action, FullStreamAction, ExceptionAction
|
|
11
|
+
|
|
12
|
+
ACCEPT = Action.ACCEPT
|
|
13
|
+
DROP = Action.DROP
|
|
14
|
+
REJECT = Action.REJECT
|
|
15
|
+
|
|
16
|
+
#: Set on a function by `@pyfilter`. Marking the function *is* the registration — there
|
|
17
|
+
#: is no registry to keep, to clear between files, or to race two compilations against.
|
|
18
|
+
PYFILTER_MARK = "__firegex_pyfilter__"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def pyfilter(func):
|
|
22
|
+
"""Mark a function as a filter, and hand it back unchanged.
|
|
23
|
+
|
|
24
|
+
The mark lives on the function, so the file that defines it holds the whole list:
|
|
25
|
+
`collect_pyfilters` reads a module's globals in definition order. What that replaced
|
|
26
|
+
was a set of *names* on a module-global attribute of this decorator, which had three
|
|
27
|
+
consequences worth not having back — a set has no order, so the order filters ran in
|
|
28
|
+
was whatever hashing produced; the state outlived the file, so every caller had to
|
|
29
|
+
clear it before and after; and a name registered from somewhere the module globals
|
|
30
|
+
could not reach (a nested `def`) was registered and then not found.
|
|
31
|
+
|
|
32
|
+
The function is returned as it is, not wrapped. The wrapper it used to get forwarded
|
|
33
|
+
`*args` and did nothing else, at the cost of a stack frame in every traceback the
|
|
34
|
+
operator has to read.
|
|
35
|
+
"""
|
|
36
|
+
setattr(func, PYFILTER_MARK, True)
|
|
37
|
+
return func
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def collect_pyfilters(namespace: dict) -> list[str]:
|
|
41
|
+
"""The filters a module defines, in the order it defines them.
|
|
42
|
+
|
|
43
|
+
Definition order is the order they run in, which is the only order a reader of the
|
|
44
|
+
file can predict.
|
|
45
|
+
"""
|
|
46
|
+
seen, names = set(), []
|
|
47
|
+
for name, obj in namespace.items():
|
|
48
|
+
if callable(obj) and getattr(obj, PYFILTER_MARK, False) and id(obj) not in seen:
|
|
49
|
+
seen.add(id(obj))
|
|
50
|
+
names.append(name)
|
|
51
|
+
return names
|
|
52
|
+
|
|
53
|
+
__all__ = [
|
|
54
|
+
"ACCEPT", "DROP", "REJECT",
|
|
55
|
+
"Action", "FullStreamAction", "ExceptionAction", "pyfilter", "collect_pyfilters",
|
|
56
|
+
"RawPacket", "TCPInputStream", "TCPOutputStream", "TCPClientStream", "TCPServerStream",
|
|
57
|
+
"HttpHistory", "HttpStreamHistory"
|
|
58
|
+
]
|
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
from inspect import Parameter, signature
|
|
2
|
+
from itertools import product
|
|
3
|
+
from firegex.pyfilters.internals.models import Action, ExceptionAction, FullStreamAction
|
|
4
|
+
from firegex.pyfilters.internals.models import FilterHandler, PacketHandlerResult
|
|
5
|
+
import functools
|
|
6
|
+
from firegex.pyfilters.internals.data import DataStreamCtx
|
|
7
|
+
from firegex.pyfilters.internals.exceptions import NotReadyToRun, StreamFullReject, DropPacket, RejectConnection, StreamFullDrop
|
|
8
|
+
from firegex.pyfilters.internals.data import RawPacket
|
|
9
|
+
|
|
10
|
+
def _type_name(annotation) -> str:
|
|
11
|
+
return getattr(annotation, "__name__", None) or str(annotation)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _wanted_by(filters: list[str], glob: dict) -> list[tuple[str, object, list[object]]]:
|
|
15
|
+
"""`(name, func, annotations)` for each named filter, looked up once.
|
|
16
|
+
|
|
17
|
+
Both passes below need the same three things, and reading them twice is how the two
|
|
18
|
+
ended up disagreeing about what to say when an annotation is not a model — one
|
|
19
|
+
complained that no protocol provides it, the other that this file's protocol does
|
|
20
|
+
not. One reading, one message.
|
|
21
|
+
"""
|
|
22
|
+
found = []
|
|
23
|
+
for name in filters:
|
|
24
|
+
if not isinstance(name, str):
|
|
25
|
+
raise Exception("Invalid filter list: must be a list of strings")
|
|
26
|
+
func = glob.get(name)
|
|
27
|
+
if func is None:
|
|
28
|
+
raise Exception(f"Filter {name} not found")
|
|
29
|
+
if not callable(func):
|
|
30
|
+
raise Exception(f"{func} is not a function")
|
|
31
|
+
annotations = []
|
|
32
|
+
for param, spec in signature(func).parameters.items():
|
|
33
|
+
if spec.annotation is Parameter.empty:
|
|
34
|
+
# The library decides when to call a filter from what it asks for, so a
|
|
35
|
+
# parameter that asks for nothing leaves it with nothing to go on.
|
|
36
|
+
raise Exception(
|
|
37
|
+
f"Parameter {param!r} of {name} has no type annotation. Annotate it "
|
|
38
|
+
f"with what the filter wants to be given — RawPacket, a TCP stream, "
|
|
39
|
+
f"an HTTP model — because that is what decides when it is called."
|
|
40
|
+
)
|
|
41
|
+
annotations.append(spec.annotation)
|
|
42
|
+
found.append((name, func, annotations))
|
|
43
|
+
return found
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def infer_proto(wanted: list[tuple[str, object, list[object]]]) -> str:
|
|
47
|
+
"""Which application protocol a file speaks, read off what it asks for.
|
|
48
|
+
|
|
49
|
+
A filter file does not declare its protocol; it shows it. Asking for an
|
|
50
|
+
`HttpRequest` is what makes a file an HTTP filter, and asking only for a
|
|
51
|
+
`RawPacket` or a TCP stream is what makes one protocol-agnostic — so the same file
|
|
52
|
+
can hold both, and a TCP service can carry a filter that parses HTTP alongside one
|
|
53
|
+
that does not.
|
|
54
|
+
|
|
55
|
+
The one thing that cannot work is a file asking for two *different* application
|
|
56
|
+
protocols, because a connection is only ever one of them. That is refused here,
|
|
57
|
+
naming both functions, rather than being resolved by picking a winner.
|
|
58
|
+
"""
|
|
59
|
+
from firegex.pyfilters.models import type_annotations_associations
|
|
60
|
+
|
|
61
|
+
candidates = set(type_annotations_associations)
|
|
62
|
+
# What actually narrowed the choice, kept so a conflict can be explained in terms
|
|
63
|
+
# of the operator's own code instead of a protocol name they never typed.
|
|
64
|
+
narrowing: list[tuple[str, object, set[str]]] = []
|
|
65
|
+
|
|
66
|
+
for name, _func, annotations in wanted:
|
|
67
|
+
for annotation in annotations:
|
|
68
|
+
protos = {
|
|
69
|
+
proto
|
|
70
|
+
for proto, types in type_annotations_associations.items()
|
|
71
|
+
if annotation in types
|
|
72
|
+
}
|
|
73
|
+
if not protos:
|
|
74
|
+
known = sorted({
|
|
75
|
+
_type_name(t)
|
|
76
|
+
for types in type_annotations_associations.values()
|
|
77
|
+
for t in types
|
|
78
|
+
})
|
|
79
|
+
raise Exception(
|
|
80
|
+
f"Invalid type annotation {_type_name(annotation)} for function "
|
|
81
|
+
f"{name}: no protocol provides it. Available: " + ", ".join(known)
|
|
82
|
+
)
|
|
83
|
+
if not candidates & protos:
|
|
84
|
+
clash = next((n for n in narrowing if not n[2] & protos), None)
|
|
85
|
+
if clash:
|
|
86
|
+
raise Exception(
|
|
87
|
+
f"{name} asks for {_type_name(annotation)}, which only "
|
|
88
|
+
f"{'/'.join(sorted(protos))} provides, while {clash[0]} asks for "
|
|
89
|
+
f"{_type_name(clash[1])}, which only "
|
|
90
|
+
f"{'/'.join(sorted(clash[2]))} does. A connection speaks one "
|
|
91
|
+
f"application protocol, so one file cannot filter both: split "
|
|
92
|
+
f"them into two filters on the same service."
|
|
93
|
+
)
|
|
94
|
+
raise Exception(
|
|
95
|
+
f"{name} asks for {_type_name(annotation)}, which no protocol in "
|
|
96
|
+
f"this file can provide"
|
|
97
|
+
)
|
|
98
|
+
if candidates & protos != candidates:
|
|
99
|
+
narrowing.append((name, annotation, protos))
|
|
100
|
+
candidates &= protos
|
|
101
|
+
|
|
102
|
+
return simplest_proto(candidates)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def simplest_proto(candidates=None) -> str:
|
|
106
|
+
"""The least specific protocol among `candidates` — all of them by default.
|
|
107
|
+
|
|
108
|
+
A file that only wants raw payloads is a TCP filter, not an HTTP one that happens
|
|
109
|
+
never to parse, so "fewest models" is what "simplest" means.
|
|
110
|
+
"""
|
|
111
|
+
from firegex.pyfilters.models import type_annotations_associations
|
|
112
|
+
return min(
|
|
113
|
+
candidates if candidates else type_annotations_associations,
|
|
114
|
+
key=lambda p: (len(type_annotations_associations[p]), p),
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def generate_filter_structure(filters: list[str], proto: str | None, glob: dict) -> list[FilterHandler]:
|
|
119
|
+
"""One `FilterHandler` per filter, in the order given, for one protocol."""
|
|
120
|
+
from firegex.pyfilters.models import type_annotations_associations
|
|
121
|
+
|
|
122
|
+
wanted = _wanted_by(filters, glob)
|
|
123
|
+
if proto is None:
|
|
124
|
+
proto = infer_proto(wanted)
|
|
125
|
+
if proto not in type_annotations_associations:
|
|
126
|
+
raise Exception("Invalid protocol")
|
|
127
|
+
provides = type_annotations_associations[proto]
|
|
128
|
+
|
|
129
|
+
handlers = []
|
|
130
|
+
for name, func, annotations in wanted:
|
|
131
|
+
for annotation in annotations:
|
|
132
|
+
if annotation not in provides:
|
|
133
|
+
raise Exception(
|
|
134
|
+
f"Invalid type annotation {_type_name(annotation)} for function "
|
|
135
|
+
f"{name}: this file speaks {proto}, which does not provide it"
|
|
136
|
+
)
|
|
137
|
+
handlers.append(FilterHandler(
|
|
138
|
+
func=func,
|
|
139
|
+
name=func.__name__,
|
|
140
|
+
params={a: provides[a] for a in annotations},
|
|
141
|
+
proto=proto,
|
|
142
|
+
))
|
|
143
|
+
return handlers
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def get_filters_info(code:str, proto:str|None = None) -> list[FilterHandler]:
|
|
147
|
+
"""The filters a file defines. `proto` is inferred from the code when not given.
|
|
148
|
+
|
|
149
|
+
Only the operator's own source is `exec`ed. Everything around it used to be too —
|
|
150
|
+
the import, the registry reset before and after — which was a string compiled at
|
|
151
|
+
runtime for each step of what a plain call does, and which existed only because the
|
|
152
|
+
registry was global state that had to be cleared around every use.
|
|
153
|
+
"""
|
|
154
|
+
from firegex.pyfilters import collect_pyfilters
|
|
155
|
+
|
|
156
|
+
glob: dict = {}
|
|
157
|
+
exec(code, glob, glob)
|
|
158
|
+
return generate_filter_structure(collect_pyfilters(glob), proto, glob)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def get_filter_names(code:str, proto:str|None = None) -> list[str]:
|
|
162
|
+
return [ele.name for ele in get_filters_info(code, proto)]
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def get_code_proto(code:str) -> str:
|
|
166
|
+
"""Which application protocol a file speaks, for a caller that needs to say so.
|
|
167
|
+
|
|
168
|
+
Raises the same errors `get_filters_info` does, so a file that mixes two protocols
|
|
169
|
+
is reported here with the reason rather than silently classified as one of them.
|
|
170
|
+
"""
|
|
171
|
+
from firegex.pyfilters.models import type_annotations_associations
|
|
172
|
+
infos = get_filters_info(code)
|
|
173
|
+
if infos:
|
|
174
|
+
return infos[0].proto
|
|
175
|
+
# A file with no filters yet speaks the simplest protocol there is: it asks for
|
|
176
|
+
# nothing, so nothing is ruled out.
|
|
177
|
+
return min(
|
|
178
|
+
type_annotations_associations,
|
|
179
|
+
key=lambda p: (len(type_annotations_associations[p]), p),
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
#: What a model raises while being built, and the verdict it means. `NotReadyToRun` is
|
|
183
|
+
#: not here: it is the ordinary "there is nothing to give this filter yet", handled by
|
|
184
|
+
#: skipping the call rather than by ending the packet.
|
|
185
|
+
#: Each is credited to what actually decided it, never to the filter function: those two
|
|
186
|
+
#: are raised only when the parser could not read the traffic, and a function the
|
|
187
|
+
#: operator wrote was listed as having refused a connection it was never shown.
|
|
188
|
+
_RAISED_VERDICT = {
|
|
189
|
+
StreamFullDrop: (Action.DROP, "@MAX_STREAM_SIZE_REACHED"),
|
|
190
|
+
StreamFullReject: (Action.REJECT, "@MAX_STREAM_SIZE_REACHED"),
|
|
191
|
+
DropPacket: (Action.DROP, "@INVALID_ENCODING"),
|
|
192
|
+
RejectConnection: (Action.REJECT, "@INVALID_ENCODING"),
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def handle_packet(glob: dict) -> None:
|
|
197
|
+
"""Run every enabled filter over the chunk in `glob`, and leave a verdict there.
|
|
198
|
+
|
|
199
|
+
The first filter to answer anything but ACCEPT ends the packet — every verdict
|
|
200
|
+
there is now terminal. `UNSTABLE_MANGLE` used to be the exception: it was remembered
|
|
201
|
+
and the walk carried on, because a later filter might still refuse the rewritten
|
|
202
|
+
chunk. It is gone, and so is that branch: see the note on `Action` in `models.py`.
|
|
203
|
+
"""
|
|
204
|
+
internal_data = DataStreamCtx(glob)
|
|
205
|
+
result = PacketHandlerResult(glob)
|
|
206
|
+
|
|
207
|
+
# Built once per packet and shared by every filter that asks for the same thing: two
|
|
208
|
+
# filters both taking an `HttpRequest` parse the chunk once between them.
|
|
209
|
+
built = {RawPacket: internal_data.current_pkt}
|
|
210
|
+
|
|
211
|
+
for filter in internal_data.filter_call_info:
|
|
212
|
+
args = []
|
|
213
|
+
for data_type, build in filter.params.items():
|
|
214
|
+
if data_type not in built:
|
|
215
|
+
try:
|
|
216
|
+
built[data_type] = build(internal_data)
|
|
217
|
+
except NotReadyToRun:
|
|
218
|
+
built[data_type] = None
|
|
219
|
+
except tuple(_RAISED_VERDICT) as raised:
|
|
220
|
+
action, matched_by = _RAISED_VERDICT[type(raised)]
|
|
221
|
+
result.action = action
|
|
222
|
+
result.matched_by = matched_by or filter.name
|
|
223
|
+
return result.set_result()
|
|
224
|
+
if built[data_type] is None:
|
|
225
|
+
args = None # nothing to give this filter yet; it is not called
|
|
226
|
+
break
|
|
227
|
+
args.append(built[data_type])
|
|
228
|
+
|
|
229
|
+
if args is None:
|
|
230
|
+
continue
|
|
231
|
+
|
|
232
|
+
# A model may hand back several of itself — a chunk carrying three HTTP requests
|
|
233
|
+
# is three calls, not one call with a list. Where more than one parameter does
|
|
234
|
+
# that, the filter sees every combination.
|
|
235
|
+
for call_args in product(*(a if isinstance(a, list) else [a] for a in args)):
|
|
236
|
+
res = filter.func(*call_args)
|
|
237
|
+
if res is None or res == Action.ACCEPT:
|
|
238
|
+
continue
|
|
239
|
+
if not isinstance(res, Action):
|
|
240
|
+
raise Exception(f"Invalid return type {type(res)} for function {filter.name}")
|
|
241
|
+
result.matched_by = filter.name
|
|
242
|
+
result.action = res
|
|
243
|
+
return result.set_result()
|
|
244
|
+
|
|
245
|
+
return result.set_result() # nothing refused it: ACCEPT
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
#: The knobs a filter file may set, and what each one is kept as. Three copies of "if
|
|
249
|
+
#: the name is there and the value is the right type, store it" differing only in the
|
|
250
|
+
#: name and the type; the one that was *not* a copy had drifted — it checked `Action`
|
|
251
|
+
#: where the setter and the documentation both said `ExceptionAction`, so the knob could
|
|
252
|
+
#: not be set at all, in either direction.
|
|
253
|
+
_SETTINGS = {
|
|
254
|
+
"FGEX_STREAM_MAX_SIZE": ("stream_max_size", int),
|
|
255
|
+
"FGEX_FULL_STREAM_ACTION": ("full_stream_action", FullStreamAction),
|
|
256
|
+
"FGEX_INVALID_ENCODING_ACTION": ("invalid_encoding_action", ExceptionAction),
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def compile(glob: dict) -> None:
|
|
261
|
+
"""Prepare a filter file's globals to be handed packets.
|
|
262
|
+
|
|
263
|
+
Called once per connection, after the operator's source has been executed into
|
|
264
|
+
`glob`: it resolves which functions run, in what order, over what models, and reads
|
|
265
|
+
the file's settings.
|
|
266
|
+
"""
|
|
267
|
+
internal_data = DataStreamCtx(glob, init_pkt=False)
|
|
268
|
+
|
|
269
|
+
glob["print"] = functools.partial(print, flush=True)
|
|
270
|
+
|
|
271
|
+
# Which functions run: `None` for every one the file defines, a list for exactly
|
|
272
|
+
# those of them. A name the file no longer defines is passed over rather than
|
|
273
|
+
# refusing the whole file — a switched-off function deleted from the code afterwards
|
|
274
|
+
# would otherwise stop every other filter in it from loading. Worked out here, in the
|
|
275
|
+
# process that has just run the module, so the caller never has to run it to ask.
|
|
276
|
+
from firegex.pyfilters import collect_pyfilters
|
|
277
|
+
defined = collect_pyfilters(glob)
|
|
278
|
+
enabled = glob.get("__firegex_pyfilter_enabled")
|
|
279
|
+
if enabled is None:
|
|
280
|
+
names = defined
|
|
281
|
+
else:
|
|
282
|
+
wanted = set(enabled)
|
|
283
|
+
names = [name for name in defined if name in wanted]
|
|
284
|
+
# Absent means "read it off the code", which is the normal case: the file shows
|
|
285
|
+
# which protocol it speaks by what its filters ask for, so nothing has to be kept
|
|
286
|
+
# in step with it by hand.
|
|
287
|
+
internal_data.filter_call_info = generate_filter_structure(
|
|
288
|
+
names, glob.get("__firegex_proto"), glob
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
for name, (attribute, kind) in _SETTINGS.items():
|
|
292
|
+
if name not in glob:
|
|
293
|
+
continue
|
|
294
|
+
value = glob[name]
|
|
295
|
+
if kind is int:
|
|
296
|
+
# A size is taken from anything that reads as one, and anything else is
|
|
297
|
+
# ignored — which is what the other two settings already did, and what this
|
|
298
|
+
# one only claimed to do: `int(value)` raises for a typo, so a file setting
|
|
299
|
+
# `FGEX_STREAM_MAX_SIZE = "1MB"` did not get the default, it failed to load
|
|
300
|
+
# altogether. The operator lost every filter in that file over an optional
|
|
301
|
+
# knob, and the message named neither the setting nor the file.
|
|
302
|
+
try:
|
|
303
|
+
value = int(value)
|
|
304
|
+
except (TypeError, ValueError):
|
|
305
|
+
continue
|
|
306
|
+
if value <= 0:
|
|
307
|
+
continue
|
|
308
|
+
elif not isinstance(value, kind):
|
|
309
|
+
continue
|
|
310
|
+
setattr(internal_data, attribute, value)
|
|
311
|
+
|
|
312
|
+
PacketHandlerResult(glob).reset_result()
|
|
313
|
+
|
|
314
|
+
def fake_exit(*_a, **_k):
|
|
315
|
+
print("WARNING: This function should not be called", flush=True)
|
|
316
|
+
|
|
317
|
+
glob["exit"] = fake_exit
|