firegex 3.0.0__tar.gz → 3.2.0__tar.gz

{firegex-3.0.0/firegex.egg-info → firegex-3.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: firegex
-Version: 3.0.0
+Version: 3.2.0
 Summary: Firegex client
 Home-page: https://github.com/pwnzer0tt1/firegex
 Author: Pwnzer0tt1
@@ -13,8 +13,11 @@ Description-Content-Type: text/markdown
 Requires-Dist: typer==0.15.2
 Requires-Dist: pydantic>=2
 Requires-Dist: typing-extensions>=4.7.1
+Requires-Dist: zstd
+Requires-Dist: brotli
 Requires-Dist: watchfiles
 Requires-Dist: fgex
+Requires-Dist: websockets
 Requires-Dist: pyllhttp
 Dynamic: author
 Dynamic: author-email
@@ -122,7 +125,6 @@ This handler will be called twice: one for the request headers and one for the r
 - headers: dict - The headers of the request
 - user_agent: str - The user agent of the request
 - content_encoding: str - The content encoding of the request
-- has_begun: bool - It's true if the request has begun
 - body: bytes - The body of the request
 - headers_complete: bool - It's true if the headers are complete
 - message_complete: bool - It's true if the message is complete
@@ -150,7 +152,6 @@ This handler will be called twice: one for the response headers and one for the
 - headers: dict - The headers of the response
 - user_agent: str - The user agent of the response
 - content_encoding: str - The content encoding of the response
-- has_begun: bool - It's true if the response has begun
 - body: bytes - The body of the response
 - headers_complete: bool - It's true if the headers are complete
 - message_complete: bool - It's true if the message is complete

{firegex-3.0.0 → firegex-3.2.0}/README.md

@@ -94,7 +94,6 @@ This handler will be called twice: one for the request headers and one for the r
 - headers: dict - The headers of the request
 - user_agent: str - The user agent of the request
 - content_encoding: str - The content encoding of the request
-- has_begun: bool - It's true if the request has begun
 - body: bytes - The body of the request
 - headers_complete: bool - It's true if the headers are complete
 - message_complete: bool - It's true if the message is complete
@@ -122,7 +121,6 @@ This handler will be called twice: one for the response headers and one for the
 - headers: dict - The headers of the response
 - user_agent: str - The user agent of the response
 - content_encoding: str - The content encoding of the response
-- has_begun: bool - It's true if the response has begun
 - body: bytes - The body of the response
 - headers_complete: bool - It's true if the headers are complete
 - message_complete: bool - It's true if the message is complete

firegex-3.2.0/firegex/__init__.py

@@ -0,0 +1,5 @@
+
+__version__ = "3.2.0" if "{" not in "3.2.0" else "0.0.0"
+
+#Exported functions
+__all__ = []

{firegex-3.0.0 → firegex-3.2.0}/firegex/nfproxy/internals/__init__.py

@@ -75,12 +75,9 @@ def handle_packet(glob: dict) -> None:
     
     cache_call = {} # Cache of the data handler calls
     cache_call[RawPacket] = internal_data.current_pkt
-    
-    final_result = Action.ACCEPT
+
     result = PacketHandlerResult(glob)
     
-    func_name = None
-    mangled_packet = None
     for filter in internal_data.filter_call_info:
         final_params = []
         skip_call = False
@@ -116,24 +113,36 @@ def handle_packet(glob: dict) -> None:
         if skip_call:
             continue
         
-        res = context_call(glob, filter.func, *final_params)
-        
-        if res is None:
-            continue #ACCEPTED
-        if not isinstance(res, Action):
-            raise Exception(f"Invalid return type {type(res)} for function {filter.name}")
-        if res == Action.MANGLE:
-            mangled_packet = internal_data.current_pkt.raw_packet
-        if res != Action.ACCEPT:
-            func_name = filter.name
-            final_result = res
-            break
-    
-    result.action = final_result
-    result.matched_by = func_name
-    result.mangled_packet = mangled_packet
+        # Create an iterator with all the calls to be done
+        def try_to_call(params:list):
+            is_base_call = True
+            for i in range(len(params)):
+                if isinstance(params[i], list):
+                    new_params = params.copy()
+                    for ele in params[i]:
+                        new_params[i] = ele
+                        yield from try_to_call(new_params)
+                    is_base_call = False
+                    break
+            if is_base_call:
+                yield context_call(glob, filter.func, *params)
+                    
+        for res in try_to_call(final_params):
+            if res is None:
+                continue #ACCEPTED
+            if not isinstance(res, Action):
+                raise Exception(f"Invalid return type {type(res)} for function {filter.name}")
+            if res == Action.MANGLE:
+                result.matched_by = filter.name
+                result.mangled_packet = internal_data.current_pkt.raw_packet
+                result.action = Action.MANGLE
+            elif res != Action.ACCEPT:
+                result.matched_by = filter.name
+                result.action = res
+                result.mangled_packet = None
+                return result.set_result()
     
-    return result.set_result()
+    return result.set_result() # Will be MANGLE or ACCEPT
 
 
 def compile(glob:dict) -> None:
@@ -148,13 +157,17 @@ def compile(glob:dict) -> None:
 
     if "FGEX_STREAM_MAX_SIZE" in glob and int(glob["FGEX_STREAM_MAX_SIZE"]) > 0:
         internal_data.stream_max_size = int(glob["FGEX_STREAM_MAX_SIZE"])
-    else:
-        internal_data.stream_max_size = 1*8e20 # 1MB default value
     
     if "FGEX_FULL_STREAM_ACTION" in glob and isinstance(glob["FGEX_FULL_STREAM_ACTION"], FullStreamAction):
         internal_data.full_stream_action = glob["FGEX_FULL_STREAM_ACTION"]
-    else:
-        internal_data.full_stream_action = FullStreamAction.FLUSH
+    
+    if "FGEX_INVALID_ENCODING_ACTION" in glob and isinstance(glob["FGEX_INVALID_ENCODING_ACTION"], Action):
+        internal_data.invalid_encoding_action = glob["FGEX_INVALID_ENCODING_ACTION"]
     
     PacketHandlerResult(glob).reset_result()
+    
+    def fake_exit(*_a, **_k):
+        print("WARNING: This function should not be called", flush=True)
+    
+    glob["exit"] = fake_exit
 

{firegex-3.0.0 → firegex-3.2.0}/firegex/nfproxy/internals/data.py

@@ -1,5 +1,5 @@
 from firegex.nfproxy.internals.models import FilterHandler
-from firegex.nfproxy.internals.models import FullStreamAction
+from firegex.nfproxy.internals.models import FullStreamAction, ExceptionAction
 
 class RawPacket:
     "class rapresentation of the nfqueue packet sent in python context by the c++ core"
@@ -120,23 +120,39 @@ class DataStreamCtx:
     @property
     def stream_max_size(self) -> int:
         if "stream_max_size" not in self.__data.keys():
-            self.__data["stream_max_size"] = 1*8e20
+            self.__data["stream_max_size"] = 1*8e20 # 1MB default value
         return self.__data.get("stream_max_size")
     
     @stream_max_size.setter
     def stream_max_size(self, v: int):
+        if not isinstance(v, int):
+            raise Exception("Invalid data type, data MUST be of type int")
         self.__data["stream_max_size"] = v
     
     @property
     def full_stream_action(self) -> FullStreamAction:
         if "full_stream_action" not in self.__data.keys():
-            self.__data["full_stream_action"] = "flush"
+            self.__data["full_stream_action"] = FullStreamAction.FLUSH
         return self.__data.get("full_stream_action")
     
     @full_stream_action.setter
     def full_stream_action(self, v: FullStreamAction):
+        if not isinstance(v, FullStreamAction):
+            raise Exception("Invalid data type, data MUST be of type FullStreamAction")
         self.__data["full_stream_action"] = v
     
+    @property
+    def invalid_encoding_action(self) -> ExceptionAction:
+        if "invalid_encoding_action" not in self.__data.keys():
+            self.__data["invalid_encoding_action"] = ExceptionAction.REJECT
+        return self.__data.get("invalid_encoding_action")
+
+    @invalid_encoding_action.setter
+    def invalid_encoding_action(self, v: ExceptionAction):
+        if not isinstance(v, ExceptionAction):
+            raise Exception("Invalid data type, data MUST be of type ExceptionAction")
+        self.__data["invalid_encoding_action"] = v
+    
     @property
     def data_handler_context(self) -> dict:
         if "data_handler_context" not in self.__data.keys():

{firegex-3.0.0 → firegex-3.2.0}/firegex/nfproxy/internals/exceptions.py

@@ -13,3 +13,4 @@ class RejectConnection(Exception):
 
 class StreamFullReject(Exception):
     "raise this exception if you want to reject the connection due to full stream"
+

{firegex-3.0.0 → firegex-3.2.0}/firegex/nfproxy/internals/models.py

@@ -8,6 +8,13 @@ class Action(Enum):
     REJECT = 2
     MANGLE = 3
 
+class ExceptionAction(Enum):
+    """Action to be taken by the filter when an exception occurs (used in some cases)"""
+    ACCEPT = 0
+    DROP = 1
+    REJECT = 2
+    NOACTION = 3
+
 class FullStreamAction(Enum):
     """Action to be taken by the filter when the stream is full"""
     FLUSH = 0
@@ -40,5 +47,3 @@ class PacketHandlerResult:
     
     def reset_result(self) -> None:
         self.glob["__firegex_pyfilter_result"] = None
-
-

firegex-3.2.0/firegex/nfproxy/models/http.py

@@ -0,0 +1,591 @@
+import pyllhttp         
+from firegex.nfproxy.internals.exceptions import NotReadyToRun
+from firegex.nfproxy.internals.data import DataStreamCtx
+from firegex.nfproxy.internals.exceptions import StreamFullDrop, StreamFullReject, RejectConnection, DropPacket
+from firegex.nfproxy.internals.models import FullStreamAction, ExceptionAction
+from dataclasses import dataclass, field
+from collections import deque
+from typing import Type
+from zstd import ZSTD_uncompress
+import gzip
+import io
+import zlib
+import brotli
+from websockets.frames import Frame
+from websockets.extensions.permessage_deflate import PerMessageDeflate
+from pyllhttp import PAUSED_H2_UPGRADE, PAUSED_UPGRADE
+
+@dataclass
+class InternalHTTPMessage:
+    """Internal class to handle HTTP messages"""
+    url: str|None = field(default=None)
+    headers: dict[str, str] = field(default_factory=dict)
+    lheaders: dict[str, str] = field(default_factory=dict) # lowercase copy of the headers
+    body: bytes|None = field(default=None)
+    body_decoded: bool = field(default=False)
+    headers_complete: bool = field(default=False)
+    message_complete: bool = field(default=False)
+    status: str|None = field(default=None)
+    total_size: int = field(default=0)
+    user_agent: str = field(default_factory=str)
+    content_encoding: str = field(default=str)
+    content_type: str = field(default=str)
+    keep_alive: bool = field(default=False)
+    should_upgrade: bool = field(default=False)
+    http_version: str = field(default=str)
+    method: str = field(default=str)
+    content_length: int = field(default=0)
+    stream: bytes = field(default_factory=bytes)
+    ws_stream: list[Frame] = field(default_factory=list) # Decoded websocket stream
+    upgrading_to_h2: bool = field(default=False)
+    upgrading_to_ws: bool = field(default=False)
+
+@dataclass
+class InternalHttpBuffer:
+    """Internal class to handle HTTP messages"""
+    _url_buffer: bytes = field(default_factory=bytes)
+    _raw_header_fields: dict[str, str|list[str]] = field(default_factory=dict)
+    _header_fields: dict[str, str] = field(default_factory=dict)
+    _body_buffer: bytes = field(default_factory=bytes)
+    _status_buffer: bytes = field(default_factory=bytes)
+    _current_header_field: bytes = field(default_factory=bytes)
+    _current_header_value: bytes = field(default_factory=bytes)
+    _ws_packet_stream: bytes = field(default_factory=bytes)
+
+class InternalCallbackHandler():
+    
+    buffers = InternalHttpBuffer()
+    msg = InternalHTTPMessage()
+    save_body = True
+    raised_error = False
+    has_begun = False
+    messages: deque[InternalHTTPMessage] = deque()
+    _ws_extentions = None
+    _ws_raised_error = False
+
+    def reset_data(self):
+        self.msg = InternalHTTPMessage()
+        self.buffers = InternalHttpBuffer()
+        self.messages.clear()
+
+    def on_message_begin(self):
+        self.buffers = InternalHttpBuffer()
+        self.msg = InternalHTTPMessage()
+        self.has_begun = True
+    
+    def on_url(self, url):
+        self.buffers._url_buffer += url
+        self.msg.total_size += len(url)
+    
+    def on_url_complete(self):
+        self.msg.url = self.buffers._url_buffer.decode(errors="ignore")
+        self.buffers._url_buffer = b""
+
+    def on_status(self, status: bytes):
+        self.msg.total_size += len(status)
+        self.buffers._status_buffer += status
+    
+    def on_status_complete(self):
+        self.msg.status = self.buffers._status_buffer.decode(errors="ignore")
+        self.buffers._status_buffer = b""
+    
+    def on_header_field(self, field):
+        self.msg.total_size += len(field)
+        self.buffers._current_header_field += field
+
+    def on_header_field_complete(self):
+        pass # Nothing to do
+        
+    def on_header_value(self, value):
+        self.msg.total_size += len(value)
+        self.buffers._current_header_value += value
+
+    def on_header_value_complete(self):
+        if self.buffers._current_header_field:
+            k, v = self.buffers._current_header_field.decode(errors="ignore"), self.buffers._current_header_value.decode(errors="ignore")
+            old_value = self.buffers._raw_header_fields.get(k, None)
+            
+            # raw headers are stored as thay were, considering to check changes between headers encoding
+            if isinstance(old_value, list): 
+                old_value.append(v)
+            elif isinstance(old_value, str):
+                self.buffers._raw_header_fields[k] = [old_value, v]
+            else:
+                self.buffers._raw_header_fields[k] = v
+            
+            # Decoding headers normally
+            kl = k.lower()
+            if kl in self.buffers._header_fields:
+                self.buffers._header_fields[kl] += f", {v}" # Should be considered as a single list separated by commas as said in the RFC
+            else:
+                self.buffers._header_fields[kl] = v
+            
+        self.buffers._current_header_field = b""
+        self.buffers._current_header_value = b""
+    
+    def on_headers_complete(self):
+        self.msg.headers = self.buffers._raw_header_fields
+        self.msg.lheaders = self.buffers._header_fields
+        self.buffers._raw_header_fields = {}
+        self.buffers._current_header_field = b""
+        self.buffers._current_header_value = b""
+        self.msg.headers_complete = True
+        self.msg.method = self.method_parsed
+        self.msg.content_length = self.content_length_parsed
+        self.msg.should_upgrade = self.should_upgrade
+        self.msg.keep_alive = self.keep_alive
+        self.msg.http_version = self.http_version
+        self.msg.content_type = self.content_type
+        self.msg.content_encoding = self.content_encoding
+        self.msg.user_agent = self.user_agent
+
+    def on_body(self, body: bytes):
+        if self.save_body:
+            self.msg.total_size += len(body)
+            self.buffers._body_buffer += body
+
+    def on_message_complete(self):
+        self.msg.body = self.buffers._body_buffer
+        self.msg.should_upgrade = self.should_upgrade
+        self.buffers._body_buffer = b""
+        encodings = [ele.strip() for ele in self.content_encoding.lower().split(",")]
+        decode_success = True
+        decoding_body = self.msg.body
+        for enc in reversed(encodings):
+            if not enc:
+                continue
+            if enc == "deflate":
+                try:
+                    decompress = zlib.decompressobj(-zlib.MAX_WBITS)
+                    decoding_body = decompress.decompress(decoding_body)
+                    decoding_body += decompress.flush()
+                except Exception as e:
+                    print(f"Error decompressing deflate: {e}: skipping", flush=True)
+                    decode_success = False
+                    break
+            elif enc == "br":
+                try:
+                    decoding_body = brotli.decompress(decoding_body)
+                except Exception as e:
+                    print(f"Error decompressing brotli: {e}: skipping", flush=True)
+                    decode_success = False
+                    break
+            elif enc == "gzip" or enc == "x-gzip": #https://datatracker.ietf.org/doc/html/rfc2616#section-3.5      
+                try:
+                    if "gzip" in self.content_encoding.lower():
+                        with gzip.GzipFile(fileobj=io.BytesIO(decoding_body)) as f:
+                            decoding_body = f.read()
+                except Exception as e:
+                    print(f"Error decompressing gzip: {e}: skipping", flush=True)
+                    decode_success = False
+                    break
+            elif enc == "zstd":
+                try:
+                    decoding_body = ZSTD_uncompress(decoding_body)
+                except Exception as e:
+                    print(f"Error decompressing zstd: {e}: skipping", flush=True)
+                    decode_success = False
+                    break
+            elif enc == "identity":
+                pass # No need to do anything https://datatracker.ietf.org/doc/html/rfc2616#section-3.5 (it's possible to be found also if it should't be used)
+            else:
+                decode_success = False
+                break
+        
+        if decode_success:
+            self.msg.body = decoding_body
+            self.msg.body_decoded = True
+
+        self.msg.message_complete = True
+        self.has_begun = False
+        if not self._packet_to_stream():
+            self.messages.append(self.msg)
+    
+    @property
+    def user_agent(self) -> str:
+        return self.msg.lheaders.get("user-agent", "")
+    
+    @property
+    def content_encoding(self) -> str:
+        return self.msg.lheaders.get("content-encoding", "")
+    
+    @property
+    def content_type(self) -> str:
+        return self.msg.lheaders.get("content-type", "")
+
+    @property
+    def keep_alive(self) -> bool:
+        return self.should_keep_alive
+
+    @property
+    def should_upgrade(self) -> bool:
+        return self.is_upgrading
+    
+    @property
+    def http_version(self) -> str:
+        if self.major and self.minor:
+            return f"{self.major}.{self.minor}"
+        else:
+            return ""
+    
+    @property
+    def method_parsed(self) -> str:
+        return self.method
+
+    @property
+    def total_size(self) -> int:
+        """Total size used by the parser"""
+        tot = self.msg.total_size
+        for msg in self.messages:
+            tot += msg.total_size
+        return tot
+    
+    @property
+    def content_length_parsed(self) -> int:
+        return self.content_length
+    
+    def _is_input(self) -> bool:
+        raise NotImplementedError()
+    
+    def _packet_to_stream(self):
+        return self.should_upgrade and self.save_body
+    
+    def _stream_parser(self, data: bytes):
+        if self.msg.upgrading_to_ws:
+            if self._ws_raised_error:
+                self.msg.stream += data
+                self.msg.total_size += len(data)
+                return
+            self.buffers._ws_packet_stream += data
+            while True:
+                try:
+                    new_frame, self.buffers._ws_packet_stream = self._parse_websocket_frame(self.buffers._ws_packet_stream)
+                except Exception as e:
+                    self._ws_raised_error = True
+                    self.msg.stream += self.buffers._ws_packet_stream
+                    self.buffers._ws_packet_stream = b""
+                    self.msg.total_size += len(data)
+                    return
+                if new_frame is None:
+                    break
+                self.msg.ws_stream.append(new_frame)
+                self.msg.total_size += len(new_frame.data)
+        if self.msg.upgrading_to_h2:
+            self.msg.total_size += len(data)
+            self.msg.stream += data
+    
+    def _parse_websocket_ext(self):
+        ext_ws = []
+        req_ext = []
+        for ele in self.msg.lheaders.get("sec-websocket-extensions", "").split(","):
+            for xt in ele.split(";"):
+                req_ext.append(xt.strip().lower())
+
+        for ele in req_ext:
+            if ele == "permessage-deflate":
+                ext_ws.append(PerMessageDeflate(False, False, 15, 15))
+        return ext_ws
+    
+    def _parse_websocket_frame(self, data: bytes) -> tuple[Frame|None, bytes]:
+        # mask = is_input
+        if self._ws_extentions is None:
+            self._ws_extentions = self._parse_websocket_ext()
+        read_buffering = bytearray()
+        def read_exact(n: int):
+            nonlocal read_buffering
+            buffer = bytearray(read_buffering)
+            while len(buffer) < n:
+                data = yield
+                if data is None:
+                    raise RuntimeError("Should not send None to this generator")
+                buffer.extend(data)
+            new_data = bytes(buffer[:n])
+            read_buffering = buffer[n:]
+            return new_data
+
+        parsing = Frame.parse(read_exact, extensions=self._ws_extentions, mask=self._is_input())
+        parsing.send(None)
+        try:
+            parsing.send(bytearray(data))
+        except StopIteration as e:
+            return e.value, read_buffering
+        
+        return None, read_buffering
+    
+    def parse_data(self, data: bytes):
+        if self._packet_to_stream(): # This is a websocket upgrade!
+            self._stream_parser(data)
+        else:
+            try:
+                reason, consumed = self.execute(data)
+                if reason == PAUSED_UPGRADE:
+                    self.msg.upgrading_to_ws = True
+                    self.msg.message_complete = True
+                    self._stream_parser(data[consumed:])
+                elif reason == PAUSED_H2_UPGRADE:
+                    self.msg.upgrading_to_h2 = True
+                    self.msg.message_complete = True
+                    self._stream_parser(data[consumed:])
+            except Exception as e:
+                self.raised_error = True
+                raise e
+    
+    def pop_message(self):
+        return self.messages.popleft()
+    
+    def __repr__(self):
+        return f"<InternalCallbackHandler msg={self.msg} buffers={self.buffers} save_body={self.save_body} raised_error={self.raised_error} has_begun={self.has_begun} messages={self.messages}>"
+    
+
+class InternalHttpRequest(InternalCallbackHandler, pyllhttp.Request):
+    def __init__(self):
+        super(InternalCallbackHandler, self).__init__()
+        super(pyllhttp.Request, self).__init__()
+    
+    def _is_input(self):
+        return True
+        
+class InternalHttpResponse(InternalCallbackHandler, pyllhttp.Response):
+    def __init__(self):
+        super(InternalCallbackHandler, self).__init__()
+        super(pyllhttp.Response, self).__init__()
+    
+    def _is_input(self):
+        return False
+        
+class InternalBasicHttpMetaClass:
+    """Internal class to handle HTTP requests and responses"""
+    
+    def __init__(self, parser: InternalHttpRequest|InternalHttpResponse, msg: InternalHTTPMessage):
+        self._parser = parser
+        self.raised_error = False
+        self._message: InternalHTTPMessage|None = msg
+        self._contructor_hook()
+    
+    def _contructor_hook(self):
+        pass
+    
+    @property
+    def total_size(self) -> int:
+        """Total size of the stream"""
+        return self._parser.total_size
+    
+    @property
+    def url(self) -> str|None:
+        """URL of the message"""
+        return self._message.url
+    
+    @property
+    def headers(self) -> dict[str, str]:
+        """Headers of the message"""
+        return self._message.headers
+    
+    @property
+    def user_agent(self) -> str:
+        """User agent of the message"""
+        return self._message.user_agent
+    
+    @property
+    def content_encoding(self) -> str:
+        """Content encoding of the message"""
+        return self._message.content_encoding
+    
+    @property
+    def body(self) -> bytes:
+        """Body of the message"""
+        return self._message.body
+    
+    @property
+    def headers_complete(self) -> bool:
+        """If the headers are complete"""
+        return self._message.headers_complete
+    
+    @property
+    def message_complete(self) -> bool:
+        """If the message is complete"""
+        return self._message.message_complete
+    
+    @property
+    def http_version(self) -> str:
+        """HTTP version of the message"""
+        return self._message.http_version
+
+    @property
+    def keep_alive(self) -> bool:
+        """If the message should keep alive"""
+        return self._message.keep_alive
+
+    @property
+    def should_upgrade(self) -> bool:
+        """If the message should upgrade"""
+        return self._parser.should_upgrade
+
+    @property
+    def content_length(self) -> int|None:
+        """Content length of the message"""
+        return self._message.content_length
+
+    @property
+    def upgrading_to_h2(self) -> bool:
+        """If the message is upgrading to HTTP/2"""
+        return self._message.upgrading_to_h2
+    
+    @property
+    def upgrading_to_ws(self) -> bool:
+        """If the message is upgrading to Websocket"""
+        return self._message.upgrading_to_ws
+
+    @property
+    def ws_stream(self) -> list[Frame]:
+        """Websocket stream"""
+        return self._message.ws_stream
+    
+    @property
+    def stream(self) -> bytes:
+        """Stream of the message"""
+        return self._message.stream
+    
+    def get_header(self, header: str, default=None) -> str:
+        """Get a header from the message without caring about the case"""
+        return self._message.lheaders.get(header.lower(), default)
+    
+    @staticmethod
+    def _associated_parser_class() -> Type[InternalHttpRequest]|Type[InternalHttpResponse]:
+        raise NotImplementedError()
+    
+    @staticmethod
+    def _before_fetch_callable_checks(internal_data: DataStreamCtx):
+        return True
+    
+    @classmethod
+    def _fetch_packet(cls, internal_data: DataStreamCtx):
+        if internal_data.current_pkt is None or internal_data.current_pkt.is_tcp is False:
+            raise NotReadyToRun()
+        
+        ParserType = cls._associated_parser_class()
+        
+        parser = internal_data.data_handler_context.get(cls, None)
+        if parser is None or parser.raised_error:
+            parser: InternalHttpRequest|InternalHttpResponse = ParserType()
+            internal_data.data_handler_context[cls] = parser
+        
+        if not cls._before_fetch_callable_checks(internal_data):
+            raise NotReadyToRun()
+
+        # Memory size managment
+        if parser.total_size+len(internal_data.current_pkt.data) > internal_data.stream_max_size:
+            match internal_data.full_stream_action:
+                case FullStreamAction.FLUSH:
+                    # Deleting parser and re-creating it
+                    parser.messages.clear()
+                    parser.msg.total_size -= len(parser.msg.stream)
+                    parser.msg.stream = b""
+                    parser.msg.total_size -= len(parser.msg.body)
+                    parser.msg.body = b""
+                    print("[WARNING] Flushing stream", flush=True)
+                    if parser.total_size+len(internal_data.current_pkt.data) > internal_data.stream_max_size:
+                        parser.reset_data()
+                case FullStreamAction.REJECT:
+                    raise StreamFullReject()
+                case FullStreamAction.DROP:
+                    raise StreamFullDrop()
+                case FullStreamAction.ACCEPT:
+                    raise NotReadyToRun()
+        
+        headers_were_set = parser.msg.headers_complete
+        try:
+            parser.parse_data(internal_data.current_pkt.data)
+        except Exception as e:
+            match internal_data.invalid_encoding_action:
+                case ExceptionAction.REJECT:
+                    raise RejectConnection()
+                case ExceptionAction.DROP:
+                    raise DropPacket()
+                case ExceptionAction.NOACTION:
+                    raise e
+                case ExceptionAction.ACCEPT:
+                    raise NotReadyToRun()
+
+        messages_tosend:list[InternalHTTPMessage] = []
+        for i in range(len(parser.messages)):
+            messages_tosend.append(parser.pop_message())
+        
+        if len(messages_tosend) > 0:
+            headers_were_set = False # New messages completed so the current message headers were not set in this case
+        
+        if not headers_were_set and parser.msg.headers_complete:
+            messages_tosend.append(parser.msg) # Also the current message needs to be sent due to complete headers
+        
+        if parser._packet_to_stream():
+            messages_tosend.append(parser.msg) # Also the current message needs to beacase a stream is going on
+        
+        messages_to_call = len(messages_tosend)
+        
+        if messages_to_call == 0:
+            raise NotReadyToRun()
+        elif messages_to_call == 1:
+            return cls(parser, messages_tosend[0])
+        
+        return [cls(parser, ele) for ele in messages_tosend]
+
+class HttpRequest(InternalBasicHttpMetaClass):
+    """
+    HTTP Request handler
+    This data handler will be called twice, first with the headers complete, and second with the body complete
+    """
+    
+    @staticmethod
+    def _associated_parser_class() -> Type[InternalHttpRequest]:
+        return InternalHttpRequest
+
+    @staticmethod
+    def _before_fetch_callable_checks(internal_data: DataStreamCtx):
+        return internal_data.current_pkt.is_input
+
+    @property
+    def method(self) -> bytes:
+        """Method of the request"""
+        return self._parser.msg.method
+    
+    def __repr__(self):
+        return f"<HttpRequest method={self.method} url={self.url} headers={self.headers} body=[{0 if not self.body else len(self.body)} bytes] http_version={self.http_version} keep_alive={self.keep_alive} should_upgrade={self.should_upgrade} headers_complete={self.headers_complete} message_complete={self.message_complete} content_length={self.content_length} stream={self.stream} ws_stream={self.ws_stream}>"
+
+class HttpResponse(InternalBasicHttpMetaClass):
+    """
+    HTTP Response handler
+    This data handler will be called twice, first with the headers complete, and second with the body complete
+    """
+
+    @staticmethod
+    def _associated_parser_class() -> Type[InternalHttpResponse]:
+        return InternalHttpResponse
+
+    @staticmethod
+    def _before_fetch_callable_checks(internal_data: DataStreamCtx):
+        return not internal_data.current_pkt.is_input
+
+    @property
+    def status_code(self) -> int:
+        """Status code of the response"""
+        return self._parser.msg.status
+    
+    def __repr__(self):
+        return f"<HttpResponse status_code={self.status_code} url={self.url} headers={self.headers} body=[{0 if not self.body else len(self.body)} bytes] http_version={self.http_version} keep_alive={self.keep_alive} should_upgrade={self.should_upgrade} headers_complete={self.headers_complete} message_complete={self.message_complete} content_length={self.content_length} stream={self.stream} ws_stream={self.ws_stream}>"
+
+class HttpRequestHeader(HttpRequest):
+    """
+    HTTP Request Header handler
+    This data handler will be called only once, the headers are complete, the body will be empty and not buffered
+    """
+        
+    def _contructor_hook(self):
+        self._parser.save_body = False
+
+class HttpResponseHeader(HttpResponse):
+    """
+    HTTP Response Header handler
+    This data handler will be called only once, the headers are complete, the body will be empty and not buffered
+    """
+    
+    def _contructor_hook(self):
+        self._parser.save_body = False

{firegex-3.0.0 → firegex-3.2.0}/firegex/nfproxy/models/tcp.py

@@ -71,7 +71,7 @@ class TCPInputStream(InternalTCPStream):
 
 TCPClientStream = TCPInputStream
 
-class TCPOutputStream:
+class TCPOutputStream(InternalTCPStream):
     """
     This datamodel will assemble the TCP output stream from the server sent data.
     The function that use this data model will be handled when:

{firegex-3.0.0 → firegex-3.2.0/firegex.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: firegex
-Version: 3.0.0
+Version: 3.2.0
 Summary: Firegex client
 Home-page: https://github.com/pwnzer0tt1/firegex
 Author: Pwnzer0tt1
@@ -13,8 +13,11 @@ Description-Content-Type: text/markdown
 Requires-Dist: typer==0.15.2
 Requires-Dist: pydantic>=2
 Requires-Dist: typing-extensions>=4.7.1
+Requires-Dist: zstd
+Requires-Dist: brotli
 Requires-Dist: watchfiles
 Requires-Dist: fgex
+Requires-Dist: websockets
 Requires-Dist: pyllhttp
 Dynamic: author
 Dynamic: author-email
@@ -122,7 +125,6 @@ This handler will be called twice: one for the request headers and one for the r
 - headers: dict - The headers of the request
 - user_agent: str - The user agent of the request
 - content_encoding: str - The content encoding of the request
-- has_begun: bool - It's true if the request has begun
 - body: bytes - The body of the request
 - headers_complete: bool - It's true if the headers are complete
 - message_complete: bool - It's true if the message is complete
@@ -150,7 +152,6 @@ This handler will be called twice: one for the response headers and one for the
 - headers: dict - The headers of the response
 - user_agent: str - The user agent of the response
 - content_encoding: str - The content encoding of the response
-- has_begun: bool - It's true if the response has begun
 - body: bytes - The body of the response
 - headers_complete: bool - It's true if the headers are complete
 - message_complete: bool - It's true if the message is complete

{firegex-3.0.0 → firegex-3.2.0}/firegex.egg-info/requires.txt

@@ -1,6 +1,9 @@
 typer==0.15.2
 pydantic>=2
 typing-extensions>=4.7.1
+zstd
+brotli
 watchfiles
 fgex
+websockets
 pyllhttp

firegex-3.2.0/requirements.txt

@@ -0,0 +1,9 @@
+typer==0.15.2
+pydantic>=2
+typing-extensions>=4.7.1
+zstd    # waiting for pull request to be merged
+brotli  # waiting for pull request to be merged
+watchfiles
+fgex
+websockets
+pyllhttp

{firegex-3.0.0 → firegex-3.2.0}/setup.py

@@ -6,7 +6,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 with open('requirements.txt', 'r', encoding='utf-8') as f:
     required = [ele.strip() for ele in f.read().splitlines() if not ele.strip().startswith("#") and ele.strip() != ""]
 
-VERSION = "3.0.0"
+VERSION = "3.2.0"
 
 setuptools.setup(
     name="firegex",

firegex-3.0.0/firegex/__init__.py

@@ -1,5 +0,0 @@
-
-__version__ = "3.0.0" if "{" not in "3.0.0" else "0.0.0"
-
-#Exported functions
-__all__ = []

firegex-3.0.0/firegex/nfproxy/models/http.py

@@ -1,357 +0,0 @@
-import pyllhttp         
-from firegex.nfproxy.internals.exceptions import NotReadyToRun
-from firegex.nfproxy.internals.data import DataStreamCtx
-from firegex.nfproxy.internals.exceptions import StreamFullDrop, StreamFullReject
-from firegex.nfproxy.internals.models import FullStreamAction
-
-class InternalCallbackHandler():
-    
-    url: str|None = None
-    _url_buffer: bytes = b""
-    headers: dict[str, str] = {}
-    lheaders: dict[str, str] = {} # Lowercase headers
-    _header_fields: dict[bytes, bytes] = {}
-    has_begun: bool = False
-    body: bytes = None
-    _body_buffer: bytes = b""
-    headers_complete: bool = False
-    message_complete: bool = False
-    status: str|None = None
-    _status_buffer: bytes = b""
-    _current_header_field = b""
-    _current_header_value = b""
-    _save_body = True
-    total_size = 0
-
-    def on_message_begin(self):
-        self.has_begun = True
-    
-    def on_url(self, url):
-        self.total_size += len(url)
-        self._url_buffer += url
-    
-    def on_url_complete(self):
-        self.url = self._url_buffer.decode(errors="ignore")
-        self._url_buffer = None
-    
-    def on_header_field(self, field):
-        self.total_size += len(field)
-        self._current_header_field += field
-
-    def on_header_field_complete(self):
-        self._current_header_field = self._current_header_field
-        
-    def on_header_value(self, value):
-        self.total_size += len(value)
-        self._current_header_value += value
-
-    def on_header_value_complete(self):
-        if self._current_header_value is not None and self._current_header_field is not None:
-            self._header_fields[self._current_header_field.decode(errors="ignore")] = self._current_header_value.decode(errors="ignore")
-        self._current_header_field = b""
-        self._current_header_value = b""
-    
-    def on_headers_complete(self):
-        self.headers_complete = True
-        self.headers = self._header_fields
-        self.lheaders = {k.lower(): v for k, v in self._header_fields.items()}
-        self._header_fields = {}
-        self._current_header_field = b""
-        self._current_header_value = b""
-
-    def on_body(self, body: bytes):
-        if self._save_body:
-            self.total_size += len(body)
-            self._body_buffer += body
-
-    def on_message_complete(self):
-        self.body = self._body_buffer
-        self._body_buffer = b""
-        try:
-            if "gzip" in self.content_encoding.lower():
-                import gzip
-                import io
-                with gzip.GzipFile(fileobj=io.BytesIO(self.body)) as f:
-                    self.body = f.read()
-        except Exception as e:
-            print(f"Error decompressing gzip: {e}: skipping", flush=True)
-        self.message_complete = True
-    
-    def on_status(self, status: bytes):
-        self.total_size += len(status)
-        self._status_buffer += status
-    
-    def on_status_complete(self):
-        self.status = self._status_buffer.decode(errors="ignore")
-        self._status_buffer = b""
-    
-    @property
-    def user_agent(self) -> str:
-        return self.lheaders.get("user-agent", "")
-    
-    @property
-    def content_encoding(self) -> str:
-        return self.lheaders.get("content-encoding", "")
-    
-    @property
-    def content_type(self) -> str:
-        return self.lheaders.get("content-type", "")
-
-    @property
-    def keep_alive(self) -> bool:
-        return self.should_keep_alive
-
-    @property
-    def should_upgrade(self) -> bool:
-        return self.is_upgrading
-    
-    @property
-    def http_version(self) -> str:
-        return f"{self.major}.{self.minor}"
-    
-    @property
-    def method_parsed(self) -> str:
-        return self.method.decode(errors="ignore")
-    
-    @property
-    def content_length_parsed(self) -> int:
-        return self.content_length
-    
-
-class InternalHttpRequest(InternalCallbackHandler, pyllhttp.Request):
-    def __init__(self):
-        super(InternalCallbackHandler, self).__init__()
-        super(pyllhttp.Request, self).__init__()
-        
-class InternalHttpResponse(InternalCallbackHandler, pyllhttp.Response):
-    def __init__(self):
-        super(InternalCallbackHandler, self).__init__()
-        super(pyllhttp.Response, self).__init__()
-        
-class InternalBasicHttpMetaClass:
-    """Internal class to handle HTTP requests and responses"""
-    
-    def __init__(self):
-        self._parser: InternalHttpRequest|InternalHttpResponse
-        self._headers_were_set = False
-        self.stream = b""
-        self.raised_error = False
-    
-    @property
-    def total_size(self) -> int:
-        """Total size of the stream"""
-        return self._parser.total_size
-    
-    @property
-    def url(self) -> str|None:
-        """URL of the message"""
-        return self._parser.url
-    
-    @property
-    def headers(self) -> dict[str, str]:
-        """Headers of the message"""
-        return self._parser.headers
-    
-    @property
-    def user_agent(self) -> str:
-        """User agent of the message"""
-        return self._parser.user_agent
-    
-    @property
-    def content_encoding(self) -> str:
-        """Content encoding of the message"""
-        return self._parser.content_encoding
-    
-    @property
-    def has_begun(self) -> bool:
-        """If the message has begun"""
-        return self._parser.has_begun
-    
-    @property
-    def body(self) -> bytes:
-        """Body of the message"""
-        return self._parser.body
-    
-    @property
-    def headers_complete(self) -> bool:
-        """If the headers are complete"""
-        return self._parser.headers_complete
-    
-    @property
-    def message_complete(self) -> bool:
-        """If the message is complete"""
-        return self._parser.message_complete
-    
-    @property
-    def http_version(self) -> str:
-        """HTTP version of the message"""
-        return self._parser.http_version
-
-    @property
-    def keep_alive(self) -> bool:
-        """If the message should keep alive"""
-        return self._parser.keep_alive
-
-    @property
-    def should_upgrade(self) -> bool:
-        """If the message should upgrade"""
-        return self._parser.should_upgrade
-
-    @property
-    def content_length(self) -> int|None:
-        """Content length of the message"""
-        return self._parser.content_length_parsed
-    
-    def get_header(self, header: str, default=None) -> str:
-        """Get a header from the message without caring about the case"""
-        return self._parser.lheaders.get(header.lower(), default)
-    
-    def _packet_to_stream(self, internal_data: DataStreamCtx):
-        return self.should_upgrade and self._parser._save_body
-    
-    def _fetch_current_packet(self, internal_data: DataStreamCtx):
-        if self._packet_to_stream(internal_data): # This is a websocket upgrade!
-            self._parser.total_size += len(internal_data.current_pkt.data)
-            self.stream += internal_data.current_pkt.data
-        else:
-            try:
-                self._parser.execute(internal_data.current_pkt.data)
-                if not self._parser.message_complete and self._parser.headers_complete and len(self._parser._body_buffer) == self._parser.content_length_parsed:
-                    self._parser.on_message_complete()
-            except Exception as e:
-                self.raised_error = True
-                print(f"Error parsing HTTP packet: {e} {internal_data.current_pkt}", self, flush=True)
-                raise e
-
-    #It's called the first time if the headers are complete, and second time with body complete
-    def _after_fetch_callable_checks(self, internal_data: DataStreamCtx):
-        if self._parser.headers_complete and not self._headers_were_set:
-            self._headers_were_set = True
-            return True
-        return self._parser.message_complete or self.should_upgrade
-    
-    def _before_fetch_callable_checks(self, internal_data: DataStreamCtx):
-        return True
-
-    def _trigger_remove_data(self, internal_data: DataStreamCtx):
-        return self.message_complete and not self.should_upgrade
-    
-    @classmethod
-    def _fetch_packet(cls, internal_data: DataStreamCtx):
-        if internal_data.current_pkt is None or internal_data.current_pkt.is_tcp is False:
-            raise NotReadyToRun()
-        
-        datahandler:InternalBasicHttpMetaClass = internal_data.data_handler_context.get(cls, None)
-        if datahandler is None or datahandler.raised_error:
-            datahandler = cls()
-            internal_data.data_handler_context[cls] = datahandler
-        
-        if not datahandler._before_fetch_callable_checks(internal_data):
-            raise NotReadyToRun()
-
-        # Memory size managment
-        if datahandler.total_size+len(internal_data.current_pkt.data) > internal_data.stream_max_size:
-            match internal_data.full_stream_action:
-                case FullStreamAction.FLUSH:
-                    datahandler = cls()
-                    internal_data.data_handler_context[cls] = datahandler
-                case FullStreamAction.REJECT:
-                    raise StreamFullReject()
-                case FullStreamAction.DROP:
-                    raise StreamFullDrop()
-                case FullStreamAction.ACCEPT:
-                    raise NotReadyToRun()
-        
-        datahandler._fetch_current_packet(internal_data)
-
-        if not datahandler._after_fetch_callable_checks(internal_data):
-            raise NotReadyToRun()
-        
-        if datahandler._trigger_remove_data(internal_data):
-            if internal_data.data_handler_context.get(cls):
-                del internal_data.data_handler_context[cls]
-        
-        return datahandler
-
-class HttpRequest(InternalBasicHttpMetaClass):
-    """
-    HTTP Request handler
-    This data handler will be called twice, first with the headers complete, and second with the body complete
-    """
-    
-    def __init__(self):
-        super().__init__()
-        # These will be used in the metaclass
-        self._parser: InternalHttpRequest = InternalHttpRequest()
-        self._headers_were_set = False
-
-    @property
-    def method(self) -> bytes:
-        """Method of the request"""
-        return self._parser.method_parsed
-    
-    def _before_fetch_callable_checks(self, internal_data: DataStreamCtx):
-        return internal_data.current_pkt.is_input
-    
-    def __repr__(self):
-        return f"<HttpRequest method={self.method} url={self.url} headers={self.headers} body={self.body} http_version={self.http_version} keep_alive={self.keep_alive} should_upgrade={self.should_upgrade} headers_complete={self.headers_complete} message_complete={self.message_complete} has_begun={self.has_begun} content_length={self.content_length} stream={self.stream}>"
-
-class HttpResponse(InternalBasicHttpMetaClass):
-    """
-    HTTP Response handler
-    This data handler will be called twice, first with the headers complete, and second with the body complete
-    """
-
-    def __init__(self):
-        super().__init__()
-        self._parser: InternalHttpResponse = InternalHttpResponse()
-        self._headers_were_set = False
-
-    @property
-    def status_code(self) -> int:
-        """Status code of the response"""
-        return self._parser.status
-    
-    def _before_fetch_callable_checks(self, internal_data: DataStreamCtx):
-        return not internal_data.current_pkt.is_input
-    
-    def __repr__(self):
-        return f"<HttpResponse status_code={self.status_code} url={self.url} headers={self.headers} body={self.body} http_version={self.http_version} keep_alive={self.keep_alive} should_upgrade={self.should_upgrade} headers_complete={self.headers_complete} message_complete={self.message_complete} has_begun={self.has_begun} content_length={self.content_length} stream={self.stream}>"
-
-class HttpRequestHeader(HttpRequest):
-    """
-    HTTP Request Header handler
-    This data handler will be called only once, the headers are complete, the body will be empty and not buffered
-    """
-    
-    def __init__(self):
-        super().__init__()
-        self._parser._save_body = False
-    
-    def _before_fetch_callable_checks(self, internal_data: DataStreamCtx):
-        return internal_data.current_pkt.is_input and not self._headers_were_set
-    
-    def _after_fetch_callable_checks(self, internal_data: DataStreamCtx):
-        if self._parser.headers_complete and not self._headers_were_set:
-            self._headers_were_set = True
-            return True
-        return False
-
-class HttpResponseHeader(HttpResponse):
-    """
-    HTTP Response Header handler
-    This data handler will be called only once, the headers are complete, the body will be empty and not buffered
-    """
-    
-    def __init__(self):
-        super().__init__()
-        self._parser._save_body = False
-    
-    def _before_fetch_callable_checks(self, internal_data: DataStreamCtx):
-        return not internal_data.current_pkt.is_input and not self._headers_were_set
-    
-    def _after_fetch_callable_checks(self, internal_data: DataStreamCtx):
-        if self._parser.headers_complete and not self._headers_were_set:
-            self._headers_were_set = True
-            return True
-        return False

firegex-3.0.0/requirements.txt

@@ -1,6 +0,0 @@
-typer==0.15.2
-pydantic>=2
-typing-extensions>=4.7.1
-watchfiles
-fgex
-pyllhttp