polyfile-weave 0.5.5__py3-none-any.whl

polyfile/html.py

@@ -0,0 +1,135 @@
+import base64
+import math
+import mimetypes
+import os
+import unicodedata
+
+jinja2 = None
+
+
+TEMPLATE_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'templates')
+TEMPLATE = None
+
+
+def assign_ids(sbud):
+    i = 0
+    matches = list(sbud['struc'])
+    stack = list(matches)
+    while stack:
+        m = stack.pop()
+        assert 'uid' not in m
+        m['uid'] = i
+        i += 1
+        stack += list(m['subEls'])
+    return matches
+
+
+def generate(file_path, sbud):
+    global TEMPLATE, jinja2
+    if jinja2 is None:
+        # Dynamically load jinja2 at runtime so it is not an installation dependency for setup.py
+        import jinja2 as j2
+        jinja2 = j2
+
+    if TEMPLATE is None:
+        TEMPLATE = jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATE_DIR)).get_template('template.html')
+
+    matches = assign_ids(sbud)
+
+    input_bytes = sbud['length']
+    with open(file_path, 'rb') as input_file:
+        class ReadUnicode():
+            def __init__(self):
+                self.reset = False
+
+            def tell(self):
+                if not self.reset:
+                    return 0
+                else:
+                    return input_file.tell()
+
+            def translate(self, b, monospace=True):
+                if b is None or len(b) == 0 or b == b' ':
+                    return ' '
+                elif b == b'\n':
+                    if monospace:
+                        return '\u2424'
+                    else:
+                        return '\u2424</span><br /><span>'
+                elif b == b'\t':
+                    if monospace:
+                        return b'\xE2\xAD\xBE'
+                    else:
+                        return '\t'
+                elif b == b'\r':
+                    return '\u240d'
+                try:
+                    u = b.decode('utf-8')
+                    if unicodedata.category(u) == 'Cc':
+                        # This is a control character
+                        return '\ufffd'
+                    else:
+                        return u
+                except UnicodeDecodeError:
+                    return '\ufffd'
+
+            def __iter__(self):
+                input_file.seek(0)
+                i = 0
+                while True:
+                    b = input_file.read(1)
+                    if b is None or len(b) < 1:
+                        break
+                    yield i, self.translate(b, monospace=False)
+                    i += 1
+
+            def __call__(self):
+                if not self.reset:
+                    input_file.seek(0)
+                    self.reset = True
+                b = input_file.read(1)
+                return self.translate(b)
+
+        mime_type = mimetypes.guess_type(file_path)[0]
+        if mime_type is None:
+            mime_type = 'application/octet-stream'
+
+        def _decoded_matches(m=None):
+            if m is None:
+                return
+            if 'decoded' in m:
+                decoded = ''
+                for b in base64.b64decode(m['decoded']).replace(b'\r\n', b'<br />') \
+                        .replace(b'\n', b'<br />').replace(b'\r', b'\xE2\x8F\x8E'):
+                    try:
+                        decoded += bytes([b]).decode('utf-8')
+                    except UnicodeDecodeError:
+                        decoded += f"\\x{int(b)}"
+                yield m, decoded
+            if 'subEls' in m:
+                for a in m['subEls']:
+                    yield from _decoded_matches(a)
+
+        def decoded_matches():
+            for m in matches:
+                yield from _decoded_matches(m)
+
+        return TEMPLATE.render(
+            filename=os.path.split(file_path)[-1],
+            encoded=sbud['b64contents'],
+            matches=matches,
+            input_file=input_file,
+            input_bytes=input_bytes,
+            math=math,
+            read_unicode=ReadUnicode(),
+            mime_type=mime_type,
+            decoded_matches=decoded_matches
+        )
+
+
+if __name__ == '__main__':
+    import json
+    import sys
+
+    with open(sys.argv[2], 'r') as f:
+        print(generate(sys.argv[1], json.load(f)))

polyfile/http/__init__.py

@@ -0,0 +1 @@
+from .matcher import parse_http_11

polyfile/http/defacto.py

@@ -0,0 +1,37 @@
+from abnf.grammars.misc import load_grammar_rules
+from abnf.grammars import rfc9110, rfc3986
+from abnf import Rule as _Rule
+from typing import List, Tuple
+
+inherited_rulelist: List[Tuple[str, _Rule]] = [
+    ("ipv4", rfc3986.Rule("IPv4address")),
+    ("ipv6", rfc3986.Rule("IPv6address")),
+    ("uri-host", rfc9110.Rule("uri-host")),
+]
+
+
+@load_grammar_rules(inherited_rulelist)
+class Rule(_Rule):
+    """A place to define *hop-by-hop* headers which do not have an RFC or other standard, but are in common use. As linked, these definitions are primarily based on Mozilla documentation for now.
+
+    These are highly worth parsing and examining since they can be spoofed and are untrustworthy if not added by a reverse proxy on a hop-by-hop request path.
+
+    There are several variants on X-Forwarded-Proto header included.
+    """
+
+    grammar: List[str] = [
+        'defacto-header = "X-Forwarded-For:" OWS X-Forwarded-For OWS / "X-Forwarded-Host:" OWS X-Forwarded-Host OWS / "X-Forwarded-Proto:" OWS X-Forwarded-Proto OWS / "Front-End-Https:" OWS Front-End-Https OWS / "X-Forwarded-Protocol:" OWS X-Forwarded-Protocol OWS / "X-Forwarded-Ssl:" OWS X-Forwarded-Ssl OWS / "X-Url-Scheme:" OWS X-Url-Scheme OWS',
+        # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For
+        'X-Forwarded-For = xff-client *( 0*1SP "," 0*1SP xff-proxy )',
+        "xff-client = ipv4 / ipv6",
+        "xff-proxy = ipv4 / ipv6",
+        # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-Host - value should be the domain name of the forwarded server
+        "X-Forwarded-Host = uri-host",
+        # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-Proto
+        'proto = "http" / "https"',
+        "X-Forwarded-Proto = proto",
+        'Front-End-Https = "on"',
+        "X-Forwarded-Protocol = proto",
+        'X-Forwarded-Ssl = "on"',
+        "X-Url-Scheme = proto",
+    ]

polyfile/http/deprecated.py

@@ -0,0 +1,51 @@
+from abnf.grammars.misc import load_grammar_rules
+from abnf.grammars import rfc9110, rfc3986, rfc9111
+from abnf import Rule as _Rule
+from typing import List, Tuple
+
+inherited_rulelist: List[Tuple[str, _Rule]] = [
+    ("Accept-Charset", rfc9110.Rule("Accept-Charset")),
+    ("Authentication-Info", rfc9110.Rule("Authentication-Info")),
+    ("token", rfc9110.Rule("token")),
+    ("quoted-string", rfc9110.Rule("quoted-string")),
+    ("HTTP-date", rfc9110.Rule("HTTP-date")),
+    ("Host", rfc9110.Rule("Host")),
+]
+
+
+@load_grammar_rules(inherited_rulelist)
+class Rule(_Rule):
+    """
+    Request headers which are in general deprecated by modern browsers, but may still be included from spoofed user agents or unusual user agents.
+    """
+
+    grammar: List[str] = [
+        'deprecated-header = "Accept-Charset:" OWS Accept-Charset OWS / "Authentication-Info:" OWS Authentication-Info OWS / "DNT:" OWS DNT OWS / "DPR:" OWS DPR OWS / "Expect-CT:" OWS Expect-CT OWS / "Pragma:" OWS Pragma OWS / "Viewport-Width:" OWS Viewport-Width OWS / "Warning:" OWS Warning OWS / "Width:" OWS Width OWS',
+        # https://www.w3.org/TR/tracking-dnt/#dnt-header-field
+        'DNT = ( "0" / "1" ) *DNT-extension',
+        # DNT-extension excludes CTL, SP, DQUOTE, comma, backslash
+        "DNT-extension = %x21 / %x23-2B / %x2D-5B / %x5D-7E",
+        # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/DPR
+        'DPR = 1*DIGIT "." 1*DIGIT',
+        # https://www.rfc-editor.org/rfc/rfc9163#section-2.1
+        'Expect-CT           = expect-ct-directive *( OWS "," OWS expect-ct-directive )',
+        'expect-ct-directive = directive-name [ "=" directive-value ]',
+        "directive-name      = token",
+        "directive-value     = token / quoted-string",
+        # https://httpwg.org/specs/rfc9111.html#field.pragma
+        'Pragma = "no-cache"',
+        # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Viewport-Width
+        # The width of the user's viewport in CSS pixels, rounded up to the nearest integer.
+        "Viewport-Width = 1*DIGIT",
+        # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Warning
+        "Warning = warn-code warn-agent warn-text *warn-date",
+        # https://www.iana.org/assignments/http-warn-codes/http-warn-codes.xhtml
+        'warn-code = "110" / "111" / "112" / "113" / "199" / "214" / "299"',
+        "warn-agent = Host / pseudonym",
+        "warn-text = quoted-string",
+        "warn-date = HTTP-date",
+        "pseudonym = token",
+        # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Width
+        # The width of the resource in physical pixels, rounded up to the nearest integer.
+        "Width = 1*DIGIT",
+    ]

polyfile/http/experimental.py

@@ -0,0 +1,67 @@
+from abnf.grammars.misc import load_grammar_rules
+from abnf.grammars import rfc9110, rfc3986
+from . import structured_headers
+from abnf import Rule as _Rule
+from typing import List, Tuple
+
+rulelist: List[Tuple[str, _Rule]] = [
+    ("OWS", rfc9110.Rule("OWS")),
+    ("sh-list", structured_headers.Rule("sh-list")),
+    ("sh-string", structured_headers.Rule("sh-string")),
+    ("sh-decimal", structured_headers.Rule("sh-decimal")),
+    ("sh-boolean", structured_headers.Rule("sh-boolean")),
+]
+
+
+@load_grammar_rules(rulelist)
+class Rule(_Rule):
+    """Request headers defined as 'experimental' by the Mozilla developer documentation, which have partial cross browser support.
+
+    Many of these headers convey client hints (indicated by the CH segment in some, but not all, related header names).
+
+    Most of these also require only simple ABNF to string together the values, so they are collected here for brevity.
+    """
+
+    grammar: List[str] = [
+        'experimental-header = "Device-Memory:" OWS Device-Memory OWS / "Downlink:" OWS Downlink OWS / "Early-Data:" OWS Early-Data OWS / "ECT:" OWS ECT OWS / "RTT:" OWS RTT OWS / "Save-Data:" OWS Save-Data OWS / "Sec-CH-UA-Arch:" OWS Sec-CH-UA-Arch OWS / "Sec-CH-UA-Bitness:" OWS Sec-CH-UA-Bitness OWS / "Sec-CH-UA-Form-Factor:" OWS Sec-CH-UA-Form-Factor / "Sec-CH-UA-Full-Version:" OWS Sec-CH-UA-Full-Version OWS / "Sec-CH-UA-Full-Version-List:" OWS Sec-CH-UA-Full-Version-List OWS / "Sec-CH-UA-Mobile:" OWS Sec-CH-UA-Mobile OWS / "Sec-CH-UA-Model:" OWS Sec-CH-UA-Model OWS / "Sec-CH-UA-Platform:" OWS Sec-CH-UA-Platform OWS / "Sec-CH-UA-Platform-Version:" OWS Sec-CH-UA-Platform-Version OWS / "Sec-GPC:" OWS Sec-GPC OWS / "Sec-CH-Prefers-Reduced-Motion:" OWS Sec-CH-Prefers-Reduced-Motion OWS',
+        # https://www.w3.org/TR/device-memory/#iana-device-memory
+        'Device-Memory = "0.25" / "0.5" / "1" / "2" / "4" / "8"',
+        # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Downlink
+        # https://wicg.github.io/netinfo/#dom-networkinformation-downlink
+        "Downlink = sh-decimal",
+        # https://httpwg.org/specs/rfc8470.html#header
+        'Early-Data = "1"',
+        # https://wicg.github.io/netinfo/#ect-request-header-field
+        'ECT = "2g" / "3g" / "4g" / "slow-2g"',
+        # https://wicg.github.io/netinfo/#rtt-request-header-field
+        "RTT = 1*DIGIT",
+        # https://wicg.github.io/savedata/#save-data-request-header-field TODO Structured Headers RFC for sh-list
+        'Save-Data = "on" / sh-list',
+        # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Sec-CH-UA
+        # TODO sf-list vs sh-list
+        "Sec-CH-UA = sh-list",
+        # https://wicg.github.io/ua-client-hints/#sec-ch-ua-arch
+        "Sec-CH-UA-Arch = sh-string",
+        # https://wicg.github.io/ua-client-hints/#sec-ch-ua-bitness
+        "Sec-CH-UA-Bitness = sh-string",
+        # https://wicg.github.io/ua-client-hints/#sec-ch-ua-form-factor
+        "Sec-CH-UA-Form-Factor = sh-string",
+        # https://wicg.github.io/ua-client-hints/#sec-ch-ua-full-version
+        "Sec-CH-UA-Full-Version = sh-string",
+        # https://wicg.github.io/ua-client-hints/#sec-ch-ua-full-version-list
+        "Sec-CH-UA-Full-Version-List = sh-list",
+        # https://wicg.github.io/ua-client-hints/#sec-ch-ua-mobile
+        "Sec-CH-UA-Mobile = sh-boolean",
+        # https://wicg.github.io/ua-client-hints/#sec-ch-ua-model
+        "Sec-CH-UA-Model = sh-string",
+        # https://wicg.github.io/ua-client-hints/#sec-ch-ua-platform
+        "Sec-CH-UA-Platform = sh-string",
+        # https://wicg.github.io/ua-client-hints/#sec-ch-ua-platform-version
+        "Sec-CH-UA-Platform-Version = sh-string",
+        # https://wicg.github.io/ua-client-hints/#sec-ch-ua-wow64
+        "Sec-CH-UA-WoW64 = sh-boolean",
+        # https://privacycg.github.io/gpc-spec/#the-sec-gpc-header-field-for-http-requests
+        'Sec-GPC = "1"',
+        # https://wicg.github.io/user-preference-media-features-headers/#sec-ch-prefers-reduced-motion
+        'Sec-CH-Prefers-Reduced-Motion = "no-preference" / "reduce"',
+    ]