cryptoparser 0.11.0__tar.gz → 0.11.2__tar.gz

{CryptoParser-0.11.0 → CryptoParser-0.11.2}/CHANGELOG.rst

@@ -2,6 +2,47 @@
 Changelog
 =========
 
+-------------------
+0.11.2 - 2023-11-13
+-------------------
+
+Features
+========
+
+-  HTTP(S) (``http``)
+
+   -  Headers (``headers``)
+
+      -  add parsers for generic headers
+         (`NEL <https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/NEL>`__ (Network Error Logging),
+         `Set-Cookie <https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie>`__)
+      -  add parsers for security related headers
+         (`HTTP Public Key Pinning <https://en.wikipedia.org/wiki/HTTP_Public_Key_Pinning>`__ (HPKP),
+         `X-XSS-Protection <https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-XSS-Protection>`__)
+
+Improvements
+============
+
+-  HTTP(S) (``http``)
+
+   -  Headers (``headers``)
+
+      -  implement detailed parsing of
+         `Content-Type <https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type>`__ header
+
+-------------------
+0.11.1 - 2023-11-06
+-------------------
+
+Features
+========
+
+-  SSH (``ssh``)
+
+   -  Public Keys (``pubkeys``)
+
+      -  add X.509 certificate and certificate chain related classes (#63)
+
 -------------------
 0.11.0 - 2023-10-28
 -------------------

{CryptoParser-0.11.0 → CryptoParser-0.11.2/CryptoParser.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: CryptoParser
-Version: 0.11.0
+Version: 0.11.2
 Summary: Fast and flexible security protocol parser and generator
 Author: Szilárd Pfeiffer
 Author-email: coroner@pfeifferszilard.hu
@@ -48,7 +48,7 @@ Description-Content-Type: text/x-rst
 License-File: LICENSE.txt
 Requires-Dist: asn1crypto
 Requires-Dist: attrs<22.0.1,>=20.3.0
-Requires-Dist: cryptodatahub>=0.11.0
+Requires-Dist: cryptodatahub==0.11.2
 Requires-Dist: python-dateutil
 Requires-Dist: py2-ipaddress; python_version < "3.4"
 Requires-Dist: typing; python_version < "3.5"

{CryptoParser-0.11.0 → CryptoParser-0.11.2}/CryptoParser.egg-info/requires.txt

@@ -1,6 +1,6 @@
 asn1crypto
 attrs<22.0.1,>=20.3.0
-cryptodatahub>=0.11.0
+cryptodatahub==0.11.2
 python-dateutil
 python-dateutil
 urllib3

{CryptoParser-0.11.0/CryptoParser.egg-info → CryptoParser-0.11.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: CryptoParser
-Version: 0.11.0
+Version: 0.11.2
 Summary: Fast and flexible security protocol parser and generator
 Author: Szilárd Pfeiffer
 Author-email: coroner@pfeifferszilard.hu
@@ -48,7 +48,7 @@ Description-Content-Type: text/x-rst
 License-File: LICENSE.txt
 Requires-Dist: asn1crypto
 Requires-Dist: attrs<22.0.1,>=20.3.0
-Requires-Dist: cryptodatahub>=0.11.0
+Requires-Dist: cryptodatahub==0.11.2
 Requires-Dist: python-dateutil
 Requires-Dist: py2-ipaddress; python_version < "3.4"
 Requires-Dist: typing; python_version < "3.5"

{CryptoParser-0.11.0 → CryptoParser-0.11.2}/cryptoparser/__setup__.py

@@ -2,7 +2,7 @@
 
 __title__ = 'CryptoParser'
 __technical_name__ = __title__.lower()
-__version__ = '0.11.0'
+__version__ = '0.11.2'
 __description__ = 'Fast and flexible security protocol parser and generator'
 __author__ = 'Szilárd Pfeiffer'
 __author_email__ = 'coroner@pfeifferszilard.hu'

{CryptoParser-0.11.0 → CryptoParser-0.11.2}/cryptoparser/common/base.py

@@ -2,6 +2,7 @@
 # pylint: disable=too-many-lines
 
 import abc
+import datetime
 import enum
 import json
 import math
@@ -18,6 +19,7 @@ from collections import OrderedDict
 
 import attr
 import six
+import urllib3
 
 from cryptodatahub.common.exception import InvalidValue
 from cryptodatahub.common.grade import Gradeable
@@ -58,6 +60,12 @@ class SerializableTextEncoder(object):
 
 
 class Serializable(object):  # pylint: disable=too-few-public-methods
+    _MARKDOWN_RESULT_STRING_CLASSES = (
+        CryptoDataParamsBase,
+        ipaddress.IPv4Network,
+        ipaddress.IPv6Network,
+        urllib3.util.url.Url,
+    )
     post_text_encoder = SerializableTextEncoder()
 
     @staticmethod
@@ -172,6 +180,9 @@ class Serializable(object):  # pylint: disable=too-few-public-methods
 
         if hasattr(obj, '_asdict'):
             dict_value = obj._asdict()
+            if not isinstance(dict_value, dict):
+                return False, dict_value
+
             dict_value = Serializable._filter_out_non_human_friendly(obj, dict_value, human_friendly_only=True)
         else:
             dict_value = Serializable._get_ordered_dict(obj, human_friendly_only=True)
@@ -234,10 +245,12 @@ class Serializable(object):  # pylint: disable=too-few-public-methods
                 return cls.post_text_encoder(obj.value, level)
 
             return cls.post_text_encoder(obj.name, level)
-        elif isinstance(obj, (ipaddress.IPv4Network, ipaddress.IPv6Network)):
-            return False, str(obj)
         elif attr.has(type(obj)):
             result = cls._markdown_result_complex(obj, level)
+        elif isinstance(obj, cls._MARKDOWN_RESULT_STRING_CLASSES):
+            return False, str(obj)
+        elif isinstance(obj, datetime.timedelta):
+            return False, str(obj.seconds)
         elif hasattr(obj, '_asdict'):
             result = cls._markdown_result(obj._asdict(), level)
         elif hasattr(obj, '__dict__') or isinstance(obj, dict):

{CryptoParser-0.11.0 → CryptoParser-0.11.2}/cryptoparser/common/field.py

@@ -3,13 +3,14 @@
 import abc
 import collections
 import datetime
+import json
 
 import attr
 import six
 import urllib3
 
 from cryptodatahub.common.exception import InvalidValue
-from cryptodatahub.common.types import convert_url
+from cryptodatahub.common.types import Base64Data, convert_base64_data, convert_value_to_object, convert_url
 
 from cryptoparser.common.base import Serializable
 from cryptoparser.common.exception import InvalidType, NotEnoughData
@@ -201,9 +202,11 @@ class FieldValueComponentBase(ParsableBase, Serializable):
         raise NotImplementedError()
 
     @classmethod
+    @abc.abstractmethod
     def _parse(cls, parsable):
         raise NotImplementedError()
 
+    @abc.abstractmethod
     def compose(self):
         raise NotImplementedError()
 
@@ -278,12 +281,15 @@ class FieldValueComponentKeyValueBase(FieldValueComponentBase):
     def _parse_value(cls, parser):
         raise NotImplementedError()
 
-    def _get_value_as_str(self):
+    def _get_value_as_simple_type(self):
         # neccessary only because PY2 handles multiple inheritance differently than PY3
         if isinstance(self.value, ParsableBaseNoABC):
             return self.value.compose().decode('ascii')
 
-        return str(self.value)
+        return self.value
+
+    def _get_value_as_str(self):
+        return str(self._get_value_as_simple_type())
 
     @classmethod
     def _parse(cls, parsable):
@@ -297,15 +303,22 @@ class FieldValueComponentKeyValueBase(FieldValueComponentBase):
 
         cls._check_name(parser['name'])
 
-        parser.parse_separator('=')
+        if cls.get_canonical_name():
+            parser.parse_separator('=')
         cls._parse_value(parser)
+        parsed_value = parser['value']
+        if cls.get_canonical_name():
+            parsed_value = cls(parsed_value)
 
-        return cls(parser['value']), parser.parsed_length
+        return parsed_value, parser.parsed_length
 
     def compose(self):
         composer = ComposerText()
 
-        composer.compose_string_array([self.get_canonical_name(), self._get_value_as_str()], '=')
+        if self.get_canonical_name():
+            composer.compose_string_array([self.get_canonical_name(), self._get_value_as_str()], '=')
+        else:
+            composer.compose_string(self._get_value_as_str())
 
         return composer.composed
 
@@ -377,6 +390,9 @@ class FieldValueComponentQuotedString(FieldValueComponentKeyValueBase):
     def _get_value_as_str(self):
         return '"{}"'.format(self.value)
 
+    def _get_value_as_simple_type(self):
+        return self.value
+
     @classmethod
     def _parse_value(cls, parser):
 
@@ -385,26 +401,96 @@ class FieldValueComponentQuotedString(FieldValueComponentKeyValueBase):
         parser.parse_separator('"', 0, None)
 
 
+@attr.s
+class FieldValueComponentDateTime(FieldValueComponentKeyValueBase):
+    value = attr.ib(validator=attr.validators.instance_of(datetime.datetime))
+
+    @classmethod
+    @abc.abstractmethod
+    def get_canonical_name(cls):
+        raise NotImplementedError()
+
+    @classmethod
+    def _parse_value(cls, parser):
+        parser.parse_date_time('value')
+
+    def _get_value_as_simple_type(self):
+        return self.value.strftime('%a, %d %b %Y %H:%M:%S GMT')
+
+
 @attr.s
 class FieldValueComponentTimeDelta(FieldValueComponentKeyValueBase):
-    value = attr.ib(validator=attr.validators.instance_of(datetime.timedelta))
+    value = attr.ib(
+        converter=convert_value_to_object(datetime.timedelta),
+        validator=attr.validators.instance_of(datetime.timedelta)
+    )
 
     @classmethod
     @abc.abstractmethod
     def get_canonical_name(cls):
         raise NotImplementedError()
 
+    @classmethod
+    def convert(cls, value):
+        if isinstance(value, cls):
+            return value
+        if isinstance(value, datetime.timedelta):
+            return cls(value)
+
+        return cls(datetime.timedelta(seconds=value))
+
     @classmethod
     def _parse_value(cls, parser):
         parser.parse_time_delta('value')
 
-    def _get_value_as_str(self):
-        return str(int(self.value.total_seconds()))
+    def _get_value_as_simple_type(self):
+        return int(self.value.total_seconds())
 
     def _as_markdown(self, level):
         return self._markdown_result(str(self.value), level)
 
 
+@attr.s
+class FieldValueComponentStringBase64(FieldValueComponentQuotedString):
+    value = attr.ib(
+        converter=convert_base64_data(),
+        validator=attr.validators.instance_of(Base64Data)
+    )
+
+    @classmethod
+    @abc.abstractmethod
+    def get_canonical_name(cls):
+        raise NotImplementedError()
+
+    @classmethod
+    def _check_name(cls, name):
+        cls._check_name_insensitive(name)
+
+    @classmethod
+    def _parse_value(cls, parser):
+        parser.parse_string_by_length('value')
+
+
+@attr.s
+class FieldValueComponentBool(FieldValueComponentKeyValueBase):
+    value = attr.ib(validator=attr.validators.instance_of(bool))
+
+    @classmethod
+    @abc.abstractmethod
+    def get_canonical_name(cls):
+        raise NotImplementedError()
+
+    def _get_value_as_str(self):
+        return 'yes' if self.value else 'no'
+
+    def _get_value_as_simple_type(self):
+        return self.value
+
+    @classmethod
+    def _parse_value(cls, parser):
+        parser.parse_bool('value')
+
+
 @attr.s
 class FieldValueComponentNumber(FieldValueComponentKeyValueBase):
     value = attr.ib(validator=attr.validators.instance_of(int))
@@ -419,6 +505,26 @@ class FieldValueComponentNumber(FieldValueComponentKeyValueBase):
         parser.parse_numeric('value')
 
 
+@attr.s
+class FieldValueComponentFloat(FieldValueComponentNumber):
+    value = attr.ib(
+        converter=float,
+        validator=attr.validators.instance_of(float)
+    )
+
+    @classmethod
+    @abc.abstractmethod
+    def get_canonical_name(cls):
+        raise NotImplementedError()
+
+    def _get_value_as_simple_type(self):
+        return self.value
+
+    @classmethod
+    def _parse_value(cls, parser):
+        parser.parse_float('value')
+
+
 @attr.s
 class FieldValueComponentPercent(FieldValueComponentNumber):
     def __attrs_post_init__(self):
@@ -431,6 +537,57 @@ class FieldValueComponentPercent(FieldValueComponentNumber):
         raise NotImplementedError()
 
 
+@attr.s
+class FieldValueComponentStringEnumParams(object):
+    code = attr.ib(validator=attr.validators.instance_of(six.string_types))
+
+
+@attr.s
+class FieldValueComponentStringEnum(FieldValueComponentKeyValueBase):
+    value = attr.ib()
+
+    @classmethod
+    @abc.abstractmethod
+    def get_canonical_name(cls):
+        raise NotImplementedError()
+
+    @classmethod
+    @abc.abstractmethod
+    def _get_value_type(cls):
+        raise NotImplementedError()
+
+    @value.validator
+    def _validator_value(self, _, value):
+        if not isinstance(value, self._get_value_type()):
+            raise InvalidValue(value, type(self), 'value')
+
+    @classmethod
+    def _parse_value(cls, parser):
+        try:
+            parser.parse_parsable('value', cls._get_value_type())
+        except InvalidValue as e:
+            six.raise_from(InvalidValue(e.value.decode('ascii'), cls, 'value'), e)
+
+    def _get_value_as_simple_type(self):
+        return self.value.value.code
+
+
+@attr.s
+class FieldValueComponentStringEnumOption(FieldValueComponentStringEnum):
+    @classmethod
+    @abc.abstractmethod
+    def _get_value_type(cls):
+        raise NotImplementedError()
+
+    @classmethod
+    def get_canonical_name(cls):
+        return ''
+
+    @classmethod
+    def _check_name(cls, name):
+        pass
+
+
 @attr.s
 class FieldValueComponentString(FieldValueComponentKeyValueBase):
     value = attr.ib(validator=attr.validators.optional(attr.validators.instance_of(six.string_types)))
@@ -467,7 +624,7 @@ class FieldValueComponentUrl(FieldValueComponentKeyValueBase):
     def _parse_value(cls, parser):
         parser.parse_string_by_length('value', item_class=convert_url())
 
-    def _get_value_as_str(self):
+    def _get_value_as_simple_type(self):
         if self.value.scheme == 'mailto':
             value = 'mailto:' + self.value.path[1:]
         else:
@@ -476,7 +633,7 @@ class FieldValueComponentUrl(FieldValueComponentKeyValueBase):
         return value
 
     def _as_markdown(self, level):
-        return self._markdown_result(self._get_value_as_str(), level)
+        return self._markdown_result(self._get_value_as_simple_type(), level)
 
 
 class FieldValueBase(ParsableBase, Serializable):
@@ -489,25 +646,54 @@ class FieldValueBase(ParsableBase, Serializable):
     def compose(self):
         raise NotImplementedError()
 
-
-class FieldValueMultiple(FieldValueBase):
-    @classmethod
-    @abc.abstractmethod
-    def _get_header_value_list_class(cls):
-        raise NotImplementedError()
-
     @classmethod
-    def _get_attr_to_component_name_dict(cls, attr_fields_dict):
-        attr_to_component_name_dict = {}
+    def _get_attr_to_validator_type_dict(cls, attr_fields_dict):
+        attr_to_component_name_dict = []
 
         for attribute in attr_fields_dict.values():
             validator = attribute.validator
             if is_validator_optional(validator):
                 validator = validator.validator
 
-            attr_to_component_name_dict[attribute.name] = validator.type
+            attr_to_component_name_dict.append((attribute.name, validator.type))
+
+        return collections.OrderedDict(attr_to_component_name_dict)
 
-        return attr_to_component_name_dict
+
+class FieldsJson(FieldValueBase):
+    @classmethod
+    def _parse(cls, parsable):
+        try:
+            raw_values = json.loads(parsable.decode('ascii'), object_pairs_hook=collections.OrderedDict)
+        except ValueError as e:  # json.decoder.JSONDecodeError is derived from ValueError
+            six.raise_from(InvalidValue(six.ensure_text(parsable, 'ascii'), cls, 'value'), e)
+
+        attr_fields_dict = attr.fields_dict(cls)
+
+        return cls(**{
+            attribute_name: raw_values[validator_class.get_canonical_name()]
+            for attribute_name, validator_class in cls._get_attr_to_validator_type_dict(attr_fields_dict).items()
+            if validator_class.get_canonical_name() in raw_values
+        }), len(parsable)
+
+    def compose(self):
+        attr_fields_dict = attr.fields_dict(type(self))
+
+        return json.dumps(collections.OrderedDict([
+            (
+                validator_class.get_canonical_name(),
+                getattr(self, attribute_name)._get_value_as_simple_type()  # pylint: disable=protected-access
+            )
+            for attribute_name, validator_class in self._get_attr_to_validator_type_dict(attr_fields_dict).items()
+            if getattr(self, attribute_name) is not None
+        ])).encode('ascii')
+
+
+class FieldValueMultiple(FieldValueBase):
+    @classmethod
+    @abc.abstractmethod
+    def _get_header_value_list_class(cls):
+        raise NotImplementedError()
 
     @classmethod
     def _parse_basic_params(cls, attr_to_component_name_dict, attr_fields_dict, components, params):
@@ -524,10 +710,12 @@ class FieldValueMultiple(FieldValueBase):
                 if attribute.default == attr.NOTHING:
                     raise InvalidValue(None, cls, name)
 
+                component = attribute.default
+
             if attr_to_component_name_dict[name].get_canonical_name() in components:
                 parsable = components.pop(attr_to_component_name_dict[name].get_canonical_name())
-                if parsable is None:  # value is None in  case of optional values
-                    parsable = attr_to_component_name_dict[name].get_canonical_name()
+                if parsable is None:  # value is None in case of optional values
+                    parsable = component
                 else:
                     parsable = '='.join([attr_to_component_name_dict[name].get_canonical_name(), parsable])
                 params[name] = attr_to_component_name_dict[name].parse_exact_size(six.ensure_binary(parsable, 'ascii'))
@@ -553,7 +741,7 @@ class FieldValueMultiple(FieldValueBase):
                 extension = (name, attribute)
             else:
                 raise NotImplementedError()
-        attr_to_component_name_dict = cls._get_attr_to_component_name_dict(attr_fields_dict)
+        attr_to_component_name_dict = cls._get_attr_to_validator_type_dict(attr_fields_dict)
 
         components = cls._get_header_value_list_class().parse_exact_size(parsable).value
 

{CryptoParser-0.11.0 → CryptoParser-0.11.2}/cryptoparser/common/parse.py

@@ -206,8 +206,10 @@ class ParserText(ParserBase):
             'separator', self._parsed_length, separator, min_length, max_length
         )
 
-    def _parse_numeric_array(self, name, item_num, separator, converter):
+    def _parse_numeric_array(  # pylint: disable=too-many-arguments
+            self, name, item_num, separator, converter, is_floating):
         value = []
+        floating_point_found = False
         last_item_offset = self._parsed_length
         item_offset = self._parsed_length
         while True:
@@ -217,8 +219,14 @@ class ParserText(ParserBase):
             if item_offset == last_item_offset:
                 raise InvalidValue(self._parsable[self._parsed_length:], type(self), name)
 
-            if item_offset != last_item_offset:
-                value.append(converter(self._parsable[last_item_offset:item_offset]))
+            if (is_floating and
+                    not floating_point_found and item_offset < len(self._parsable) and
+                    six.indexbytes(self._parsable, item_offset) == ord('.')):
+                item_offset += 1
+                floating_point_found = True
+                continue
+
+            value.append(converter(self._parsable[last_item_offset:item_offset]))
 
             if item_offset == len(self._parsable) or (item_num is not None and len(value) == item_num):
                 break
@@ -230,20 +238,38 @@ class ParserText(ParserBase):
                     six.raise_from(InvalidValue(self._parsable[self._parsed_length:item_offset], type(self), name), e)
 
             last_item_offset = item_offset
+            floating_point_found = False
 
         return value, item_offset - self._parsed_length
 
     def parse_numeric(self, name, converter=int):
-        value, parsed_length = self._parse_numeric_array(name, 1, None, converter)
+        value, parsed_length = self._parse_numeric_array(name, 1, None, converter, False)
+        self._parsed_values[name] = value[0]
+        self._parsed_length += parsed_length
+
+    def parse_float(self, name, converter=float):
+        value, parsed_length = self._parse_numeric_array(name, 1, None, converter, True)
         self._parsed_values[name] = value[0]
         self._parsed_length += parsed_length
 
     def parse_numeric_array(self, name, item_num, separator, converter=int):
-        value, parsed_length = self._parse_numeric_array(name, item_num, separator, converter)
+        value, parsed_length = self._parse_numeric_array(name, item_num, separator, converter, False)
 
         self._parsed_values[name] = value
         self._parsed_length += parsed_length
 
+    def parse_bool(self, name):
+        for string_value, bool_value in (('yes', True), ('no', False)):
+            try:
+                self.parse_string(name, string_value)
+            except InvalidValue:
+                pass
+            else:
+                self._parsed_values[name] = bool_value
+                break
+        else:
+            raise InvalidValue(self._parsable[self._parsed_length:], type(self), name)
+
     def parse_string(self, name, value):
         min_length = len(value)
         max_length = min_length
@@ -442,7 +468,7 @@ class ParserText(ParserBase):
         self._parsed_length = len(self._parsable)
 
     def parse_time_delta(self, name):
-        value, parsed_length = self._parse_numeric_array(name, 1, None, int)
+        value, parsed_length = self._parse_numeric_array(name, 1, None, int, False)
 
         try:
             time_delta = datetime.timedelta(seconds=value[0])
@@ -586,10 +612,14 @@ class ParserBinary(ParserBase):
             six.raise_from(InvalidValue(value, converter, name), e)
         self._parsed_length += len(parsed_bytes)
 
-    def parse_raw(self, name, size):
+    def parse_raw(self, name, size, converter=bytearray):
         parsed_bytes = self._parse_bytes(size)
 
-        self._parsed_values[name] = bytearray(parsed_bytes)
+        try:
+            self._parsed_values[name] = converter(parsed_bytes)
+        except ValueError as e:
+            six.raise_from(InvalidValue(parsed_bytes, converter, name), e)
+
         self._parsed_length += size
 
     def parse_string(self, name, item_size, encoding, converter=str):
@@ -753,6 +783,9 @@ class ComposerText(ComposerBase):
     def compose_numeric_array(self, values, separator):
         self._compose_numeric_array(values, separator)
 
+    def compose_bool(self, value):
+        self.compose_string('yes' if value else 'no')
+
     def compose_string(self, value):
         self._compose_string_array([value, ], encoding=self._encoding, separator='')
 

{CryptoParser-0.11.0 → CryptoParser-0.11.2}/cryptoparser/common/x509.py

@@ -1,10 +1,14 @@
 # -*- coding: utf-8 -*-
 
+import collections
 import datetime
 import enum
 
 import attr
 
+import asn1crypto
+
+from cryptodatahub.common.key import PublicKeyX509Base
 from cryptodatahub.common.stores import CertificateTransparencyLog, CertificateTransparencyLogParamsBase
 
 from cryptoparser.common.base import (
@@ -99,3 +103,21 @@ class SignedCertificateTimestampList(VectorParsable):
             fallback_class=None,
             min_byte_num=0, max_byte_num=2 ** 16 - 1
         )
+
+
+class PublicKeyX509(PublicKeyX509Base):
+    @property
+    def signed_certificate_timestamps(self):
+        for extension in self._certificate['tbs_certificate']['extensions']:
+            if extension['extn_id'].dotted == '1.3.6.1.4.1.11129.2.4.2':
+                asn1_value = asn1crypto.core.load(bytes(extension['extn_value']))
+                return SignedCertificateTimestampList.parse_exact_size(bytes(asn1_value))
+
+        return SignedCertificateTimestampList([])
+
+    def _asdict(self):
+        dict_value = super(PublicKeyX509, self)._asdict()
+
+        return collections.OrderedDict(list(dict_value.items()) + [
+            ('signed_certificate_timestamps', self.signed_certificate_timestamps),
+        ])