drheaderplus 3.0.0__py3-none-any.whl

drheader/__init__.py

@@ -0,0 +1,4 @@
+
+"""Top-level package for drHEADer core."""
+
+from drheader.core import Drheader  # noqa

drheader/cli/cli.py

@@ -0,0 +1,252 @@
+"""Console script for drheader."""
+import ast
+import json
+import logging
+import os
+import sys
+from json import JSONDecodeError
+
+import click
+import jsonschema
+from click import Choice, File, ParamType
+
+from drheader import Drheader
+from drheader.cli import utils
+
+_OUTPUT_TYPES = ['json', 'table']
+
+
+class URLParamType(ParamType):
+
+    name = 'URL'
+
+    def convert(self, value, param, ctx):
+        if not value.startswith('http'):
+            scheme = click.prompt('Please select a scheme', type=Choice(['http', 'https'], case_sensitive=False))
+            value = f'{scheme}://{value}'
+        return value
+
+
+@click.group(context_settings={'show_default': True})
+@click.version_option()
+def main():
+    """Console script for drheader."""
+
+
+@main.group()
+def compare():
+    """Compare headers with drheader."""
+
+
+@main.group()
+def scan():
+    """Scan endpoints with drheader."""
+
+
+@compare.command(context_settings={
+    'default_map': {
+        'output': 'table'
+    }
+})
+@click.argument('file', type=File(), required=True)
+@click.option('--cross-origin-isolated', '-co', is_flag=True, help='Enable cross-origin isolation validations')
+@click.option('--debug', '-d', is_flag=True, help='Enable debug logging')
+@click.option('--junit', '-j', is_flag=True, help='Generate a JUnit report')
+@click.option('--merge', '-m', is_flag=True, help='Merge a custom ruleset with the default rules')
+@click.option('--output', '-o', type=Choice(_OUTPUT_TYPES, case_sensitive=False), help='Report output format')
+@click.option('--rules-file', '-rf', type=File(), help='Use a custom ruleset, loaded from file')
+@click.option('--rules-uri', '-ru', metavar='URI', help='Use a custom ruleset, downloaded from URI')
+def single(file, cross_origin_isolated, debug, junit, merge, output, rules_file, rules_uri):
+    if debug:
+        logging.basicConfig(level=logging.DEBUG)
+
+    scanner = Drheader(headers=json.loads(file.read()))
+    rules = utils.get_rules(rules_file=rules_file, rules_uri=rules_uri, merge_default=merge)
+    report = scanner.analyze(rules=rules, cross_origin_isolated=cross_origin_isolated)
+
+    if output == 'json':
+        click.echo(json.dumps(report, indent=4))
+    else:
+        click.echo()
+        if not report:
+            click.echo('No issues found!')
+        else:
+            click.echo(f'{len(report)} issues found')
+            click.echo(utils.tabulate_report(report))
+    if junit:
+        utils.file_junit_report(rules, report)
+
+    sys.exit(os.EX_SOFTWARE if report else os.EX_OK)
+
+
+@compare.command(context_settings={
+    'default_map': {
+        'output': 'table'
+    }
+})
+@click.argument('file', type=File(), required=True)
+@click.option('--cross-origin-isolated', '-co', is_flag=True, help='Enable cross-origin isolation validations')
+@click.option('--debug', '-d', is_flag=True, help='Enable debug logging')
+@click.option('--merge', '-m', is_flag=True, help='Merge a custom ruleset with the default rules')
+@click.option('--output', '-o', type=Choice(_OUTPUT_TYPES, case_sensitive=False), help='Report output format')
+@click.option('--rules-file', '-rf', type=File(), help='Use a custom ruleset, loaded from file')
+@click.option('--rules-uri', '-ru', metavar='URI', help='Use a custom ruleset, downloaded from URI')
+def bulk(file, cross_origin_isolated, debug, merge, output, rules_file, rules_uri):
+    if debug:
+        logging.basicConfig(level=logging.DEBUG)
+
+    data = json.loads(file.read())
+    with open(os.path.join(os.path.dirname(__file__), '../resources/cli/bulk_compare_schema.json')) as schema:
+        schema = json.load(schema)
+        jsonschema.validate(instance=data, schema=schema)
+
+    audit = []
+    rules = utils.get_rules(rules_file=rules_file, rules_uri=rules_uri, merge_default=merge)
+    for target in data:
+        try:
+            scanner = Drheader(headers=target['headers'])
+            report = scanner.analyze(rules=rules, cross_origin_isolated=cross_origin_isolated)
+            audit.append({'url': target['url'], 'report': report})
+        except Exception as e:
+            audit.append({'url': target['url'], 'report': [], 'error': str(e)})
+
+    if output == 'json':
+        click.echo(json.dumps(audit, indent=4))
+    else:
+        for target in audit:
+            click.echo()
+            if target.get('error'):
+                click.echo(f"{target['url']}: {target['error']}")
+            elif not target['report']:
+                click.echo(f"{target['url']}: No issues found!")
+            else:
+                click.echo(f"{target['url']}: {len(target['report'])} issues found")
+                click.echo(utils.tabulate_report(target['report']))
+
+    for target in audit:
+        if target.get('report') or target.get('error'):
+            sys.exit(os.EX_SOFTWARE)
+    else:
+        sys.exit(os.EX_OK)
+
+
+@scan.command(context_settings={
+    'default_map': {
+        'output': 'table'
+    },
+    'ignore_unknown_options': True
+})
+@click.argument('target_url', type=URLParamType(), required=True)
+@click.argument('request_args', type=click.UNPROCESSED, nargs=-1)
+@click.option('--cross-origin-isolated', '-co', is_flag=True, help='Enable cross-origin isolation validations')
+@click.option('--debug', '-d', is_flag=True, help='Enable debug logging')
+@click.option('--junit', '-j', is_flag=True, help='Generate a JUnit report')
+@click.option('--merge', '-m', is_flag=True, help='Merge a custom ruleset with the default rules')
+@click.option('--output', '-o', type=Choice(_OUTPUT_TYPES, case_sensitive=False), help='Report output format')
+@click.option('--rules-file', '-rf', type=File(), help='Use a custom ruleset, loaded from file')
+@click.option('--rules-uri', '-ru', metavar='URI', help='Use a custom ruleset, downloaded from URI')
+def single(target_url, request_args, cross_origin_isolated, debug, junit, merge, output, rules_file, rules_uri):  # noqa: E501, F811
+    if debug:
+        logging.basicConfig(level=logging.DEBUG)
+
+    kwargs = {}
+    for i in range(0, len(request_args), 2):
+        key = request_args[i].strip('-').replace('-', '_')
+        try:
+            kwargs[key] = json.loads(request_args[i + 1])
+        except JSONDecodeError:
+            try:
+                kwargs[key] = ast.literal_eval(request_args[i + 1])  # This handles bytes and tuples
+            except (SyntaxError, ValueError):
+                kwargs[key] = request_args[i + 1]
+
+    scanner = Drheader(url=target_url, **kwargs)
+    rules = utils.get_rules(rules_file=rules_file, rules_uri=rules_uri, merge_default=merge)
+    report = scanner.analyze(rules=rules, cross_origin_isolated=cross_origin_isolated)
+
+    if output == 'json':
+        click.echo(json.dumps(report, indent=4))
+    else:
+        click.echo()
+        if not report:
+            click.echo('No issues found!')
+        else:
+            click.echo(f"{target_url}: {len(report)} issues found")
+            click.echo(utils.tabulate_report(report))
+    if junit:
+        utils.file_junit_report(rules, report)
+
+    sys.exit(os.EX_SOFTWARE if report else os.EX_OK)
+
+
+@scan.command(context_settings={
+    'default_map': {
+        'file_format': 'json',
+        'output': 'table'
+    }
+})
+@click.argument('file', type=File(), required=True)
+@click.option('--cross-origin-isolated', '-co', is_flag=True, help='Enable cross-origin isolation validations')
+@click.option('--debug', '-d', is_flag=True, help='Enable debug logging')
+@click.option('--file-format', '-ff', type=Choice(['json', 'txt'], case_sensitive=False), help='FILE input format')
+@click.option('--merge', '-m', is_flag=True, help='Merge a custom ruleset with the default rules')
+@click.option('--output', '-o', type=Choice(_OUTPUT_TYPES, case_sensitive=False), help='Report output format')
+@click.option('--rules-file', '-rf', type=File(), help='Use a custom ruleset, loaded from file')
+@click.option('--rules-uri', '-ru', metavar='URI', help='Use a custom ruleset, downloaded from URI')
+def bulk(file, cross_origin_isolated, debug, file_format, merge, output, rules_file, rules_uri):  # noqa: F811
+    if debug:
+        logging.basicConfig(level=logging.DEBUG)
+
+    if file_format == 'txt':
+        urls = [{'url': url} for url in list(filter(None, file.read().splitlines()))]
+    else:
+        urls = json.loads(file.read())
+        with open(os.path.join(os.path.dirname(__file__), '../resources/cli/bulk_scan_schema.json')) as schema:
+            schema = json.load(schema)
+            jsonschema.validate(instance=urls, schema=schema)
+
+    audit = []
+    rules = utils.get_rules(rules_file=rules_file, rules_uri=rules_uri, merge_default=merge)
+    for target in urls:
+        for key, value in target.items():
+            try:
+                target[key] = ast.literal_eval(value)  # This handles bytes and tuples
+            except (SyntaxError, ValueError):
+                target[key] = value
+
+        try:
+            scanner = Drheader(**target)
+            report = scanner.analyze(rules=rules, cross_origin_isolated=cross_origin_isolated)
+            audit.append({'url': target['url'], 'report': report})
+        except Exception as e:
+            audit.append({'url': target['url'], 'report': [], 'error': str(e)})
+
+    if output == 'json':
+        click.echo(json.dumps(audit, indent=4))
+    else:
+        for target in audit:
+            click.echo()
+            if target.get('error'):
+                click.echo(f"{target['url']}: {target['error']}")
+            elif not target['report']:
+                click.echo(f"{target['url']}: No issues found!")
+            else:
+                click.echo(f"{target['url']}: {len(target['report'])} issues found")
+                click.echo(utils.tabulate_report(target['report']))
+
+    for target in audit:
+        if target.get('report') or target.get('error'):
+            sys.exit(os.EX_SOFTWARE)
+    else:
+        sys.exit(os.EX_OK)
+
+
+def start():
+    try:
+        main()
+    except Exception as e:
+        click.secho(str(e), fg='red')
+
+
+if __name__ == '__main__':
+    start()

drheader/cli/utils.py

@@ -0,0 +1,58 @@
+"""Utility functions for cli module."""
+
+import os
+
+import tabulate
+from junitparser import Failure, JUnitXml, TestCase, TestSuite
+
+from drheader import utils
+
+
+def get_rules(rules_file=None, rules_uri=None, merge_default=False):
+    if rules_file or rules_uri:
+        return utils.load_rules(rules_file=rules_file, rules_uri=rules_uri, merge_default=merge_default)
+    else:
+        return utils.default_rules()
+
+
+def tabulate_report(report):
+    rows = []
+    final_string = ''
+
+    for validation_error in report:
+        values = [[k, v] for k, v in validation_error.items()]
+        rows.append(values)
+    for validation_error in rows:
+        final_string += '----\n'
+        final_string += tabulate.tabulate(validation_error, tablefmt='presto') + '\n'
+
+    return final_string
+
+
+def file_junit_report(rules, report):
+    """Generates a JUnit XML report from a scan result.
+
+    Args:
+        rules (dict): The rules used to perform the scan.
+        report (list): The report generated from the scan.
+    """
+    test_suite = TestSuite('drHEADer')
+
+    for header in rules:
+        test_case = None
+        for validation_error in report:
+            if (title := validation_error.get('rule')).startswith(header):
+                validation_error = {k: v for k, v in validation_error.items() if k != 'rule'}
+                test_case = TestCase(title)
+                failure = Failure(message=validation_error.pop('message'))
+                failure.text = str(validation_error)
+                test_case.result = [failure]
+                test_suite.add_testcase(test_case)
+        if not test_case:
+            test_case = TestCase(header)
+            test_suite.add_testcase(test_case)
+
+    os.makedirs('reports', exist_ok=True)
+    xml = JUnitXml()
+    xml.add_testsuite(test_suite)
+    xml.write('reports/junit.xml')

drheader/core.py

@@ -0,0 +1,185 @@
+"""Main module and entry point for analysis."""
+import json
+import logging
+import os
+
+import requests
+from requests.structures import CaseInsensitiveDict
+
+from drheader import utils
+from drheader.report import Reporter
+from drheader.validators.cookie_validator import CookieValidator
+from drheader.validators.directive_validator import DirectiveValidator
+from drheader.validators.header_validator import HeaderValidator
+
+_ALLOWED_HTTP_METHODS = ['delete', 'get', 'head', 'options', 'patch', 'post', 'put']
+_CROSS_ORIGIN_HEADERS = ['cross-origin-embedder-policy', 'cross-origin-opener-policy']
+
+with open(os.path.join(os.path.dirname(__file__), 'resources/delimiters.json')) as delimiters:
+    _DELIMITERS = CaseInsensitiveDict(json.load(delimiters))
+
+
+class Drheader:
+    """Main class and entry point for analysis.
+
+    Attributes:
+        headers (CaseInsensitiveDict): The headers to analyse.
+        cookies (CaseInsensitiveDict): The cookies to analyse.
+        reporter (Reporter): Reporter instance that generates and holds the final report.
+    """
+
+    def __init__(self, headers=None, url=None, **kwargs):
+        """Initialises a Drheader instance.
+
+        At least one of <headers> and <url> must be defined. The value passed in <headers> is treated differently
+        depending on whether a URL is provided. If a URL is provided, the headers passed are treated as request headers
+        and are sent with the HTTP request to <url>. Otherwise, they are the raw headers that are analysed.
+
+        Args:
+            headers (dict): Either headers to analyse or a dict of headers to send with the request to <url>.
+            url (str): URL from which to retrieve the headers to analyse.
+
+        Raises:
+            ValueError: If neither headers nor url is provided, or if url is not a valid URL.
+        """
+        if not url:
+            if not headers:
+                raise ValueError("Nothing provided for analysis. Either 'headers' or 'url' must be defined")
+            else:
+                headers_to_analyse = json.loads(headers) if isinstance(headers, str) else headers
+        else:
+            headers_to_analyse = _get_headers_from_url(url, headers=headers, **kwargs)
+
+        self.cookies = CaseInsensitiveDict()
+        self.headers = CaseInsensitiveDict(headers_to_analyse)
+        self.reporter = Reporter()
+
+        for cookie in self.headers.get('set-cookie', []):
+            cookie = cookie.split('=', 1)
+            self.cookies[cookie[0]] = cookie[1]
+
+    def analyze(self, rules=None, cross_origin_isolated=False):
+        """Analyses headers against a drHEADer ruleset.
+
+        Args:
+            rules (dict): (optional) The rules against which to assess the headers. Default rules are used if undefined.
+            cross_origin_isolated (bool): (optional) A flag to enable cross-origin isolation rules. Default is False.
+
+        Returns:
+            A list containing all the rule violations found during analysis. The report consists of individual dict
+            items per header and rule. Each item in the report will detail the non-compliant header, the rule violated
+            and its associated severity, and, if applicable, the observed value of the header, any expected, disallowed
+            or anomalous values, and the correct delimiter. For example:
+            {
+                'rule': 'Referrer-Policy',
+                'message': 'Value does not match security policy. Exactly one of the expected items was expected',
+                'severity': 'high',
+                'value': 'origin-when-cross-origin'
+                'expected': ['same-origin', 'strict-origin-when-cross-origin']
+            }
+        """
+        if not rules:
+            rules = _translate_to_case_insensitive_dict(utils.default_rules())
+        else:
+            rules = _translate_to_case_insensitive_dict(rules)
+
+        header_validator = HeaderValidator(self.headers)
+        directive_validator = DirectiveValidator(self.headers)
+        cookie_validator = CookieValidator(self.cookies)
+
+        for header, rule_config in rules.items():
+            if header.lower() in _CROSS_ORIGIN_HEADERS and not cross_origin_isolated:
+                logging.info(f"Cross-origin isolation validations are not enabled. Skipping header '{header}'")
+                continue
+
+            if header.lower() != 'set-cookie':
+                self._validate_rules(rule_config, header_validator, header)
+            elif header in self.headers:  # Validates global rules for cookies e.g. all cookies must contain 'secure'
+                for cookie in self.cookies:
+                    self._validate_rules(rule_config, cookie_validator, header, cookie=cookie)
+
+            if 'directives' in rule_config and header in self.headers:
+                for directive, directive_config in rule_config['directives'].items():
+                    self._validate_rules(directive_config, directive_validator, header, directive=directive)
+            if 'cookies' in rule_config and header.lower() == 'set-cookie':  # Validates individual rules for cookies e.g. cookie session_id must contain 'samesite=strict'  # noqa:E501
+                for cookie, cookie_config in rule_config['cookies'].items():
+                    self._validate_rules(cookie_config, cookie_validator, header, cookie=cookie)
+        return self.reporter.report
+
+    def _validate_rules(self, config, validator, header, **kwargs):
+        """Validates rules for a single header, directive or cookie."""
+        config['delimiters'] = _DELIMITERS.get(header)
+        required = str(config['required']).strip().lower()
+
+        if required == 'true':
+            if report_item := validator.exists(config, header, **kwargs):
+                self._add_to_report(report_item)
+            else:
+                self._validate_value_rules(config, validator, header, **kwargs)
+        elif required == 'false':
+            if report_item := validator.not_exists(config, header, **kwargs):
+                self._add_to_report(report_item)
+        elif required == 'optional':
+            if cookie := kwargs.get('cookie'):
+                is_present = cookie in self.cookies
+            elif directive := kwargs.get('directive'):
+                is_present = directive in utils.parse_policy(self.headers[header], **_DELIMITERS[header], keys_only=True)  # noqa: E501
+            else:
+                is_present = header in self.headers
+
+            if is_present:
+                self._validate_value_rules(config, validator, header, **kwargs)
+
+    def _validate_value_rules(self, config, validator, header, **kwargs):
+        """Validates rules for a single header, directive or cookie."""
+        if 'value' in config:
+            if report_item := validator.value(config, header, **kwargs):
+                self._add_to_report(report_item)
+        elif 'value-any-of' in config:
+            if report_item := validator.value_any_of(config, header, **kwargs):
+                self._add_to_report(report_item)
+        elif 'value-one-of' in config:
+            if report_item := validator.value_one_of(config, header, **kwargs):
+                self._add_to_report(report_item)
+        else:
+            if 'must-avoid' in config:
+                if report_item := validator.must_avoid(config, header, **kwargs):
+                    self._add_to_report(report_item)
+            if 'must-contain' in config:
+                if report_item := validator.must_contain(config, header, **kwargs):
+                    self._add_to_report(report_item)
+            if 'must-contain-one' in config:
+                if report_item := validator.must_contain_one(config, header, **kwargs):
+                    self._add_to_report(report_item)
+
+    def _add_to_report(self, report_item):
+        """Adds a finding or list of findings to the final report."""
+        try:
+            self.reporter.add_item(report_item)
+        except AttributeError:  # For must-avoid rules on policy headers (CSP, Permissions-Policy)
+            for item in report_item:  # A separate report item is created for each directive that violates the must-avoid rule e.g. multiple directives containing 'unsafe-inline'  # noqa:E501
+                self.reporter.add_item(item)
+
+
+def _get_headers_from_url(url, method='head', **kwargs):
+    """Retrieves headers from a URL."""
+    if method.strip().lower() not in _ALLOWED_HTTP_METHODS:
+        raise ValueError(f"'{method}' is not an allowed HTTP method")
+
+    if 'timeout' not in kwargs:
+        kwargs['timeout'] = 5
+    if 'allow_redirects' not in kwargs:
+        kwargs['allow_redirects'] = True
+
+    response = requests.request(method, url, **kwargs)  # noqa: S113
+    response_headers = response.headers
+    response_headers['set-cookie'] = response.raw.headers.getlist('Set-Cookie')
+    return response_headers
+
+
+def _translate_to_case_insensitive_dict(dict_to_translate):
+    """Recursively transforms a dict into a case-insensitive dict."""
+    for key, value in dict_to_translate.items():
+        if isinstance(value, dict):
+            dict_to_translate[key] = _translate_to_case_insensitive_dict(value)
+    return CaseInsensitiveDict(dict_to_translate)

drheader/report.py

@@ -0,0 +1,70 @@
+"""Primary module for report generation and storage."""
+from enum import Enum
+from typing import NamedTuple
+
+
+class Reporter:
+    """Class to generate and store reports from a scan.
+
+    Attributes:
+        report (list): The report detailing validation failures encountered during a scan.
+    """
+
+    def __init__(self):
+        """Initialises a Reporter instance with an empty report."""
+        self.report = []
+
+    def add_item(self, item):
+        """Adds a validation failure to the report.
+
+        Args:
+            item (ReportItem): The validation failure to be added.
+        """
+        finding = {}
+        if item.directive:
+            finding['rule'] = f'{item.header} - {item.directive}'
+            finding['message'] = item.error_type.value.format('Directive')
+        elif item.cookie:
+            finding['rule'] = f'{item.header} - {item.cookie}'
+            finding['message'] = item.error_type.value.format('Cookie')
+        else:
+            finding['rule'] = item.header
+            finding['message'] = item.error_type.value.format('Header')
+
+        finding['severity'] = item.severity
+
+        if item.value:
+            finding['value'] = item.value
+        if item.expected:
+            finding['expected'] = item.expected
+            if len(item.expected) > 1 and item.delimiter:
+                finding['delimiter'] = item.delimiter
+        elif item.avoid:
+            finding['avoid'] = item.avoid
+        if item.anomalies:
+            finding['anomalies'] = item.anomalies
+        self.report.append(finding)
+
+
+class ErrorType(Enum):
+    AVOID = 'Must-Avoid directive included'
+    CONTAIN = 'Must-Contain directive missed'
+    CONTAIN_ONE = 'Must-Contain-One directive missed. At least one of the expected items was expected'
+    DISALLOWED = '{} should not be returned'
+    REQUIRED = '{} not included in response'
+    VALUE = 'Value does not match security policy'
+    VALUE_ANY = 'Value does not match security policy. At least one of the expected items was expected'
+    VALUE_ONE = 'Value does not match security policy. Exactly one of the expected items was expected'
+
+
+class ReportItem(NamedTuple):
+    severity: str
+    error_type: ErrorType
+    header: str
+    directive: str = None
+    cookie: str = None
+    value: str = None
+    avoid: list = None
+    expected: list = None
+    anomalies: list = None
+    delimiter: str = None

drheader/resources/cli/bulk_compare_schema.json

@@ -0,0 +1,20 @@
+{
+    "type": "array",
+    "items": {
+        "type": "object",
+        "properties": {
+            "url": {
+                "type": "string",
+                "format": "uri"
+            },
+            "headers": {
+                "type": "object"
+            }
+        },
+        "required": [
+            "url",
+            "headers"
+        ],
+        "additionalProperties": false
+    }
+}

drheader/resources/cli/bulk_scan_schema.json

@@ -0,0 +1,52 @@
+{
+    "type": "array",
+    "items": {
+        "type": "object",
+        "properties": {
+            "url": {
+                "type": "string",
+                "format": "uri"
+            },
+            "allow_redirects": {
+                "type": "boolean"
+            },
+            "auth": {
+                "type": "string"
+            },
+            "cert": {
+                "type": "string"
+            },
+            "cookies": {
+                "type": "object"
+            },
+            "data": {
+            },
+            "headers": {
+                "type": "object"
+            },
+            "json": {
+                "type": "object"
+            },
+            "method": {
+                "enum": [
+                    "DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"
+                ]
+            },
+            "params": {
+            },
+            "proxies": {
+                "type": "object"
+            },
+            "timeout": {
+                "type": ["number", "string"]
+            },
+            "verify": {
+                "type": ["boolean", "string"]
+            }
+        },
+        "required": [
+            "url"
+        ],
+        "additionalProperties": false
+    }
+}