toolsaf 0.2.0__py3-none-any.whl

toolsaf/adapters/android_manifest_scan.py

@@ -0,0 +1,89 @@
+"""Tool to read Android manifest XML"""
+
+import json
+from io import BufferedReader
+from pathlib import Path
+from xml.etree import ElementTree
+from typing import Dict, List, cast
+
+from toolsaf.main import ConfigurationException
+from toolsaf.common.basics import HostType
+from toolsaf.common.address import AnyAddress
+from toolsaf.core.components import Software
+from toolsaf.core.event_interface import PropertyEvent, EventInterface
+from toolsaf.core.model import IoTSystem
+from toolsaf.common.property import Properties, PropertyKey
+from toolsaf.adapters.tools import EndpointTool
+from toolsaf.common.traffic import EvidenceSource, Evidence
+from toolsaf.common.verdict import Verdict
+from toolsaf.common.android import MobilePermissions
+
+
+class AndroidManifestScan(EndpointTool):
+    """Android manifest XML tool"""
+    def __init__(self, system: IoTSystem) -> None:
+        super().__init__("android", ".xml", system)
+        self.tool.name = "Android Manifest"
+        self.categories = self.load_categories()
+
+    def load_categories(self) -> Dict[str, List[str]]:
+        """Load our Android permission category info from json"""
+        data_json_path = Path(__file__).parent / "data/android_permissions.json"
+        with open(data_json_path, "r", encoding="utf-8") as f:
+            r = json.load(f)
+            return cast(Dict[str, List[str]], r)
+
+    def process_endpoint(self, endpoint: AnyAddress, stream: BufferedReader, interface: EventInterface,
+                         source: EvidenceSource) -> None:
+        node = self.system.get_endpoint(endpoint)
+        if node.host_type != HostType.MOBILE:
+            raise ConfigurationException(f"Endpoint {endpoint} is not a Mobile application!")
+
+        if len(all_software := Software.list_software(node)) != 1:
+            raise ConfigurationException(
+                f"Endpoint {endpoint} needs to have 1 SW component only. Current number is {len(all_software)}!")
+        software = all_software[0]
+
+        evidence = Evidence(source)
+
+        tree = ElementTree.parse(stream)
+        key_set = set()
+        for uses_p in tree.getroot().iter('uses-permission'):
+            name = str(uses_p.attrib.get("{http://schemas.android.com/apk/res/android}name"))
+            if "." in name:
+                name = name[name.rindex(".") + 1:]
+
+            category = self.link_permission_to_category(name)
+            key = PropertyKey("permission", category.value)
+            key_set.add(key)
+
+            if self.load_baseline:
+                software.permissions.add(category.value)
+                ver = Verdict.PASS
+            else:
+                ver = Verdict.PASS if category.value in software.permissions else Verdict.FAIL
+
+            if self.send_events:
+                ev = PropertyEvent(evidence, software, key.verdict(ver))
+                interface.property_update(ev)
+
+        # Set verdict for permissions that were only present in the statement
+        for permission in software.permissions:
+            key = PropertyKey("permission", permission)
+            if key not in key_set:
+                key_set.add(key)
+                ver = Verdict.FAIL if not self.load_baseline else Verdict.PASS
+                ev = PropertyEvent(evidence, software, key.verdict(ver))
+                interface.property_update(ev)
+
+        if self.send_events:
+            ev = PropertyEvent(evidence, software, Properties.PERMISSIONS.value_set(key_set, self.tool.name))
+            interface.property_update(ev)
+
+    def link_permission_to_category(self, permission: str) -> MobilePermissions:
+        """Connect given permission to one of our categories.
+           If no category is found, uncategorized is returned by default."""
+        for category, permissions in self.categories.items():
+            if permission in permissions:
+                return MobilePermissions(category)
+        return MobilePermissions.UNCATEGORIZED

toolsaf/adapters/batch_import.py

@@ -0,0 +1,297 @@
+"""Batch tool-data import"""
+
+from datetime import datetime
+import json
+import logging
+import pathlib
+from io import BufferedReader
+from typing import Any, Dict, List, Optional, Set
+
+from toolsaf.common.address import Addresses, AnyAddress
+from toolsaf.common.basics import ExternalActivity
+from toolsaf.core.event_interface import EventInterface
+from toolsaf.core.model import Addressable, EvidenceNetworkSource, IoTSystem, NetworkNode
+from toolsaf.adapters.tool_finder import ToolDepiction, TOOL_FINDER
+from toolsaf.common.traffic import EvidenceSource
+
+
+class BatchImporter:
+    """Batch importer for importing a batch of files from a directory."""
+    def __init__(self, interface: EventInterface, label_filter: Optional['LabelFilter'] = None,
+                 load_baseline: bool=False) -> None:
+        self.interface = interface
+        self.system = interface.get_system()
+        self.label_filter = label_filter or LabelFilter()
+        self.logger = logging.getLogger("batch_importer")
+        self.load_baseline = load_baseline  # True to load baseline, false to check it
+        self.meta_file_count = 0
+
+        # collect evidence sources from visited tools
+        self.evidence: Dict[str, List[EvidenceSource]] = {}
+        # store batch hierarchy
+        self.batch_data: List[BatchData] = []
+
+    def import_batch(self, file: pathlib.Path) -> None:
+        """Import a batch of files from a directory or zip file recursively."""
+        if file.is_dir():
+            bd = BatchData(FileMetaInfo())
+            self._import_batch(file, bd)
+            if not self.meta_file_count:
+                self.logger.warning("No 00meta.json files found")
+            self.batch_data.append(bd)
+        else:
+            raise ValueError(f"Expected directory, got {file.as_posix()}")
+
+    def _import_batch(self, file: pathlib.Path, parent: 'BatchData') -> None:
+        """Import a batch of files from a directory or zip file recursively."""
+        parent_info = parent.meta_info
+        self.logger.info("scanning %s", file.as_posix())
+        if file.is_dir():
+            dir_name = file.name
+            meta_file = file / "00meta.json"
+            if meta_file.is_file():
+                # the directory has data files
+                if meta_file.stat().st_size == 0:
+                    info = FileMetaInfo(dir_name, parent=parent_info) # meta_file is empty
+                    b_data = BatchData(info)
+                else:
+                    try:
+                        with meta_file.open("rb") as f:
+                            b_data = BatchData.parse_from_stream(f, dir_name, self.system, parent_meta=parent_info)
+                            info = b_data.meta_info
+                    except Exception as e:
+                        raise ValueError(f"Error in {meta_file.as_posix()}") from e
+                self.evidence.setdefault(info.label, [])
+                self.meta_file_count += 1
+            else:
+                info = FileMetaInfo(parent=parent_info)
+                b_data = BatchData(info)
+            parent.sub_data.append(b_data)
+
+            # get tool info by file type
+            tool_dep = TOOL_FINDER.by_file_type(info.file_type)
+
+            # list files/directories to process, files first
+            proc_list = []
+            for a_file in sorted(file.iterdir(), key=lambda f: (f.is_dir(), f.name)):
+                if a_file == meta_file:
+                    continue
+                prefix = a_file.name[:1]
+                if prefix in {".", "_"}:
+                    continue
+                postfix = a_file.name[-1:]
+                if postfix in {"~"}:
+                    continue
+                proc_list.append(a_file)
+
+            # sort files to specified order, if any
+            if info.file_load_order:
+                proc_list = FileMetaInfo.sort_load_order(proc_list, info.file_load_order)
+
+            # filter by label
+            skip_processing = not self.label_filter.filter(info.label)
+
+            # give all files to the tool
+            all_files = tool_dep.filter_files_itself()
+            if all_files:
+                # process all files by one tool
+                self._do_process_files(proc_list, b_data, tool_dep, skip_processing)
+
+            if not info.label:
+                self.logger.info("skipping all files as no 00meta.json")
+
+            # recursively scan the directory
+            for a_file in proc_list:
+                if info and a_file.is_file():
+                    if all_files or not info.label:
+                        continue
+                    # process the files individually
+                    if not info.default_include and info.label not in self.label_filter.included:
+                        self.logger.debug("skipping (default=False) %s", a_file.as_posix())
+                        continue # skip file if not explicitly included
+                    with a_file.open("rb") as f:
+                        self._do_process(f, a_file, b_data, tool_dep, skip_processing)
+                else:
+                    self._import_batch(a_file, b_data)
+
+    def _do_process(self, stream: BufferedReader, file_path: pathlib.Path, data: 'BatchData', tool: ToolDepiction,
+                    skip_processing: bool) -> None:
+        """Process a file """
+        info = data.meta_info
+        if not skip_processing:
+            self.logger.info("processing (%s) %s", info.label, file_path.as_posix())
+
+        file_name = file_path.name
+        file_ext = file_path.suffix.lower()
+        reader = tool.create_tool(self.system, "" if info.from_pipe else file_ext)
+
+        try:
+            if reader:
+                ev = info.source.rename(name=reader.tool.name, base_ref=file_path.as_posix(),
+                                        label=info.label)
+                # tool-specific code can override, if knows better
+                ev.timestamp = datetime.fromtimestamp(file_path.stat().st_mtime)
+                self.evidence.setdefault(info.label, []).append(ev)
+                if skip_processing:
+                    self.logger.info("skipping (%s) %s", info.label, file_path.as_posix())
+                    return
+                reader.load_baseline = info.load_baseline or self.load_baseline
+                reader.process_file(stream, file_name, self.interface, ev)
+                data.sources.append(ev)
+                return
+
+        except Exception as e:
+            raise ValueError(f"Error in {file_name}") from e
+        self.logger.info("skipping unsupported '%s' type %s", file_name, info.file_type)
+
+    def _do_process_files(self, files: List[pathlib.Path], data: 'BatchData', tool: ToolDepiction,
+                          skip_processing: bool) -> None:
+        """Process files"""
+        info = data.meta_info
+        reader = tool.create_tool(self.system)
+        if not reader:
+            return
+        reader.load_baseline = info.load_baseline or self.load_baseline
+
+        if skip_processing:
+            self.logger.info("skipping (%s) data files", info.label)
+            ev = info.source.rename(name=reader.tool.name)
+            self.evidence.setdefault(info.label, []).append(ev)
+            return
+
+        unmapped = reader.get_processed_files()
+        for fn in files:
+            if not fn.is_file():
+                continue  # directories called later
+            ev = info.source.rename(name=reader.tool.name, base_ref=fn.as_posix(), label=info.label)
+            self.evidence.setdefault(info.label, []).append(ev)
+            with fn.open("rb") as f:
+                # tool-specific code can override, if knows better
+                ev.timestamp = datetime.fromtimestamp(fn.stat().st_mtime)
+                done = reader.process_file(f, fn.name, self.interface, ev)
+            if done:
+                data.sources.append(ev)
+                unmapped.remove(fn.name)
+            else:
+                self.logger.info("unprocessed (%s) file %s", info.label, fn.as_posix())
+        if unmapped:
+            self.logger.debug("no files for %s", sorted(unmapped))
+
+
+class FileMetaInfo:
+    """Batch file information."""
+    def __init__(self, label: str="", file_type: str="", parent: Optional['FileMetaInfo'] = None) -> None:
+        self.label = label
+        self.name = label
+        self.file_load_order: List[str] = []
+        self.file_type = file_type
+        self.from_pipe = False
+        self.load_baseline = False
+        self.default_include = True
+        self.source = EvidenceNetworkSource(file_type)
+        if parent:
+            self.source.address_map.update(parent.source.address_map)
+            self.source.activity_map.update(parent.source.activity_map)
+
+    @classmethod
+    def sort_load_order(cls, files: List[pathlib.Path], load_order: List[str]) -> List[pathlib.Path]:
+        """Sort files according to load order"""
+        proc_files = {f.name: f for f in files}
+        sorted_files = []
+        for fn in load_order:
+            if fn in proc_files:
+                sorted_files.append(proc_files[fn])
+                del proc_files[fn]
+        sorted_files.extend(proc_files.values())
+        return sorted_files
+
+    def __repr__(self) -> str:
+        return f"{self.name}: file_type: {self.file_type}, label: {self.label}"
+
+
+class BatchData:
+    """Batch data hierarchy"""
+    def __init__(self, meta_info: FileMetaInfo):
+        self.meta_info = meta_info
+        self.sub_data: List[BatchData] = []
+        self.address_map: Dict[AnyAddress, Addressable] = {}
+        self.activity_map: Dict[NetworkNode, ExternalActivity] = {}
+        self.sources: List[EvidenceSource] = []
+
+    def __repr__(self) -> str:
+        return str(self.meta_info)
+
+    @classmethod
+    def parse_from_stream(cls, stream: BufferedReader, directory_name: str, system: IoTSystem,
+                          parent_meta: Optional['FileMetaInfo'] = None) -> 'BatchData':
+        """Parse from stream"""
+        return cls.parse_from_json(json.load(stream), directory_name, system, parent_meta)
+
+    @classmethod
+    def parse_from_json(cls, json_data: Dict[str, Any], directory_name: str, system: IoTSystem,
+                          parent_meta: Optional['FileMetaInfo'] = None) -> 'BatchData':
+        """Parse from JSON"""
+        label = str(json_data.get("label", directory_name))
+        file_type = json_data.get("file_type", "")
+        meta_name = json_data.get("name", label)
+        info = FileMetaInfo(label, file_type, parent=parent_meta)
+        info.name = meta_name
+        info.from_pipe = bool(json_data.get("from_pipe", False))
+        info.load_baseline = bool(json_data.get("load_baseline", False))
+        info.file_load_order = json_data.get("file_order", [])
+        info.default_include = bool(json_data.get("include", True))
+
+        data = cls(info)
+
+        # read batch-specific addresses
+        for add, ent_s in json_data.get("addresses", {}).items():
+            address = Addresses.parse_address(add)
+            ent = Addresses.parse_address(ent_s)
+            entity = system.find_endpoint(ent)
+            if not isinstance(entity, Addressable):
+                raise ValueError(f"Unknown entity {ent_s}")
+            data.address_map[address] = entity
+            info.source.address_map[address] = entity
+
+        # read batch-specific external activity policies
+        for ent_s, policy_n in json_data.get("external_activity", {}).items():
+            ent = Addresses.parse_address(ent_s)
+            node = system.find_endpoint(ent)
+            if not isinstance(node, NetworkNode):
+                raise ValueError(f"Unknown entity '{ent_s}'")
+            policy = ExternalActivity[policy_n]
+            data.activity_map[node] = policy
+            info.source.activity_map[node] = policy
+        return data
+
+
+class LabelFilter:
+    """Filter labels"""
+    def __init__(self, label_specification: str="") -> None:
+        """Initialize the filter"""
+        self.explicit_include = True
+        self.included: Set[str] = set()
+        self.excluded: Set[str] = set()
+        spec = label_specification.strip()
+        if spec == "":
+            self.explicit_include = False
+            return  # all included
+        for index, d in enumerate(spec.split(",")):
+            remove = d.startswith("^")
+            if remove:
+                # remove label
+                if index == 0:
+                    self.explicit_include = False
+                self.excluded.add(d[1:])
+            else:
+                # include label
+                self.included.add(d)
+        intersect = self.included.intersection(self.excluded)
+        if intersect:
+            raise ValueError(f"Labels in both included and excluded: {intersect}")
+
+    def filter(self, label: str) -> bool:
+        """Filter the label"""
+        if self.explicit_include:
+            return label in self.included
+        return label not in self.excluded

toolsaf/adapters/censys_scan.py

@@ -0,0 +1,78 @@
+"""Censys scan result tool"""
+
+import argparse
+from io import BufferedReader
+import json
+import logging
+import pathlib
+
+from censys.search import CensysHosts
+
+from toolsaf.common.address import Protocol, EndpointAddress, AnyAddress
+from toolsaf.core.event_interface import PropertyAddressEvent, EventInterface
+from toolsaf.core.model import IoTSystem, NetworkNode, Host
+from toolsaf.common.property import Properties
+from toolsaf.adapters.tools import EndpointTool
+from toolsaf.common.traffic import EvidenceSource, ServiceScan, Evidence, HostScan
+from toolsaf.common.verdict import Verdict
+
+
+class CensysScan(EndpointTool):
+    """Censys scan tool"""
+    def __init__(self, system: IoTSystem) -> None:
+        super().__init__("censys", ".json", system)
+        self.tool.name = "Censys"
+
+    def filter_node(self, node: NetworkNode) -> bool:
+        return isinstance(node, Host)
+
+    def process_endpoint(self, endpoint: AnyAddress, stream: BufferedReader, interface: EventInterface,
+                         source: EvidenceSource) -> None:
+        raw = json.load(stream)
+
+        evidence = Evidence(source)
+
+        host_services = set()
+        for s in raw.get('services', []):
+            service_name = s.get('service_name', '')
+            protocol = Protocol.get_protocol(service_name.upper())
+            transport = Protocol.protocol(s.get('transport_protocol'), Protocol.ANY)
+            port = int(s['port'])
+
+            self.logger.info("%s %s %d: %s", endpoint, transport, port, service_name)
+            if service_name == "UNKNOWN":
+                service_name = ""
+            if service_name:
+                service_name = f"{service_name} in port {port}"
+            elif transport:
+                service_name = f"{transport.value} {port}"
+            addr = EndpointAddress(endpoint, transport, port)
+            interface.service_scan(ServiceScan(evidence, addr, service_name))
+
+            if protocol == Protocol.HTTP:
+                status_code = s.get('http', {}).get('response', {}).get('status_code')
+                if status_code == 301:
+                    # 301 Permanently Moved
+                    txt = f"{status_code} Permanently Moved"
+                    ev = PropertyAddressEvent(evidence, addr, Properties.HTTP_REDIRECT.verdict(Verdict.PASS, txt))
+                    interface.property_address_update(ev)
+            host_services.add(addr)
+        # other services were not seen
+        interface.host_scan(HostScan(evidence, endpoint, host_services))
+
+
+if __name__ == "__main__":
+    arg_parser = argparse.ArgumentParser()
+    arg_parser.add_argument("--base-dir", default="censys", help="Base dir to create files into")
+    arg_parser.add_argument("addresses", nargs="*", help="Address to resolve using from Censys service")
+    args = arg_parser.parse_args()
+    logging.basicConfig(format='%(message)s', level='INFO')
+    base_dir = pathlib.Path(args.base_dir)
+
+    m = CensysHosts()
+    for a in args.addresses or []:
+        save_file = base_dir / f"{a}.json"
+        print(f"Scan and save {save_file.as_posix()}")
+        info = m.view(a)
+        with save_file.open("w") as f:
+            json.dump(info, f, indent=4, sort_keys=True)

toolsaf/adapters/certmitm_reader.py

@@ -0,0 +1,71 @@
+"""Certmitm reader"""
+
+import json
+from zipfile import ZipFile
+from io import BufferedReader
+from typing import Set, Tuple, Dict, Any, cast
+
+from toolsaf.common.address import HWAddresses, DNSName, Protocol
+from toolsaf.core.event_interface import EventInterface, PropertyEvent
+from toolsaf.adapters.tools import SystemWideTool
+from toolsaf.core.model import IoTSystem, Host, Service
+from toolsaf.common.traffic import EvidenceSource, Evidence, IPFlow
+from toolsaf.common.property import PropertyKey
+from toolsaf.common.verdict import Verdict
+
+
+class CertMITMReader(SystemWideTool):
+    """Read MITM logs created by certmitm"""
+    def __init__(self, system: IoTSystem) -> None:
+        super().__init__("certmitm", system)
+        self.tool.name = "certmitm tool"
+        self.data_file_suffix = ".zip"
+
+    def process_file(self, data: BufferedReader, file_name: str, interface: EventInterface,
+                     source: EvidenceSource) -> bool:
+        """Read log file"""
+        evidence = Evidence(source)
+        connections: Set[Tuple[str, str, str]] = set()
+        dns_names: Set[DNSName] = set()
+
+        # certmitm stores found issues in JSON format to errors.txt
+        with ZipFile(data) as zip_file:
+            for file in zip_file.filelist:
+                if "errors.txt" in file.filename:
+                    with zip_file.open(file.filename) as error_file:
+                        for conn_str in error_file.read().decode("utf-8").rstrip().split("\n"):
+                            conn_json = cast(Dict[str, Any], json.loads(conn_str))
+                            connections.add(
+                                (conn_json['client'], conn_json['destination']['ip'], conn_json['destination']['port'])
+                            )
+
+        for connection in connections:
+            connection_source, target, port = connection
+            flow = IPFlow.tcp_flow(
+                HWAddresses.NULL.data, connection_source, 0,
+                HWAddresses.NULL.data, target, int(port))
+            flow.evidence = evidence
+            PropertyKey("certmitm").put_verdict(flow.properties, Verdict.FAIL)
+            interface.connection(flow)
+
+        # Workaround for showing that certmitm was used
+        with ZipFile(data) as zip_file:
+            for file in zip_file.filelist:
+                if "certificates" in file.filename:
+                    dns_name = DNSName(file.filename.split("/")[-1].split("_")[0])
+                    if dns_name in dns_names:
+                        continue
+                    dns_names.add(dns_name)
+                    if (endpoint := self.system.find_endpoint(dns_name)):
+                        if not isinstance(endpoint, Host):
+                            continue
+                        for endpoint_connection in endpoint.connections:
+                            if not isinstance(endpoint_connection.target, Service):
+                                continue
+                            key = PropertyKey(self.tool_label)
+                            if endpoint_connection.target.protocol == Protocol.TLS \
+                            and key not in endpoint_connection.properties:
+                                ev = PropertyEvent(evidence, endpoint_connection, key.verdict(Verdict.PASS))
+                                interface.property_update(ev)
+
+        return True