linear-mcp-fast 0.1.0__py3-none-any.whl

ccl_chromium_reader/ccl_shared_proto_db_downloads.py

@@ -0,0 +1,189 @@
+"""
+Copyright 2022, CCL Forensics
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+
+__version__ = "0.3"
+__description__ = "A module for reading downloads from the Chrome/Chromium shared_proto_db leveldb data store"
+__contact__ = "Alex Caithness"
+
+import datetime
+import io
+import os
+import pathlib
+import sys
+import typing
+
+from .storage_formats import ccl_leveldb
+from .serialization_formats import ccl_protobuff as pb
+from .download_common import Download
+
+CHROME_EPOCH = datetime.datetime(1601, 1, 1, 0, 0, 0)
+
+
+def chrome_milli_time(milliseconds: typing.Optional[int], allow_none=True) -> typing.Optional[datetime.datetime]:
+    if milliseconds is not None:
+        if milliseconds == 0xffffffffffffffff:
+            return CHROME_EPOCH
+        else:
+            return CHROME_EPOCH + datetime.timedelta(milliseconds=milliseconds)
+    elif allow_none:
+        return None
+    raise ValueError("milliseconds cannot be None")
+
+
+def read_datetime(stream) -> typing.Optional[datetime.datetime]:
+    return chrome_milli_time(pb.read_le_varint(stream))
+
+
+# https://source.chromium.org/chromium/chromium/src/+/main:components/download/database/proto/download_entry.proto;l=86
+
+HttpRequestHeader_Structure = {
+    1: pb.ProtoDecoder("key", pb.read_string),
+    2: pb.ProtoDecoder("value", pb.read_string)
+}
+
+ReceivedSlice_Structure = {
+    1: pb.ProtoDecoder("offset", pb.read_le_varint),
+    2: pb.ProtoDecoder("received_bytes", pb.read_le_varint),
+    3: pb.ProtoDecoder("finished", lambda x: pb.read_le_varint(x) != 0)
+}
+
+InProgressInfo_Structure = {
+    1: pb.ProtoDecoder("url_chain", pb.read_string),  # string
+    2: pb.ProtoDecoder("referrer_url", pb.read_string),  # string
+    3: pb.ProtoDecoder("site_url", pb.read_string),  # string  // deprecated
+    4: pb.ProtoDecoder("tab_url", pb.read_string),  # string
+    5: pb.ProtoDecoder("tab_referrer_url", pb.read_string),  # string
+    6: pb.ProtoDecoder("fetch_error_body", lambda x: pb.read_le_varint(x) != 0),  # bool
+    7: pb.ProtoDecoder("request_headers", lambda x: pb.read_embedded_protobuf(x, HttpRequestHeader_Structure, True)),  # HttpRequestHeader
+    8: pb.ProtoDecoder("etag", pb.read_string),  # string
+    9: pb.ProtoDecoder("last_modified", pb.read_string),  # string
+    10: pb.ProtoDecoder("total_bytes", pb.read_le_varint),  # int64:
+    11: pb.ProtoDecoder("mime_type", pb.read_string),  # string
+    12: pb.ProtoDecoder("original_mime_type", pb.read_string),  # string
+    13: pb.ProtoDecoder("current_path", pb.read_blob),  # bytes  // Serialized pickles to support string16: TODO
+    14: pb.ProtoDecoder("target_path", pb.read_blob),  # bytes   // Serialized pickles to support string16: TODO
+    15: pb.ProtoDecoder("received_bytes", pb.read_le_varint),  # int64:
+    16: pb.ProtoDecoder("start_time", read_datetime),  # int64:
+    17: pb.ProtoDecoder("end_time", read_datetime),  # int64:
+    18: pb.ProtoDecoder("received_slices", lambda x: pb.read_embedded_protobuf(x, ReceivedSlice_Structure, True)),  # ReceivedSlice
+    19: pb.ProtoDecoder("hash", pb.read_blob),  # string
+    20: pb.ProtoDecoder("transient", lambda x: pb.read_le_varint(x) != 0),  # bool
+    21: pb.ProtoDecoder("state", pb.read_le_varint32),  # int32:
+    22: pb.ProtoDecoder("danger_type", pb.read_le_varint32),  # int32:
+    23: pb.ProtoDecoder("interrupt_reason", pb.read_le_varint32),  # int32:
+    24: pb.ProtoDecoder("paused", lambda x: pb.read_le_varint(x) != 0),  # bool
+    25: pb.ProtoDecoder("metered", lambda x: pb.read_le_varint(x) != 0),  # bool
+    26: pb.ProtoDecoder("bytes_wasted", pb.read_le_varint),  # int64:
+    27: pb.ProtoDecoder("auto_resume_count", pb.read_le_varint32),  # int32:
+    # 28: pb.ProtoDecoder("download_schedule", None)  # DownloadSchedule  // // Deprecated.
+    # 29: pb.ProtoDecoder("reroute_info", pb),  # enterprise_connectors.DownloadItemRerouteInfo TODO
+    30: pb.ProtoDecoder("credentials_mode", pb.read_le_varint32),  # int32:  // network::mojom::CredentialsMode
+    31: pb.ProtoDecoder("range_request_from", pb.read_le_varint),  # int64:
+    32: pb.ProtoDecoder("range_request_to", pb.read_le_varint),  # int64:
+    33: pb.ProtoDecoder("serialized_embedder_download_data", pb.read_string)  # string
+}
+
+DownloadInfo_structure = {
+    1: pb.ProtoDecoder("guid", pb.read_string),
+    2: pb.ProtoDecoder("id", pb.read_le_varint32),
+    # 3 UkmInfo
+    4: pb.ProtoDecoder("in_progress_info", lambda x: pb.read_embedded_protobuf(x, InProgressInfo_Structure, True))
+}
+
+DownloadDbEntry_structure = {
+    1: pb.ProtoDecoder("download_info", lambda x: pb.read_embedded_protobuf(x, DownloadInfo_structure, True))
+}
+
+
+def read_downloads(
+        shared_proto_db_folder: typing.Union[str, os.PathLike],
+        *, handle_errors=False, utf16_paths=True) -> typing.Iterator[Download]:
+    ldb_path = pathlib.Path(shared_proto_db_folder)
+    with ccl_leveldb.RawLevelDb(ldb_path) as ldb:
+        for rec in ldb.iterate_records_raw():
+            if rec.state != ccl_leveldb.KeyState.Live:
+                continue
+
+            key = rec.user_key
+            record_type, specific_key = key.split(b"_", 1)
+            if record_type == b"21":
+                with io.BytesIO(rec.value) as f:
+                    obj = pb.ProtoObject(
+                        0xa, "root", pb.read_protobuff(f, DownloadDbEntry_structure, use_friendly_tag=True))
+                try:
+                    download = Download.from_pb(rec.seq, obj, target_path_is_utf_16=utf16_paths)
+                except ValueError as ex:
+                    print(f"Error reading a download: {ex}", file=sys.stderr)
+                    if handle_errors:
+                        continue
+                    else:
+                        raise
+
+                yield download
+
+
+def report_downloads(
+        shared_proto_db_folder: typing.Union[str, os.PathLike],
+        out_csv_path: typing.Union[str, os.PathLike], utf16_paths=True):
+
+    with pathlib.Path(out_csv_path).open("tx", encoding="utf-8", newline="") as out:
+        writer = csv.writer(out, csv.excel, quoting=csv.QUOTE_ALL, quotechar="\"", escapechar="\\")
+        writer.writerow([
+            "seq no",
+            "guid",
+            "start time",
+            "end time",
+            "tab url",
+            "tab referrer url",
+            "download url chain",
+            "target path",
+            "hash",
+            "total bytes",
+            "mime type",
+            "original mime type"
+        ])
+        for download in read_downloads(shared_proto_db_folder, handle_errors=True, utf16_paths=utf16_paths):
+            writer.writerow([
+                str(download.level_db_seq_no),
+                str(download.guid),
+                download.start_time,
+                download.end_time,
+                download.tab_url,
+                download.tab_referrer_url,
+                " -> ".join(download.url_chain),
+                download.target_path,
+                download.hash,
+                download.total_bytes,
+                download.mime_type,
+                download.original_mime_type
+            ])
+
+
+if __name__ == '__main__':
+    import csv
+    if len(sys.argv) < 3:
+        print(f"USAGE: {pathlib.Path(sys.argv[0]).name} <shared_proto_db folder> <out.csv> [-u8]")
+        print()
+        print("-u8\tutf-8 target paths (use this if target paths appear garbled in the output)")
+        print()
+        exit(1)
+    report_downloads(sys.argv[1], sys.argv[2], "-u8" not in sys.argv[3:])

ccl_chromium_reader/common.py

@@ -0,0 +1,19 @@
+import re
+import typing
+import collections.abc as col_abc
+
+
+KeySearch = typing.Union[str, re.Pattern, col_abc.Collection[str], col_abc.Callable[[str], bool]]
+
+
+def is_keysearch_hit(search: KeySearch, value: str):
+    if isinstance(search, str):
+        return value == search
+    elif isinstance(search, re.Pattern):
+        return search.search(value) is not None
+    elif isinstance(search, col_abc.Collection):
+        return value in set(search)
+    elif isinstance(search, col_abc.Callable):
+        return search(value)
+    else:
+        raise TypeError(f"Unexpected type: {type(search)} (expects: {KeySearch})")

ccl_chromium_reader/download_common.py

@@ -0,0 +1,78 @@
+import dataclasses
+import datetime
+import struct
+import enum
+
+from .serialization_formats import ccl_protobuff as pb
+
+
+class DownloadSource(enum.Enum):
+    shared_proto_db = 1
+    history_db = 2
+
+
+@dataclasses.dataclass(frozen=True)
+class Download:  # TODO: all of the parameters
+    record_source: DownloadSource
+    record_id: int
+    guid: str
+    hash: str
+    url_chain: tuple[str, ...]
+    tab_url: str
+    tab_referrer_url: str
+    target_path: str
+    mime_type: str
+    original_mime_type: str
+    total_bytes: str
+    start_time: datetime.datetime
+    end_time: datetime.datetime
+
+    @property
+    def level_db_seq_no(self):
+        if self.record_source == DownloadSource.shared_proto_db:
+            return self.record_id
+
+    @property
+    def record_location(self) -> str:
+        if self.record_source == DownloadSource.shared_proto_db:
+            return f"Leveldb Seq: {self.record_id}"
+        elif self.record_source == DownloadSource.history_db:
+            return f"SQLite Rowid: {self.record_id}"
+        raise NotImplementedError()
+
+    @property
+    def url(self) -> str:
+        return self.url_chain[-1]
+
+    @property
+    def file_size(self) -> int:
+        return int(self.total_bytes)
+
+    @classmethod
+    def from_pb(cls, seq: int, proto: pb.ProtoObject, *, target_path_is_utf_16=True):
+        if not proto.only("download_info").value:
+            raise ValueError("download_info is empty")
+        target_path_raw = proto.only("download_info").only("in_progress_info").only("target_path").value
+        path_proto_length, path_char_count = struct.unpack("<II", target_path_raw[0:8])
+        if path_proto_length != len(target_path_raw) - 4:
+            raise ValueError("Invalid pickle for target path")
+        if target_path_is_utf_16:
+            target_path = target_path_raw[8: 8 + (path_char_count * 2)].decode("utf-16-le")
+        else:
+            target_path = target_path_raw[8: 8 + path_char_count].decode("utf-8")
+
+        return cls(
+            DownloadSource.shared_proto_db,
+            seq,
+            proto.only("download_info").only("guid").value,
+            proto.only("download_info").only("in_progress_info").only("hash").value.hex(),
+            tuple(x.value for x in proto.only("download_info").only("in_progress_info")["url_chain"]),
+            proto.only("download_info").only("in_progress_info").only("tab_url").value,
+            proto.only("download_info").only("in_progress_info").only("tab_url_referrer").value,
+            target_path,
+            proto.only("download_info").only("in_progress_info").only("mime_type").value,
+            proto.only("download_info").only("in_progress_info").only("original_mime_type").value,
+            proto.only("download_info").only("in_progress_info").only("total_bytes").value,
+            proto.only("download_info").only("in_progress_info").only("start_time").value,
+            proto.only("download_info").only("in_progress_info").only("end_time").value,
+        )

ccl_chromium_reader/profile_folder_protocols.py

@@ -0,0 +1,276 @@
+import datetime
+import pathlib
+import typing
+import collections.abc as col_abc
+
+from .common import KeySearch, is_keysearch_hit
+
+
+class HasRecordLocationProtocol(typing.Protocol):
+    @property
+    def record_location(self) -> str:
+        raise NotImplementedError()
+
+
+@typing.runtime_checkable
+class LocalStorageRecordProtocol(HasRecordLocationProtocol, typing.Protocol):
+    @property
+    def storage_key(self) -> str:
+        raise NotImplementedError()
+
+    @property
+    def script_key(self) -> str:
+        raise NotImplementedError()
+
+    @property
+    def value(self) -> str:
+        raise NotImplementedError()
+
+
+@typing.runtime_checkable
+class SessionStorageRecordProtocol(HasRecordLocationProtocol, typing.Protocol):
+    host: typing.Optional[str]
+    key: str
+    value: str
+
+
+@typing.runtime_checkable
+class HistoryRecordProtocol(HasRecordLocationProtocol, typing.Protocol):
+    url: str
+    title: str
+    visit_time: datetime.datetime
+    # TODO: Assess whether the parent/child visits can be part of the protocol
+
+
+@typing.runtime_checkable
+class IdbKeyProtocol(typing.Protocol):
+    raw_key: bytes
+    value: typing.Any
+
+
+@typing.runtime_checkable
+class IndexedDbRecordProtocol(HasRecordLocationProtocol, typing.Protocol):
+    key: IdbKeyProtocol
+    value: typing.Any
+
+
+class CacheMetadataProtocol(typing.Protocol):
+    request_time: datetime.datetime
+    http_header_attributes: typing.Iterable[tuple[str, str]]
+
+    def get_attribute(self, attribute: str) -> list[str]:
+        raise NotImplementedError()
+
+
+class CacheKeyProtocol(typing.Protocol):
+    raw_key: str
+    url: str
+
+
+class CacheRecordProtocol(typing.Protocol):
+    key: CacheKeyProtocol
+    metadata: CacheMetadataProtocol
+    data: bytes
+    metadata_location: typing.Any
+    data_location: typing.Any
+    was_decompressed: bool
+
+
+class DownloadRecordProtocol(HasRecordLocationProtocol, typing.Protocol):
+    url: str
+    start_time: typing.Optional[datetime.datetime]
+    end_time: typing.Optional[datetime.datetime]
+    target_path: typing.Optional[str]
+    file_size: int
+
+
+@typing.runtime_checkable
+class BrowserProfileProtocol(typing.Protocol):
+    def close(self):
+        raise NotImplementedError()
+
+    def iter_local_storage_hosts(self) -> col_abc.Iterable[str]:
+        """
+        Iterates the hosts in this profile's local storage
+        """
+        raise NotImplementedError()
+
+    def iter_local_storage(
+            self, storage_key: typing.Optional[KeySearch] = None, script_key: typing.Optional[KeySearch] = None, *,
+            include_deletions=False, raise_on_no_result=False) -> col_abc.Iterable[LocalStorageRecordProtocol]:
+        """
+        Iterates this profile's local storage records
+
+        :param storage_key: storage key (host) for the records. This can be one of: a single string;
+        a collection of strings; a regex pattern; a function that takes a string and returns a bool.
+        :param script_key: script defined key for the records. This can be one of: a single string;
+        a collection of strings; a regex pattern; a function that takes a string and returns a bool.
+        :param include_deletions: if True, records related to deletions will be included
+        :param raise_on_no_result: if True (the default) if no matching storage keys are found, raise a KeyError
+        (these will have None as values).
+        :return:
+        """
+        raise NotImplementedError()
+
+    def iter_session_storage_hosts(self) -> col_abc.Iterable[str]:
+        """
+        Iterates this profile's session storage hosts
+        """
+        raise NotImplementedError()
+
+    def iter_session_storage(
+            self, host: typing.Optional[KeySearch] = None, key: typing.Optional[KeySearch] = None, *,
+            include_deletions=False, raise_on_no_result=False) -> col_abc.Iterable[SessionStorageRecordProtocol]:
+        """
+        Iterates this profile's session storage records
+
+        :param host: storage key (host) for the records. This can be one of: a single string;
+        a collection of strings; a regex pattern; a function that takes a string (each host) and
+        returns a bool; or None (the default) in which case all hosts are considered.
+        :param key: script defined key for the records. This can be one of: a single string;
+        a collection of strings; a regex pattern; a function that takes a string and returns a bool; or
+        None (the default) in which case all keys are considered.
+        :param include_deletions: if True, records related to deletions will be included (these will have None as
+        values).
+        :param raise_on_no_result: if True, if no matching storage keys are found, raise a KeyError
+
+        :return: iterable of SessionStoreValue
+        """
+        raise NotImplementedError()
+
+    def iter_indexeddb_hosts(self) -> col_abc.Iterable[str]:
+        """
+        Iterates the hosts present in the Indexed DB folder. These values are what should be used to load the databases
+        directly.
+        """
+        raise NotImplementedError()
+
+    def get_indexeddb(self, host: str):
+        """
+        Returns the database with the host provided. Should be one of the values returned by
+        :func:`~iter_indexeddb_hosts`. The database will be opened on-demand if it hasn't previously been opened.
+
+        :param host: the host to get
+        """
+        # TODO typehint return type once it's also abstracted
+        raise NotImplementedError()
+
+    def iter_indexeddb_records(
+            self, host_id: typing.Optional[KeySearch], database_name: typing.Optional[KeySearch] = None,
+            object_store_name: typing.Optional[KeySearch] = None, *,
+            raise_on_no_result=False, include_deletions=False,
+            bad_deserializer_data_handler=None) -> col_abc.Iterable[IndexedDbRecordProtocol]:
+        """
+        Iterates indexeddb records in this profile.
+
+        :param host_id: the host for the records, relates to the host-named folder in the IndexedDB folder. The
+        possible values for this profile are returned by :func:`~iter_indexeddb_hosts`. This can be one of:
+        a single string; a collection of strings; a regex pattern; a function that takes a string (each host) and
+        returns a bool; or None in which case all hosts are considered. Be cautious with supplying a parameter
+        which will lead to unnecessary databases being opened as this has a set-up time for the first time it
+        is opened.
+        :param database_name: the database name for the records. This can be one of: a single string; a collection
+        of strings; a regex pattern; a function that takes a string (each host) and returns a bool; or None (the
+        default) in which case all hosts are considered.
+        :param object_store_name: the object store name of the records. This can be one of: a single string;
+        a collection of strings; a regex pattern; a function that takes a string (each host) and returns a bool;
+        or None (the default) in which case all hosts are considered.
+        :param raise_on_no_result: if True, if no matching storage keys are found, raise a KeyError
+        :param include_deletions: if True, records related to deletions will be included (these will have None as
+        values).
+        :param bad_deserializer_data_handler: a callback function which will be executed by the underlying
+        indexeddb reader if invalid data is encountered during reading a record, rather than raising an exception.
+        The function should take two arguments: an IdbKey object (which is the key of the bad record) and a bytes
+        object (which is the raw data). The return value of the callback is ignored by the calling code. If this is
+        None (the default) then any bad data will cause an exception to be raised.
+        """
+        raise NotImplementedError()
+
+    def iterate_history_records(
+            self, url: typing.Optional[KeySearch]=None, *,
+            earliest: typing.Optional[datetime.datetime]=None,
+            latest: typing.Optional[datetime.datetime]=None) -> col_abc.Iterable[HistoryRecordProtocol]:
+        """
+        Iterates history records for this profile.
+
+        :param url: a URL to search for. This can be one of: a single string; a collection of strings;
+        a regex pattern; a function that takes a string (each host) and returns a bool; or None (the
+        default) in which case all hosts are considered.
+        :param earliest: an optional datetime which will be used to exclude records before this date.
+        NB the date should be UTC to match the database. If None, no lower limit will be placed on
+        timestamps.
+        :param latest: an optional datetime which will be used to exclude records after this date.
+        NB the date should be UTC to match the database. If None, no upper limit will be placed on
+        timestamps.
+        """
+        # TODO typehint return type once it's also abstracted
+        raise NotImplementedError()
+
+    def iterate_cache(
+            self,
+            url: typing.Optional[KeySearch]=None, *, decompress=True, omit_cached_data=False,
+            **kwargs: typing.Union[bool, KeySearch]) -> col_abc.Iterable[CacheRecordProtocol]:
+        """
+        Iterates cache records for this profile.
+
+        :param url: a URL to search for. This can be one of: a single string; a collection of strings;
+        a regex pattern; a function that takes a string (each host) and returns a bool; or None (the
+        default) in which case all records are considered.
+        :param decompress: if True (the default), data from the cache which is compressed (as per the
+        content-encoding header field) will be decompressed when read if the compression format is
+        supported (currently deflate, gzip and brotli are supported).
+        :param omit_cached_data: does not collect the cached data and omits it from each `CacheResult`
+        object. Should be faster in cases when only metadata recovery is required.
+        :param kwargs: further keyword arguments are used to search based upon header fields. The
+        keyword should be the header field name, with underscores replacing hyphens (e.g.,
+        content-encoding, becomes content_encoding). The value should be one of: a Boolean (in which
+        case only records with this field present will be included if True, and vice versa); a single
+        string; a collection of strings; a regex pattern; a function that takes a string (the value)
+        and returns a bool.
+        """
+        raise NotImplementedError()
+
+    def iter_downloads(
+            self, *, download_url: typing.Optional[KeySearch]=None,
+            tab_url: typing.Optional[KeySearch]=None) -> col_abc.Iterable[DownloadRecordProtocol]:
+        """
+        Iterates download records for this profile
+
+        :param download_url: A URL related to the downloaded resource. This can be one of: a single string;
+        a collection of strings; a regex pattern; a function that takes a string (each host) and returns a bool;
+        or None (the default) in which case all records are considered.
+        :param tab_url: A URL related to the page the user was accessing when this download was started.
+        This can be one of: a single string; a collection of strings; a regex pattern; a function that takes
+        a string (each host) and returns a bool; or None (the default) in which case all records are considered.
+        """
+        raise NotImplementedError()
+
+    @property
+    def path(self) -> pathlib.Path:
+        """The input path of this browser profile"""
+        raise NotImplementedError()
+
+    @property
+    def local_storage(self):
+        """The local storage object for this browser profile"""
+        raise NotImplementedError()
+
+    @property
+    def session_storage(self):
+        """The session storage object for this browser profile"""
+        raise NotImplementedError()
+
+    @property
+    def cache(self):
+        """The cache for this browser profile"""
+        raise NotImplementedError()
+
+    @property
+    def history(self):
+        """The history for this browser profile"""
+        raise NotImplementedError()
+
+    @property
+    def browser_type(self) -> str:
+        """The name of the browser type for this profile"""
+        raise NotImplementedError()