dissect.target 3.8.dev31__tar.gz → 3.8.dev32__tar.gz

{dissect.target-3.8.dev31/dissect.target.egg-info → dissect.target-3.8.dev32}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dissect.target
-Version: 3.8.dev31
+Version: 3.8.dev32
 Summary: This module ties all other Dissect modules together, it provides a programming API and command line tools which allow easy access to various data sources inside disk images or file collections (a.k.a. targets)
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3

{dissect.target-3.8.dev31 → dissect.target-3.8.dev32}/dissect/target/plugins/browsers/browser.py

@@ -20,11 +20,26 @@ GENERIC_HISTORY_RECORD_FIELDS = [
     ("uri", "from_url"),
     ("path", "source"),
 ]
+GENERIC_DOWNLOAD_RECORD_FIELDS = [
+    ("datetime", "ts_start"),
+    ("datetime", "ts_end"),
+    ("string", "browser"),
+    ("varint", "id"),
+    ("path", "path"),
+    ("uri", "url"),
+    ("filesize", "size"),
+    ("varint", "state"),
+    ("path", "source"),
+]
 
 BrowserHistoryRecord = create_extended_descriptor([UserRecordDescriptorExtension])(
     "browser/history", GENERIC_HISTORY_RECORD_FIELDS
 )
 
+BrowserDownloadRecord = create_extended_descriptor([UserRecordDescriptorExtension])(
+    "browser/download", GENERIC_DOWNLOAD_RECORD_FIELDS
+)
+
 
 class BrowserPlugin(Plugin):
     """General browser plugin.
@@ -51,28 +66,42 @@ class BrowserPlugin(Plugin):
         for entry in self.BROWSERS:
             try:
                 self._plugins.append(getattr(self.target, entry))
-            except Exception:  # noqa
+            except Exception:
                 target.log.exception("Failed to load browser plugin: %s", entry)
 
-    def check_compatible(self):
+    def check_compatible(self) -> bool:
+        """Perform a compatibility check with the target.
+        This function checks if any of the supported browser plugins
+        can be used. Otherwise it should raise an ``UnsupportedPluginError``.
+        Raises:
+            UnsupportedPluginError: If the plugin could not be loaded.
+        """
         if not len(self._plugins):
             raise UnsupportedPluginError("No compatible browser plugins found")
 
-    def _func(self, f):
-        for p in self._plugins:
+    def _func(self, func_name: str):
+        """Return the supported browser plugin records.
+
+        Args:
+            func_name: Exported function of the browser plugin to find.
+
+        Yields:
+            Record from the browser function.
+        """
+        for plugin_name in self._plugins:
             try:
-                for entry in getattr(p, f)():
+                for entry in getattr(plugin_name, func_name)():
                     yield entry
             except Exception:
-                self.target.log.exception("Failed to execute browser plugin: %s.%s", p._name, f)
+                self.target.log.exception("Failed to execute browser plugin: %s.%s", plugin_name._name, func_name)
 
     @export(record=BrowserHistoryRecord)
     def history(self):
-        """Return browser history records from all browsers installed.
+        """Return browser history records from all browsers installed and supported.
 
         Historical browser records for Chrome, Chromium, Edge (Chromium), Firefox, and Internet Explorer are returned.
 
-        Yields BrowserHistoryRecords with the following fields:
+        Yields BrowserHistoryRecord with the following fields:
             hostname (string): The target hostname.
             domain (string): The target domain.
             ts (datetime): Visit timestamp.
@@ -91,12 +120,38 @@ class BrowserPlugin(Plugin):
             from_url (uri): URL of the "from" visit.
             source: (path): The source file of the history record.
         """
-        for e in self._func("history"):
-            yield e
+        yield from self._func("history")
 
+    @export(record=BrowserDownloadRecord)
+    def downloads(self):
+        """Return browser download records from all browsers installed and supported.
 
-def try_idna(s):
+        Yields:
+            BrowserDownloadRecord with the following fieds:
+            hostname (string): The target hostname.
+            domain (string): The target domain.
+            ts_start (datetime): Download start timestamp.
+            ts_end (datetime): Download end timestamp.
+            browser (string): The browser from which the records are generated from.
+            id (string): Record ID.
+            path (string): Download path.
+            url (uri): Download URL.
+            size (varint): Download file size.
+            state (varint): Download state number.
+            source: (path): The source file of the download record.
+        """
+        yield from self._func("downloads")
+
+
+def try_idna(url: str) -> bytes:
+    """Attempts to convert a possible Unicode url to ASCII using the IDNA standard.
+
+    Args:
+        url: A String containing the url to be converted.
+
+    Returns: Bytes object with the ASCII version of the url.
+    """
     try:
-        return s.encode("idna")
-    except Exception:  # noqa
-        return s
+        return url.encode("idna")
+    except Exception:
+        return url

{dissect.target-3.8.dev31 → dissect.target-3.8.dev32}/dissect/target/plugins/browsers/chrome.py

@@ -1,7 +1,10 @@
 from dissect.target.helpers.descriptor_extensions import UserRecordDescriptorExtension
 from dissect.target.helpers.record import create_extended_descriptor
 from dissect.target.plugin import Plugin, export
-from dissect.target.plugins.browsers.browser import GENERIC_HISTORY_RECORD_FIELDS
+from dissect.target.plugins.browsers.browser import (
+    GENERIC_DOWNLOAD_RECORD_FIELDS,
+    GENERIC_HISTORY_RECORD_FIELDS,
+)
 from dissect.target.plugins.browsers.chromium import ChromiumMixin
 
 
@@ -20,11 +23,19 @@ class ChromePlugin(ChromiumMixin, Plugin):
         # Macos
         "Library/Application Support/Google/Chrome/Default",
     ]
-    HISTORY_RECORD = create_extended_descriptor([UserRecordDescriptorExtension])(
+    BrowserHistoryRecord = create_extended_descriptor([UserRecordDescriptorExtension])(
         "browser/chrome/history", GENERIC_HISTORY_RECORD_FIELDS
     )
+    BrowserDownloadRecord = create_extended_descriptor([UserRecordDescriptorExtension])(
+        "browser/chrome/download", GENERIC_DOWNLOAD_RECORD_FIELDS
+    )
 
-    @export(record=HISTORY_RECORD)
+    @export(record=BrowserHistoryRecord)
     def history(self):
         """Return browser history records for Google Chrome."""
-        yield from ChromiumMixin.history(self, "chrome")
+        yield from super().history("chrome")
+
+    @export(record=BrowserDownloadRecord)
+    def downloads(self):
+        """Return browser download records for Google Chrome."""
+        yield from super().downloads("chrome")

{dissect.target-3.8.dev31 → dissect.target-3.8.dev32}/dissect/target/plugins/browsers/chromium.py

@@ -1,10 +1,11 @@
+from collections import defaultdict
 from typing import Iterator
 
 from dissect.sql import sqlite3
 from dissect.sql.exceptions import Error as SQLError
 from dissect.sql.sqlite3 import SQLite3
 from dissect.util.ts import webkittimestamp
-from flow.record import Record
+from flow.record.fieldtypes import path
 
 from dissect.target.exceptions import FileNotFoundError, UnsupportedPluginError
 from dissect.target.helpers.descriptor_extensions import UserRecordDescriptorExtension
@@ -12,6 +13,7 @@ from dissect.target.helpers.fsutil import TargetPath
 from dissect.target.helpers.record import create_extended_descriptor
 from dissect.target.plugin import Plugin, export
 from dissect.target.plugins.browsers.browser import (
+    GENERIC_DOWNLOAD_RECORD_FIELDS,
     GENERIC_HISTORY_RECORD_FIELDS,
     try_idna,
 )
@@ -21,11 +23,14 @@ class ChromiumMixin:
     """Mixin class with methods for Chromium-based browsers."""
 
     DIRS = []
-    HISTORY_RECORD = create_extended_descriptor([UserRecordDescriptorExtension])(
+    BrowserHistoryRecord = create_extended_descriptor([UserRecordDescriptorExtension])(
         "browser/chromium/history", GENERIC_HISTORY_RECORD_FIELDS
     )
+    BrowserDownloadRecord = create_extended_descriptor([UserRecordDescriptorExtension])(
+        "browser/chromium/download", GENERIC_DOWNLOAD_RECORD_FIELDS
+    )
 
-    def history(self, browser_name: str = None) -> Iterator[Record]:
+    def history(self, browser_name: str = None) -> Iterator[BrowserHistoryRecord]:
         """Return browser history records from supported Chromium-based browsers.
 
         Args:
@@ -55,7 +60,7 @@ class ChromiumMixin:
         for user, db_file, db in self._iter_db("History"):
             try:
                 urls = {row.id: row for row in db.table("urls").rows()}
-                visits: dict = {}
+                visits = {}
 
                 for row in db.table("visits").rows():
                     visits[row.id] = row
@@ -67,7 +72,7 @@ class ChromiumMixin:
                     else:
                         from_visit, from_url = None, None
 
-                    yield self.HISTORY_RECORD(
+                    yield self.BrowserHistoryRecord(
                         ts=webkittimestamp(row.visit_time),
                         browser=browser_name,
                         id=row.id,
@@ -89,6 +94,66 @@ class ChromiumMixin:
             except SQLError as e:
                 self.target.log.warning("Error processing history file: %s", db_file, exc_info=e)
 
+    def downloads(self, browser_name: str = None) -> Iterator[BrowserDownloadRecord]:
+        """Return browser download records from supported Chromium-based browsers.
+
+        Args:
+            browser_name: The name of the browser as a string.
+        Yields:
+            Records with the following fields:
+                hostname (string): The target hostname.
+                domain (string): The target domain.
+                ts_start (datetime): Download start timestamp.
+                ts_ed (datetime): Download end timestamp.
+                browser (string): The browser from which the records are generated from.
+                id (string): Record ID.
+                path (string): Download path.
+                url (uri): Download URL.
+                size (varint): Download file size.
+                state (varint): Download state number.
+                source: (path): The source file of the download record.
+        Raises:
+            SQLError: If the history file could not be processed.
+        """
+        for user, db_file, db in self._iter_db("History"):
+            try:
+                download_chains = defaultdict(list)
+                for row in db.table("downloads_url_chains"):
+                    download_chains[row.id].append(row)
+
+                for chain in download_chains.values():
+                    chain.sort(key=lambda row: row.chain_index)
+
+                for row in db.table("downloads").rows():
+                    download_path = row.target_path
+                    if download_path and self.target.os == "windows":
+                        download_path = path.from_windows(download_path)
+                    elif download_path:
+                        download_path = path.from_posix()(download_path)
+
+                    url = None
+                    download_chain = download_chains.get(row.id)
+
+                    if download_chain:
+                        url = download_chain[-1].url
+                        url = try_idna(url)
+
+                    yield self.BrowserDownloadRecord(
+                        ts_start=webkittimestamp(row.start_time),
+                        ts_end=webkittimestamp(row.end_time) if row.end_time else None,
+                        browser=browser_name,
+                        id=row.get("id"),
+                        path=download_path,
+                        url=url,
+                        size=row.get("total_bytes"),
+                        state=row.get("state"),
+                        source=str(db_file),
+                        _target=self.target,
+                        _user=user,
+                    )
+            except SQLError as e:
+                self.target.log.warning("Error processing history file: %s", db_file, exc_info=e)
+
     def _iter_db(self, filename: str) -> Iterator[SQLite3]:
         """Generate a connection to a sqlite history database files.
 
@@ -151,7 +216,12 @@ class ChromiumPlugin(ChromiumMixin, Plugin):
         "AppData/Local/Chromium/User Data/Default",
     ]
 
-    @export(record=ChromiumMixin.HISTORY_RECORD)
+    @export(record=ChromiumMixin.BrowserHistoryRecord)
     def history(self):
         """Return browser history records for Chromium browser."""
-        yield from ChromiumMixin.history(self, "chromium")
+        yield from super().history("chromium")
+
+    @export(record=ChromiumMixin.BrowserDownloadRecord)
+    def downloads(self):
+        """Return browser download records for Chromium browser."""
+        yield from super().downloads("chromium")

{dissect.target-3.8.dev31 → dissect.target-3.8.dev32}/dissect/target/plugins/browsers/edge.py

@@ -1,7 +1,10 @@
 from dissect.target.helpers.descriptor_extensions import UserRecordDescriptorExtension
 from dissect.target.helpers.record import create_extended_descriptor
 from dissect.target.plugin import Plugin, export
-from dissect.target.plugins.browsers.browser import GENERIC_HISTORY_RECORD_FIELDS
+from dissect.target.plugins.browsers.browser import (
+    GENERIC_DOWNLOAD_RECORD_FIELDS,
+    GENERIC_HISTORY_RECORD_FIELDS,
+)
 from dissect.target.plugins.browsers.chromium import ChromiumMixin
 
 
@@ -16,11 +19,19 @@ class EdgePlugin(ChromiumMixin, Plugin):
         # Macos
         "Library/Application Support/Microsoft Edge/Default",
     ]
-    HISTORY_RECORD = create_extended_descriptor([UserRecordDescriptorExtension])(
+    BrowserHistoryRecord = create_extended_descriptor([UserRecordDescriptorExtension])(
         "browser/edge/history", GENERIC_HISTORY_RECORD_FIELDS
     )
+    BrowserDownloadRecord = create_extended_descriptor([UserRecordDescriptorExtension])(
+        "browser/edge/download", GENERIC_DOWNLOAD_RECORD_FIELDS
+    )
 
-    @export(record=HISTORY_RECORD)
+    @export(record=BrowserHistoryRecord)
     def history(self):
         """Return browser history records for Microsoft Edge."""
-        yield from ChromiumMixin.history(self, "edge")
+        yield from super().history("edge")
+
+    @export(record=BrowserDownloadRecord)
+    def downloads(self):
+        """Return browser download records for Microsoft Edge."""
+        yield from super().downloads("edge")

dissect.target-3.8.dev32/dissect/target/plugins/browsers/firefox.py

@@ -0,0 +1,219 @@
+import json
+from typing import Iterator
+
+from dissect.sql import sqlite3
+from dissect.sql.exceptions import Error as SQLError
+from dissect.sql.sqlite3 import SQLite3
+from dissect.util.ts import from_unix_ms, from_unix_us
+from flow.record.fieldtypes import path
+
+from dissect.target.exceptions import FileNotFoundError, UnsupportedPluginError
+from dissect.target.helpers.descriptor_extensions import UserRecordDescriptorExtension
+from dissect.target.helpers.record import create_extended_descriptor
+from dissect.target.plugin import Plugin, export
+from dissect.target.plugins.browsers.browser import (
+    GENERIC_DOWNLOAD_RECORD_FIELDS,
+    GENERIC_HISTORY_RECORD_FIELDS,
+    try_idna,
+)
+
+
+class FirefoxPlugin(Plugin):
+    """Firefox browser plugin."""
+
+    __namespace__ = "firefox"
+
+    DIRS = [
+        "AppData/Roaming/Mozilla/Firefox/Profiles",
+        "AppData/local/Mozilla/Firefox/Profiles",
+        ".mozilla/firefox",
+        "snap/firefox/common/.mozilla/firefox",
+    ]
+    BrowserHistoryRecord = create_extended_descriptor([UserRecordDescriptorExtension])(
+        "browser/firefox/history", GENERIC_HISTORY_RECORD_FIELDS
+    )
+    BrowserDownloadRecord = create_extended_descriptor([UserRecordDescriptorExtension])(
+        "browser/firefox/download", GENERIC_DOWNLOAD_RECORD_FIELDS
+    )
+
+    def __init__(self, target):
+        super().__init__(target)
+
+        self.users_dirs = []
+        for user_details in self.target.user_details.all_with_home():
+            for directory in self.DIRS:
+                cur_dir = user_details.home_path.joinpath(directory)
+                if not cur_dir.exists():
+                    continue
+                self.users_dirs.append((user_details.user, cur_dir))
+
+    def check_compatible(self):
+        if not len(self.users_dirs):
+            raise UnsupportedPluginError("No Firefox directories found")
+
+    def _iter_db(self, filename: str) -> Iterator[SQLite3]:
+        """Yield opened history database files of all users.
+
+        Args:
+            filename: The filename of the database.
+        Yields:
+            Opened SQLite3 databases.
+        """
+        for user, cur_dir in self.users_dirs:
+            for profile_dir in cur_dir.iterdir():
+                if profile_dir.is_dir():
+                    db_file = profile_dir.joinpath(filename)
+                    try:
+                        yield user, db_file, sqlite3.SQLite3(db_file.open())
+                    except FileNotFoundError:
+                        self.target.log.warning("Could not find %s file: %s", filename, db_file)
+                    except SQLError as e:
+                        self.target.log.warning("Could not open %s file: %s", filename, db_file, exc_info=e)
+
+    @export(record=BrowserHistoryRecord)
+    def history(self) -> Iterator[BrowserHistoryRecord]:
+        """Return browser history records from Firefox.
+
+        Yields BrowserHistoryRecord with the following fields:
+            hostname (string): The target hostname.
+            domain (string): The target domain.
+            ts (datetime): Visit timestamp.
+            browser (string): The browser from which the records are generated from.
+            id (string): Record ID.
+            url (uri): History URL.
+            title (string): Page title.
+            description (string): Page description.
+            rev_host (string): Reverse hostname.
+            visit_type (varint): Visit type.
+            visit_count (varint): Amount of visits.
+            hidden (string): Hidden value.
+            typed (string): Typed value.
+            session (varint): Session value.
+            from_visit (varint): Record ID of the "from" visit.
+            from_url (uri): URL of the "from" visit.
+            source: (path): The source file of the history record.
+        """
+        for user, db_file, db in self._iter_db("places.sqlite"):
+            try:
+                places = {row.id: row for row in db.table("moz_places").rows()}
+                visits = {}
+
+                for row in db.table("moz_historyvisits").rows():
+                    visits[row.id] = row
+                    place = places[row.place_id]
+
+                    if row.from_visit and row.from_visit in visits:
+                        from_visit = visits[row.from_visit]
+                        from_place = places[from_visit.place_id]
+                    else:
+                        from_visit, from_place = None, None
+
+                    yield self.BrowserHistoryRecord(
+                        ts=from_unix_us(row.visit_date),
+                        browser="firefox",
+                        id=row.id,
+                        url=try_idna(place.url),
+                        title=place.title,
+                        description=place.description,
+                        rev_host=try_idna(place.rev_shot),
+                        visit_type=row.visit_type,
+                        visit_count=place.visit_count,
+                        hidden=place.hidden,
+                        typed=place.typed,
+                        session=row.session,
+                        from_visit=row.from_visit or None,
+                        from_url=try_idna(from_place.url) if from_place else None,
+                        source=str(db_file),
+                        _target=self.target,
+                        _user=user,
+                    )
+            except SQLError as e:
+                self.target.log.warning("Error processing history file: %s", db_file, exc_info=e)
+
+    @export(record=BrowserDownloadRecord)
+    def downloads(self) -> Iterator[BrowserDownloadRecord]:
+        """Return browser download records from Firefox.
+
+        Yields BrowserDownloadRecord with the following fields:
+            hostname (string): The target hostname.
+            domain (string): The target domain.
+            ts_start (datetime): Download start timestamp.
+            ts_end (datetime): Download end timestamp.
+            browser (string): The browser from which the records are generated from.
+            id (string): Record ID.
+            path (string): Download path.
+            url (uri): Download URL.
+            size (varint): Download file size.
+            state (varint): Download state number.
+            source: (path): The source file of the download record.
+        """
+        for user, db_file, db in self._iter_db("places.sqlite"):
+            try:
+                places = {row.id: row for row in db.table("moz_places").rows()}
+                attributes = {row.id: row.name for row in db.table("moz_anno_attributes").rows()}
+                annotations = {}
+
+                for row in db.table("moz_annos"):
+                    attribute_name = attributes.get(row.anno_attribute_id, row.anno_attribute_id)
+
+                    if attribute_name == "downloads/metaData":
+                        content = json.loads(row.content)
+                    else:
+                        content = row.content
+
+                    if row.place_id not in annotations:
+                        annotations[row.place_id] = {"id": row.id}
+
+                    annotations[row.place_id][attribute_name] = {
+                        "content": content,
+                        "flags": row.flags,
+                        "expiration": row.expiration,
+                        "type": row.type,
+                        "date_added": from_unix_us(row.dateAdded),
+                        "last_modified": from_unix_us(row.lastModified),
+                    }
+
+                for place_id, annotation in annotations.items():
+                    if "downloads/metaData" not in annotation:
+                        continue
+
+                    metadata = annotation.get("downloads/metaData", {})
+
+                    ts_end = None
+                    size = None
+                    state = None
+
+                    content = metadata.get("content")
+                    if content:
+                        ts_end = metadata.get("content").get("endTime")
+                        ts_end = from_unix_ms(ts_end) if ts_end else None
+
+                        size = content.get("fileSize")
+                        state = content.get("state")
+
+                    dest_file_info = annotation.get("downloads/destinationFileURI", {})
+                    download_path = dest_file_info.get("content")
+
+                    if download_path and self.target.os == "windows":
+                        download_path = path.from_windows(download_path)
+                    elif download_path:
+                        download_path = path.from_posix(download_path)
+
+                    place = places.get(place_id)
+                    url = place.get("url")
+                    url = try_idna(url) if url else None
+
+                    yield self.BrowserDownloadRecord(
+                        ts_start=dest_file_info.get("date_added"),
+                        ts_end=ts_end,
+                        browser="firefox",
+                        id=annotation.get("id"),
+                        path=download_path,
+                        url=url,
+                        size=size,
+                        state=state,
+                        source=str(db_file),
+                        _target=self.target,
+                    )
+            except SQLError as e:
+                self.target.log.warning("Error processing history file: %s", db_file, exc_info=e)