projectaria-debug-sharing 0.1.0__py3-none-any.whl

projectaria_debug_sharing/debug_sharing/cli.py

@@ -0,0 +1,183 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+`aria-debug-report` -- file a debug report, or send the ones already kept.
+
+Deliberately small: this module is only the command line. `DebugReportCommand`
+runs it through `ProjectAriaDebugSharing`, the same object any tool builds. It
+knows nothing about any other tool: a producer that collects its own data keeps
+a report with its own sharer and lets `drain` send it.
+
+    aria-debug-report submit --description "SLAM fails on startup" \\
+        --attach mps_cli.log
+    aria-debug-report submit --from-json report.json
+    aria-debug-report drain
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import logging
+import sys
+from pathlib import Path
+from typing import Final
+
+from projectaria_debug_sharing.debug_sharing._internal.report_command import (
+    CLI_LOGGER_NAME,
+    DebugReportCommand,
+    ReportInput,
+    ReportInputError,
+)
+
+logger: logging.Logger = logging.getLogger(CLI_LOGGER_NAME)
+
+_EXAMPLES: Final[str] = """examples:
+  aria-debug-report submit --description "SLAM fails on startup" --attach mps_cli.log
+  aria-debug-report submit --from-json report.json
+  aria-debug-report drain"""
+
+_PACKAGE_LOGGER: Final[str] = "projectaria_debug_sharing"
+_CONSOLE_HANDLER: Final[str] = "aria-debug-report-console"
+
+
+def parse_key_value(pair: str) -> tuple[str, str]:
+    """An argparse type, so a typo is a usage error rather than a traceback."""
+    key, sep, value = pair.partition("=")
+    if not sep or not key:
+        raise argparse.ArgumentTypeError(f"expected KEY=VALUE, got {pair!r}")
+    return key, value
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="aria-debug-report",
+        description="File a Project Aria debug report, or send the ones already kept.",
+        epilog=_EXAMPLES,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    common = argparse.ArgumentParser(add_help=False)
+    # SUPPRESS: a subcommand's own default would overwrite a --verbose given
+    # before the subcommand.
+    common.add_argument(
+        "--verbose",
+        action="store_true",
+        default=argparse.SUPPRESS,
+        help="Debug logging.",
+    )
+    parser.add_argument("--verbose", action="store_true", help="Debug logging.")
+    commands = parser.add_subparsers(dest="command", required=True)
+
+    submit_parser = commands.add_parser(
+        "submit", parents=[common], help="File one report now; keep it if that fails."
+    )
+    submit_parser.add_argument(
+        "--description",
+        default="",
+        help="What went wrong. Required without --from-json.",
+    )
+    submit_parser.add_argument(
+        "--attach",
+        type=Path,
+        action="append",
+        default=[],
+        metavar="F",
+        help="A file to include, such as a log. Repeatable.",
+    )
+    submit_parser.add_argument(
+        "--metadata",
+        action="append",
+        type=parse_key_value,
+        default=[],
+        metavar="KEY=VALUE",
+        help="An extra detail to include in the report. Repeatable.",
+    )
+    submit_parser.add_argument(
+        "--from-json",
+        type=Path,
+        metavar="F",
+        help="Read description, attachments and metadata from a file.",
+    )
+
+    commands.add_parser(
+        "drain", parents=[common], help="Send every report that is waiting."
+    )
+    return parser
+
+
+def configure_logging(verbose: bool) -> logging.Handler:
+    """
+    Puts the CLI's own output on the console, and keeps it there. The
+    library's records, which can quote server responses, reach the console
+    only with `verbose`. Calling it again reuses the same handler rather than
+    printing every line twice.
+
+    Not `basicConfig`: a host that reconfigures the root logger with
+    `force=True` closes every handler root owns. A handler on our own logger
+    is not root's to close. Records at or above the package level still
+    propagate, so they reach the session log file as well.
+
+    Args:
+        verbose: Show the library's records, and DEBUG, on the console.
+
+    Returns:
+        The console handler.
+    """
+    package = logging.getLogger(_PACKAGE_LOGGER)
+    package.setLevel(logging.DEBUG if verbose else logging.INFO)
+    console = next(
+        (h for h in package.handlers if h.get_name() == _CONSOLE_HANDLER), None
+    )
+    if console is None:
+        console = logging.StreamHandler()
+        console.set_name(_CONSOLE_HANDLER)
+        console.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
+        package.addHandler(console)
+    console.filters.clear()
+    if not verbose:
+        console.addFilter(lambda record: record.name == logger.name)
+    return console
+
+
+def main() -> int:
+    """
+    Returns:
+        The exit code: 0 when everything was filed, 1 when something was not,
+        2 for a usage error, 130 when interrupted.
+    """
+    args = _parser().parse_args()
+    configure_logging(args.verbose)
+    command = DebugReportCommand()
+    try:
+        if args.command == "drain":
+            return asyncio.run(command.drain())
+
+        report_input = ReportInput.from_command_line(
+            description=args.description,
+            attachments=args.attach,
+            metadata=args.metadata,
+            from_json=args.from_json,
+        )
+        return asyncio.run(command.submit(report_input))
+    except ReportInputError as e:
+        logger.error("%s", e)
+        return 2
+    except KeyboardInterrupt:
+        logger.error("Interrupted.")
+        return 130
+
+
+if __name__ == "__main__":
+    sys.exit(main())

projectaria_debug_sharing/debug_sharing/collectors.py

@@ -0,0 +1,472 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""What a report carries: the `Collector` protocol and the collectors every tool needs."""
+
+from __future__ import annotations
+
+import asyncio
+import importlib.metadata
+import io
+import json
+import logging
+import os
+import platform
+import shutil
+import sys
+import zipfile
+from collections.abc import Sequence
+from pathlib import Path
+from typing import BinaryIO, Final, Protocol, runtime_checkable
+
+from projectaria_commons.commons.constants import CONFIG_DIR
+from projectaria_debug_sharing.debug_sharing._internal.plain_files import (
+    describe_entry,
+    list_own_plain_files,
+    open_directory,
+    open_plain_file,
+    open_plain_file_at,
+    zip_date_time,
+)
+from projectaria_debug_sharing.debug_sharing.types import Attachment, CollectionScope
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+DEFAULT_MAX_ATTACHMENT_BYTES: Final[int] = 384 * 1024
+"""Cap on one collected file. A log grows with the session and researchers are
+often on poor links, so the tail is kept: the failure prompting the report is at
+the end."""
+
+DEFAULT_MAX_ARCHIVE_BYTES: Final[int] = 4 * 1024 * 1024
+"""Cap on the files one directory archive reads, before compression."""
+
+MAX_LISTED_ENTRIES: Final[int] = 200
+
+_BYTES_PER_GIB: Final[int] = 1024**3
+
+
+@runtime_checkable
+class Collector(Protocol):
+    """Contributes attachments to a report, within one consent scope."""
+
+    @property
+    def name(self) -> str:
+        """Identifies the collector, and names the attachment it makes."""
+        ...
+
+    @property
+    def scope(self) -> CollectionScope:
+        """The consent category the collector reads under."""
+        ...
+
+    @property
+    def disclosure(self) -> str:
+        """What is collected, as shown to the user before anything is read."""
+        ...
+
+    async def collect(self) -> Sequence[Attachment]:
+        """
+        Returns:
+            The attachments, or none when there is nothing to collect.
+        """
+        ...
+
+
+class FileCollector(Collector):
+    """Attaches one file from disk, for tools whose logs are already there."""
+
+    def __init__(
+        self,
+        name: str,
+        path: Path,
+        *,
+        disclosure: str,
+        scope: CollectionScope = CollectionScope.LOGS,
+        content_type: str = "text/plain",
+        max_bytes: int | None = DEFAULT_MAX_ATTACHMENT_BYTES,
+        optional: bool = False,
+    ) -> None:
+        """
+        Args:
+            name: The collector's and the attachment's name.
+            path: The file. Refused if it is a link or any non-plain file.
+            disclosure: What the user is told is collected.
+            scope: The consent category.
+            content_type: The attachment's type.
+            max_bytes: Cap on the attachment, truncation notice included; the
+                tail is kept. None for no cap.
+            optional: A missing file collects nothing instead of failing. Any
+                other problem, such as a directory at the path, still fails.
+
+        Raises:
+            ValueError: `max_bytes` is not positive.
+        """
+        if max_bytes is not None and max_bytes <= 0:
+            raise ValueError(f"max_bytes must be positive, got {max_bytes}")
+        self._name = name
+        self._path = path
+        self._disclosure = disclosure
+        self._scope = scope
+        self._content_type = content_type
+        self._max_bytes = max_bytes
+        self._optional = optional
+
+    @property
+    def name(self) -> str:
+        """The collector's name, also given to its attachment."""
+        return self._name
+
+    @property
+    def scope(self) -> CollectionScope:
+        """The consent category the file is read under."""
+        return self._scope
+
+    @property
+    def disclosure(self) -> str:
+        """What the user is told is collected."""
+        return self._disclosure
+
+    async def collect(self) -> Sequence[Attachment]:
+        """
+        Reads the file off the event loop.
+
+        Returns:
+            One attachment with the file's bytes, cut to their tail if over the
+            cap, or none if the file is optional and missing.
+
+        Raises:
+            FileNotFoundError: The file is missing and not optional.
+            OSError: The file is not a plain file, or cannot be read.
+        """
+        data = await asyncio.to_thread(self._read)
+        if data is None:
+            return []
+        return [
+            Attachment(
+                name=self._name,
+                data=data,
+                content_type=self._content_type,
+                filename=self._path.name,
+            )
+        ]
+
+    def _read(self) -> bytes | None:
+        try:
+            handle = open_plain_file(self._path)
+        except FileNotFoundError:
+            if self._optional:
+                return None
+            raise
+        with handle:
+            if self._max_bytes is None:
+                return handle.read()
+            return _read_within(handle, self._max_bytes, self._path.name)
+
+
+class DirectoryCollector(Collector):
+    """
+    Attaches this user's files in one directory as a single zip, newest first,
+    so a cap drops the oldest runs. Fails on platforms without race-free
+    directory handles (Windows) rather than read a directory that can be
+    swapped underneath it.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        path: Path,
+        *,
+        disclosure: str,
+        scope: CollectionScope = CollectionScope.LOGS,
+        filename: str | None = None,
+        max_bytes: int = DEFAULT_MAX_ARCHIVE_BYTES,
+        pattern: str = "*",
+    ) -> None:
+        """
+        Args:
+            name: The collector's and the attachment's name.
+            path: The directory. Refused if it is a link.
+            disclosure: What the user is told is collected.
+            scope: The consent category.
+            filename: The archive's name. Defaults to `<name>.zip`.
+            max_bytes: Cap on the bytes read from the directory. Older files
+                are left out once it is reached; the newest is kept, cut to its
+                tail if it alone is larger.
+            pattern: Only file names matching this glob are collected, such
+                as `*.log`, so a directory the user configured elsewhere
+                cannot pull in unrelated files.
+
+        Raises:
+            ValueError: `max_bytes` is not positive.
+        """
+        if max_bytes <= 0:
+            raise ValueError(f"max_bytes must be positive, got {max_bytes}")
+        self._name = name
+        self._path = path
+        self._disclosure = disclosure
+        self._scope = scope
+        self._filename = filename or f"{name}.zip"
+        self._max_bytes = max_bytes
+        self._pattern = pattern
+
+    @property
+    def name(self) -> str:
+        """The collector's name, also given to its attachment."""
+        return self._name
+
+    @property
+    def scope(self) -> CollectionScope:
+        """The consent category the directory is read under."""
+        return self._scope
+
+    @property
+    def disclosure(self) -> str:
+        """What the user is told is collected."""
+        return self._disclosure
+
+    async def collect(self) -> Sequence[Attachment]:
+        """
+        Runs off the event loop: listing and compressing inline would stall
+        every other task, including a cancellation waiting to land.
+
+        Returns:
+            One zip of the matching files, or none if the directory is missing,
+            is not a directory, or holds no matching file of this user's.
+
+        Raises:
+            OSError: The directory is a link, or the platform cannot read it
+                race-free.
+        """
+        data = await asyncio.to_thread(self._archive)
+        if data is None:
+            return []
+        return [
+            Attachment(
+                name=self._name,
+                data=data,
+                content_type="application/zip",
+                filename=self._filename,
+            )
+        ]
+
+    def _archive(self) -> bytes | None:
+        try:
+            dir_fd = open_directory(self._path)
+        except (FileNotFoundError, NotADirectoryError):
+            return None
+        try:
+            names = list_own_plain_files(dir_fd, self._pattern)
+            return self._build_archive(names, dir_fd) if names else None
+        finally:
+            os.close(dir_fd)
+
+    def _build_archive(self, names: Sequence[str], dir_fd: int) -> bytes:
+        """
+        A failed read adds nothing for that file, rather than a truncated
+        member, and does not stop the rest.
+        """
+        buffer = io.BytesIO()
+        archived = 0
+        budget = self._max_bytes
+        with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive:
+            for name in names:
+                try:
+                    opened = open_plain_file_at(name, dir_fd)
+                except OSError as e:
+                    logger.warning("Skipping %s: %s", name, e.strerror or e)
+                    continue
+                if opened is None:
+                    logger.warning("Skipping %s: changed since it was listed", name)
+                    continue
+                handle, info = opened
+                with handle:
+                    if archived and info.st_size > budget:
+                        break
+                    try:
+                        data = _read_within(handle, budget, name)
+                    except OSError as e:
+                        logger.warning("Skipping %s: %s", name, e.strerror or e)
+                        continue
+                entry = zipfile.ZipInfo(name, zip_date_time(info.st_mtime))
+                entry.compress_type = zipfile.ZIP_DEFLATED
+                entry.external_attr = (info.st_mode & 0xFFFF) << 16
+                archive.writestr(entry, data)
+                archived += 1
+                budget -= len(data)
+
+        logger.info(
+            "Archived %d of %d file(s) from %s", archived, len(names), self._path.name
+        )
+        return buffer.getvalue()
+
+
+class EnvironmentCollector(Collector):
+    """
+    Describes the machine every ARK tool runs on: tool and Python versions, OS,
+    CPU, disk, and the names in the Project Aria config folder. Never the
+    contents of a file, a host name or a user name.
+    """
+
+    def __init__(self, *, distribution: str, config_dir: Path = CONFIG_DIR) -> None:
+        """
+        Args:
+            distribution: The installed package whose version is reported.
+            config_dir: The Project Aria config folder to list.
+        """
+        self._distribution = distribution
+        self._config_dir = config_dir
+
+    @property
+    def name(self) -> str:
+        """Always "environment", which also names the JSON attachment."""
+        return "environment"
+
+    @property
+    def scope(self) -> CollectionScope:
+        """Always `CollectionScope.ENVIRONMENT`."""
+        return CollectionScope.ENVIRONMENT
+
+    @property
+    def disclosure(self) -> str:
+        """What the user is told is collected."""
+        return (
+            "the tool and Python versions, and this machine's OS, CPU and "
+            "disk, with a listing of your ~/.projectaria folder -- names, "
+            "kinds and sizes only, never the contents"
+        )
+
+    async def collect(self) -> Sequence[Attachment]:
+        """
+        Describes the machine off the event loop. A section that cannot be
+        read reports its error instead of failing the rest.
+
+        Returns:
+            One `environment.json` attachment.
+        """
+        payload = await asyncio.to_thread(self._describe_machine)
+        return [
+            Attachment(
+                name=self.name,
+                data=json.dumps(payload, indent=2, sort_keys=True).encode("utf-8"),
+                content_type="application/json",
+                filename=f"{self.name}.json",
+            )
+        ]
+
+    def _describe_machine(self) -> dict[str, object]:
+        return {
+            "tool": self._describe_tool(),
+            "os": self._describe_os(),
+            "python": self._describe_python(),
+            "hardware": self._describe_hardware(),
+            "aria_home": self._describe_config_dir(),
+        }
+
+    def _describe_tool(self) -> dict[str, str]:
+        """The version is "unknown" when running from a build tree, not a wheel."""
+        try:
+            version = importlib.metadata.version(self._distribution)
+        except importlib.metadata.PackageNotFoundError:
+            version = "unknown"
+        return {"distribution": self._distribution, "version": version}
+
+    def _describe_config_dir(self) -> dict[str, object]:
+        """
+        Names, kinds and sizes in the config folder, so support can see its
+        layout. An entry that cannot be read is listed with its error instead
+        of costing the rest; errors give `strerror`, never the path.
+        """
+        if not self._config_dir.is_dir():
+            return {"exists": False}
+        try:
+            children = sorted(self._config_dir.iterdir())
+        except OSError as e:
+            return {"exists": True, "error": e.strerror or type(e).__name__}
+        entries: list[dict[str, object]] = []
+        for path in children[:MAX_LISTED_ENTRIES]:
+            try:
+                entries.append(describe_entry(path))
+            except OSError as e:
+                entries.append(
+                    {"name": path.name, "error": e.strerror or type(e).__name__}
+                )
+        return {
+            "exists": True,
+            "entries": entries,
+            "truncated": len(children) > MAX_LISTED_ENTRIES,
+        }
+
+    @staticmethod
+    def _describe_os() -> dict[str, str]:
+        """
+        `release` only: `platform.version()` and `platform.platform()` carry the
+        kernel build string, which on a self-built kernel names the build host.
+        """
+        return {
+            "system": platform.system(),
+            "release": platform.release(),
+            "machine": platform.machine(),
+        }
+
+    @staticmethod
+    def _describe_python() -> dict[str, str]:
+        return {
+            "version": sys.version,
+            "implementation": platform.python_implementation(),
+        }
+
+    @staticmethod
+    def _describe_hardware() -> dict[str, object]:
+        """
+        Disk space in whole GiB, enough to see a full disk without exact figures.
+        An error gives `strerror`, not `str()`, which would name the home directory.
+        """
+        info: dict[str, object] = {"cpu_count": os.cpu_count()}
+        try:
+            usage = shutil.disk_usage(Path.home())
+        except OSError as e:
+            info["disk_error"] = e.strerror or type(e).__name__
+        else:
+            info["disk_total_gib"] = round(usage.total / _BYTES_PER_GIB)
+            info["disk_free_gib"] = round(usage.free / _BYTES_PER_GIB)
+        return info
+
+
+def _read_tail(handle: BinaryIO, max_bytes: int, name: str) -> bytes:
+    """
+    The end of the file behind a notice of what was dropped, within
+    `max_bytes` notice included. The size is clamped to the file's current
+    end, so a stat taken before a rotation shrank it still reads its tail.
+    """
+    size = min(os.fstat(handle.fileno()).st_size, handle.seek(0, os.SEEK_END))
+    kept = max(max_bytes - len(_truncation_notice(size)), 0)
+    handle.seek(max(size - kept, 0))
+    tail = handle.read(kept)
+    logger.info("Truncating %s, keeping the last %d of %d bytes", name, len(tail), size)
+    return (_truncation_notice(max(size - len(tail), 0)) + tail)[:max_bytes]
+
+
+def _read_within(handle: BinaryIO, max_bytes: int, name: str) -> bytes:
+    """
+    At most `max_bytes`, found by reading one byte past the cap rather than
+    trusting an earlier stat that an active log has outgrown.
+    """
+    head = handle.read(max_bytes + 1)
+    if len(head) <= max_bytes:
+        return head
+    return _read_tail(handle, max_bytes, name)
+
+
+def _truncation_notice(omitted: int) -> bytes:
+    return f"[truncated: {omitted} earlier bytes omitted]\n".encode("utf-8")

projectaria_debug_sharing/debug_sharing/command.py

@@ -0,0 +1,36 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pyre-strict
+
+"""
+`aria-debug-report`'s commands, for another tool's command line to run as its
+own, under its own identity and with collectors of its own.
+"""
+
+from __future__ import annotations
+
+from projectaria_debug_sharing.debug_sharing._internal.report_command import (
+    CLI_LOGGER_NAME,
+    DebugReportCommand,
+    ReportInput,
+    ReportInputError,
+)
+
+__all__ = [
+    "CLI_LOGGER_NAME",
+    "DebugReportCommand",
+    "ReportInput",
+    "ReportInputError",
+]