projectaria-debug-sharing 0.1.0__py3-none-any.whl

projectaria_debug_sharing/__init__.py

@@ -0,0 +1,22 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pyre-strict
+
+from projectaria_debug_sharing.debug_sharing.types import Attachment, DebugReport
+
+__all__ = [
+    "Attachment",
+    "DebugReport",
+]

projectaria_debug_sharing/debug_sharing/__init__.py

@@ -0,0 +1,15 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pyre-strict

projectaria_debug_sharing/debug_sharing/_internal/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

projectaria_debug_sharing/debug_sharing/_internal/delivery.py

@@ -0,0 +1,252 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Getting a report to Meta: now if possible, later if not. Package-internal:
+nothing outside this package imports it.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+
+from projectaria_debug_sharing.debug_sharing._internal.spool import ReportSpool
+from projectaria_debug_sharing.debug_sharing.submitter import (
+    Delivery,
+    DrainResult,
+    SubmissionError,
+    Submitter,
+)
+from projectaria_debug_sharing.debug_sharing.types import (
+    DebugReport,
+    SpooledReportError,
+)
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class _Attempt:
+    bug_id: str | None = None
+    will_retry: bool = False
+
+
+class ReportDelivery:
+    """
+    Sends reports now, keeps the ones that fail in a `ReportSpool`, and sends
+    those on a later `drain`.
+    """
+
+    def __init__(self, submitter: Submitter, spool: ReportSpool | None = None) -> None:
+        """
+        Args:
+            submitter: Where reports are sent.
+            spool: Where failed reports are kept. Defaults to `ReportSpool()`.
+        """
+        self._submitter = submitter
+        self._spool: ReportSpool = spool or ReportSpool()
+
+    async def deliver(self, report: DebugReport) -> Delivery:
+        """
+        Sends a report, and keeps it for `drain` if that fails and a retry
+        could succeed. Sending a report now means the user asked for it, so it
+        is kept as requested. A refused report is not kept, and any copy of it
+        saved earlier is deleted, or marked refused if it cannot be, so it is
+        never sent again.
+
+        A copy of the report saved earlier, for example while the tool finished
+        something the report mentions, is claimed for the send, so no drain can
+        file it meanwhile. It is deleted once the report is filed, replaced if
+        a retryable send fails, and put back if the send is cancelled. If the
+        replacement cannot be saved, it stays kept in its place as it was
+        saved, requested or not. A drain that claimed it first may still file
+        it too.
+
+        Args:
+            report: The report to send.
+
+        Returns:
+            A `Delivery`: the bug id when the report was filed; otherwise the
+            error, and where the report was kept for a later drain, if it was.
+        """
+        held = self._claim_kept(report.report_id)
+        try:
+            bug_id = await self._submitter.submit(report)
+        except SubmissionError as e:
+            if not e.retryable:
+                self._discard_refused(held, report.report_id, e)
+                return Delivery(error=e)
+            self._put_back(held)
+            return Delivery(error=e, spooled=self._keep(report))
+        except BaseException:
+            self._put_back(held)
+            raise
+        self._discard_kept(held, report.report_id)
+        return Delivery(bug_id=bug_id)
+
+    async def drain(self, *, requested_only: bool = False) -> DrainResult:
+        """
+        Sends every report still waiting, deleting each once it is filed.
+
+        Args:
+            requested_only: Send only reports the user asked to send. For an
+                unattended drain.
+
+        Returns:
+            A `DrainResult`: the bug ids filed, how many reports wait for the
+            next drain, and how many were rejected.
+
+        Raises:
+            OSError: The spool cannot be listed, so whether any report is
+                waiting is unknown.
+        """
+        self._spool.secure()
+        self._spool.prune()
+        sent: list[str] = []
+        waiting = 0
+        rejected = 0
+        for report_dir in self._spool.pending(requested_only=requested_only):
+            try:
+                claimed = self._spool.claim(report_dir)
+            except OSError as e:
+                logger.warning(
+                    "%s: could not claim it: %s", report_dir.name, e.strerror or e
+                )
+                waiting += 1
+                continue
+            if claimed is None:
+                continue
+            attempt = await self._send_claimed(claimed, report_dir)
+            if attempt.bug_id is not None:
+                sent.append(attempt.bug_id)
+            elif attempt.will_retry:
+                waiting += 1
+            else:
+                rejected += 1
+        return DrainResult(sent=tuple(sent), waiting=waiting, rejected=rejected)
+
+    def _keep(self, report: DebugReport) -> Path | None:
+        """
+        The spooled report, the copy saved earlier if it could not be saved, or
+        None, logged, if neither is kept.
+        """
+        try:
+            return self._spool.save(report)
+        except (OSError, ValueError) as e:
+            logger.error("Could not keep report %s: %s", report.report_id, e)
+            return self._keep_earlier_copy(report.report_id)
+
+    def _keep_earlier_copy(self, report_id: str) -> Path | None:
+        """
+        The copy saved earlier, still in place after a fresh save failed, or
+        None. It is kept as it was saved: one collected on the user's behalf
+        stays so, and unattended drains still skip it.
+        """
+        kept = self._spool.find(report_id)
+        if kept is not None:
+            logger.warning("%s: a copy saved earlier is kept instead", report_id)
+        return kept
+
+    def _discard_refused(
+        self, held: tuple[Path, Path] | None, report_id: str, error: SubmissionError
+    ) -> None:
+        """
+        Deletes any copy of a refused report saved earlier, claimed for the
+        send or not, so no drain sends it. One that cannot be deleted is marked
+        refused instead, and put back if it was claimed.
+        """
+        kept = held[0] if held is not None else self._spool.find(report_id)
+        if kept is None or self._spool.discard(kept):
+            return
+        if self._spool.mark_failed(kept, str(error)):
+            self._put_back(held)
+            return
+        logger.warning(
+            "%s: refused, but a kept copy is still on disk; it may be sent again",
+            report_id,
+        )
+
+    def _claim_kept(self, report_id: str) -> tuple[Path, Path] | None:
+        """
+        The kept copy of a report, claimed, and where it is put back; None when
+        no copy is kept, a drain claimed it first, or it could not be claimed.
+        """
+        report_dir = self._spool.find(report_id)
+        if report_dir is None:
+            return None
+        try:
+            claimed = self._spool.claim(report_dir)
+        except OSError as e:
+            logger.warning(
+                "%s: could not claim the kept copy: %s", report_id, e.strerror or e
+            )
+            return None
+        return None if claimed is None else (claimed, report_dir)
+
+    def _put_back(self, held: tuple[Path, Path] | None) -> None:
+        if held is not None:
+            self._spool.release(*held)
+
+    def _discard_kept(self, held: tuple[Path, Path] | None, report_id: str) -> None:
+        if held is not None and not self._spool.discard(held[0]):
+            logger.warning(
+                "%s: filed but a kept copy is still on disk; it may refile", report_id
+            )
+
+    async def _send_claimed(self, claimed: Path, report_dir: Path) -> _Attempt:
+        """Releases the report on any failure, cancellation included."""
+        try:
+            bug_id = await self._submitter.submit(self._spool.load(claimed))
+        except SpooledReportError as e:
+            return self._refuse(claimed, report_dir, e)
+        except SubmissionError as e:
+            if e.retryable:
+                return self._release_for_retry(claimed, report_dir, e)
+            return self._refuse(claimed, report_dir, e)
+        except BaseException:
+            self._spool.release(claimed, report_dir)
+            raise
+        self._discard_filed(claimed, report_dir, bug_id)
+        return _Attempt(bug_id=bug_id)
+
+    def _release_for_retry(
+        self, claimed: Path, report_dir: Path, error: SubmissionError
+    ) -> _Attempt:
+        logger.info("%s: will retry: %s", report_dir.name, error)
+        self._spool.release(claimed, report_dir)
+        return _Attempt(will_retry=True)
+
+    def _refuse(self, claimed: Path, report_dir: Path, error: Exception) -> _Attempt:
+        """
+        Marks the report refused and puts it back. One that cannot be marked is
+        discarded instead: put back unmarked, the next drain would send it again.
+        """
+        logger.warning("%s: not retrying: %s", report_dir.name, error)
+        if self._spool.mark_failed(claimed, str(error)):
+            self._spool.release(claimed, report_dir)
+        else:
+            logger.warning(
+                "%s: could not be marked refused, so it is discarded", report_dir.name
+            )
+            self._spool.discard(claimed)
+        return _Attempt()
+
+    def _discard_filed(self, claimed: Path, report_dir: Path, bug_id: str) -> None:
+        logger.info("%s: filed as bug %s", report_dir.name, bug_id)
+        if not self._spool.discard(claimed):
+            logger.warning(
+                "%s: filed but still on disk; it may refile", report_dir.name
+            )

projectaria_debug_sharing/debug_sharing/_internal/file_access.py

@@ -0,0 +1,130 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Files and directories only their owner can read or change."""
+
+from __future__ import annotations
+
+import errno
+import os
+from pathlib import Path, PurePosixPath
+from typing import Final
+
+PRIVATE_DIR_MODE: Final[int] = 0o700
+PRIVATE_FILE_MODE: Final[int] = 0o600
+
+_WRITABLE_BY_OTHERS: Final[int] = 0o022
+
+
+def leaf_name(raw: str) -> str:
+    """
+    The last component of `raw`, or "" when there is no usable one. Keeps
+    `../../.ssh/id_rsa`, or `..\\..\\etc\\passwd` and `D:evil.log` on
+    Windows, inside the directory it is joined to.
+    """
+    name = PurePosixPath(raw.replace("\\", "/")).name
+    return "" if name in {".", ".."} or ":" in name else name
+
+
+def refuse_symlink(path: Path) -> None:
+    """
+    Raises:
+        OSError: `path` is a symlink, which could point anywhere. Its `errno`
+            is `ELOOP`, as for a symlink `O_NOFOLLOW` refuses.
+    """
+    if path.is_symlink():
+        raise OSError(errno.ELOOP, f"{path} is a symlink, not a directory of yours")
+
+
+def make_private_dir(path: Path) -> None:
+    """
+    Creates `path` and each missing parent owner-only, and tightens `path` if
+    it already existed. Existing parents are left alone: they are not ours.
+
+    Raises:
+        OSError: `path` could not be created or tightened. Among other reasons,
+            it is a symlink, which is never followed.
+    """
+    refuse_symlink(path)
+    missing = [
+        directory for directory in (path, *path.parents) if not directory.exists()
+    ]
+    for directory in reversed(missing):
+        directory.mkdir(mode=PRIVATE_DIR_MODE, exist_ok=True)
+    path.chmod(PRIVATE_DIR_MODE)
+
+
+def write_private(path: Path, data: bytes) -> None:
+    """
+    Writes `data` to `path`, leaving it owner-only even if it existed wider.
+
+    Raises:
+        OSError: Among other reasons, `path` is a symlink, which is never
+            followed.
+    """
+    flags = (
+        os.O_WRONLY
+        | os.O_CREAT
+        | os.O_TRUNC
+        | getattr(os, "O_BINARY", 0)
+        | getattr(os, "O_NOFOLLOW", 0)
+    )
+    descriptor = os.open(path, flags, PRIVATE_FILE_MODE)
+    with os.fdopen(descriptor, "wb") as handle:
+        if hasattr(os, "fchmod"):
+            os.fchmod(descriptor, PRIVATE_FILE_MODE)
+        else:
+            path.chmod(PRIVATE_FILE_MODE)
+        handle.write(data)
+
+
+def is_private_file(path: Path) -> bool:
+    """
+    A regular file, not a link to one, that only this user can change. A file
+    with a second hard link is refused too: that name could be any file of
+    this user's.
+    """
+    return (
+        path.is_file()
+        and not path.is_symlink()
+        and _only_user_can_write(path)
+        and _link_count(path) == 1
+    )
+
+
+def is_private_dir(path: Path) -> bool:
+    """A directory, not a link to one, that only this user can change."""
+    return path.is_dir() and not path.is_symlink() and _only_user_can_write(path)
+
+
+def _only_user_can_write(path: Path) -> bool:
+    """
+    `path` itself belongs to this user and no one else can write to it, so
+    what it holds is what this user put there. Always True on Windows, which
+    protects files with ACLs rather than mode bits.
+    """
+    try:
+        status = path.lstat()
+    except OSError:
+        return False
+    if not hasattr(os, "getuid"):
+        return True
+    return status.st_uid == os.getuid() and not status.st_mode & _WRITABLE_BY_OTHERS
+
+
+def _link_count(path: Path) -> int:
+    try:
+        return path.lstat().st_nlink
+    except OSError:
+        return 0

projectaria_debug_sharing/debug_sharing/_internal/manifest.py

@@ -0,0 +1,133 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""The `report.json` stored beside a spooled report's attachments."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Final, TypeVar
+
+from projectaria_debug_sharing.debug_sharing._internal.file_access import (
+    is_private_file,
+    write_private,
+)
+from projectaria_debug_sharing.debug_sharing.types import (
+    Attachment,
+    DebugReport,
+    SpooledReportError,
+)
+
+MANIFEST_NAME: Final[str] = "report.json"
+
+_T = TypeVar("_T")
+
+
+@dataclass(frozen=True)
+class Manifest:
+    """Everything about a spooled report except its attachments' bytes."""
+
+    report_id: str
+    description: str
+    source_app: str
+    created_time_sec: float
+    """When the report was saved. Ages it: a file's mtime moves when it is touched."""
+
+    requested: bool
+    """Whether the user asked for the report to be sent, rather than it being
+    collected on their behalf."""
+
+    metadata: dict[str, str]
+    attachments: dict[str, str]
+    """Stored file name to attachment name. The name is the multipart field the
+    attachment is sent under, which the file name cannot always reproduce."""
+
+    content_types: dict[str, str]
+    """Stored file name to the content type the attachment was given, which its
+    suffix cannot always reproduce."""
+
+    @classmethod
+    def read(cls, report_dir: Path) -> Manifest:
+        """
+        Raises:
+            SpooledReportError: The manifest is missing, could have been changed
+                by another user, or is not one this library wrote.
+        """
+        path = report_dir / MANIFEST_NAME
+        if not is_private_file(path):
+            raise SpooledReportError(
+                f"{path} is missing, or could have been changed by another user."
+            )
+        try:
+            return cls._from_json(json.loads(path.read_text(encoding="utf-8")))
+        except (OSError, ValueError) as e:
+            raise SpooledReportError(f"{path} is unreadable: {e}") from e
+
+    def write(self, report_dir: Path) -> None:
+        write_private(
+            report_dir / MANIFEST_NAME, json.dumps(asdict(self)).encode("utf-8")
+        )
+
+    def to_report(self, attachments: tuple[Attachment, ...]) -> DebugReport:
+        return DebugReport(
+            description=self.description,
+            source_app=self.source_app,
+            attachments=attachments,
+            metadata=self.metadata,
+            report_id=self.report_id,
+        )
+
+    @classmethod
+    def _from_json(cls, data: object) -> Manifest:
+        """
+        Raises:
+            ValueError: `data` is not a manifest object with every field typed
+                as this class writes it.
+        """
+        if not isinstance(data, dict):
+            raise ValueError("not a JSON object")
+        report_id = cls._field(data, "report_id", str)
+        if not report_id:
+            raise ValueError("'report_id' is empty")
+        attachments = cls._string_map(data, "attachments")
+        content_types = cls._string_map(data, "content_types")
+        if content_types.keys() != attachments.keys():
+            raise ValueError("'content_types' does not match 'attachments'")
+        return cls(
+            report_id=report_id,
+            description=cls._field(data, "description", str),
+            source_app=cls._field(data, "source_app", str),
+            created_time_sec=cls._field(data, "created_time_sec", float),
+            requested=cls._field(data, "requested", bool),
+            metadata=cls._string_map(data, "metadata"),
+            attachments=attachments,
+            content_types=content_types,
+        )
+
+    @staticmethod
+    def _field(data: Mapping[str, object], key: str, kind: type[_T]) -> _T:
+        value = data.get(key)
+        if not isinstance(value, kind):
+            raise ValueError(f"{key!r} is not a {kind.__name__}")
+        return value
+
+    @classmethod
+    def _string_map(cls, data: Mapping[str, object], key: str) -> dict[str, str]:
+        value = cls._field(data, key, dict)
+        if not all(isinstance(k, str) and isinstance(v, str) for k, v in value.items()):
+            raise ValueError(f"{key!r} is not a map of strings")
+        return value