Office365-REST-Python-Client 3.1.0__py3-none-any.whl → 3.2.0__py3-none-any.whl

office365/migration/report_io.py

@@ -0,0 +1,79 @@
+"""Report record IO — write record datasets in a chosen format.
+
+Per-format, not CSV+JSON-coupled: :func:`write_dataset` picks a writer by name so
+callers write whichever formats they need (``csv``/``json``/``ndjson``) by
+looping, and adding a format is one registration.
+"""
+
+from __future__ import annotations
+
+from os import PathLike
+from pathlib import Path
+from typing import Callable, List, Optional, Sequence
+
+RecordWriter = Callable[[List[dict], Path, Optional[Sequence[str]]], None]
+
+_EXTENSIONS = {"csv": "csv", "json": "json", "ndjson": "ndjson"}
+
+
+def _write_csv(records: list[dict], path: Path, columns: Optional[Sequence[str]] = None) -> None:
+    from office365.runtime.converters.csv_writer import write_records
+
+    with open(path, "w", newline="", encoding="utf-8") as f:
+        write_records(records, f, list(columns) if columns else None)
+
+
+def _write_json(records: list[dict], path: Path, columns: Optional[Sequence[str]] = None) -> None:
+    from office365.runtime.converters.json_file import write_json
+
+    with open(path, "w", encoding="utf-8") as f:
+        write_json(records, f, indent=2)
+
+
+def _write_ndjson(records: list[dict], path: Path, columns: Optional[Sequence[str]] = None) -> None:
+    from office365.runtime.converters.ndjson import write_ndjson
+
+    with open(path, "w", encoding="utf-8") as f:
+        write_ndjson(records, f)
+
+
+_WRITERS: dict[str, RecordWriter] = {"csv": _write_csv, "json": _write_json, "ndjson": _write_ndjson}
+
+
+def write_dataset(
+    records: list[dict],
+    dir_: str | PathLike,
+    stem: str,
+    *,
+    fmt: str = "csv",
+    columns: Optional[Sequence[str]] = None,
+) -> str:
+    """Write ``records`` under ``dir_`` as ``stem.<ext>`` in ``fmt``; returns the path.
+
+    Args:
+        records: The record rows.
+        dir_: Destination directory (created when missing).
+        stem: File stem (without extension).
+        fmt: One of ``csv``/``json``/``ndjson``.
+        columns: Optional column order for CSV.
+    """
+    try:
+        writer = _WRITERS[fmt]
+    except KeyError:
+        raise KeyError(f"Unknown report format {fmt!r}; known: {sorted(_WRITERS)}") from None
+    path = Path(dir_) / f"{stem}.{_EXTENSIONS[fmt]}"
+    path.parent.mkdir(parents=True, exist_ok=True)
+    writer(records, path, columns)
+    return str(path)
+
+
+def write_formats(
+    records: list[dict],
+    dir_: str | PathLike,
+    stem: str,
+    *,
+    formats: Sequence[str] = ("csv", "json"),
+    columns: Optional[Sequence[str]] = None,
+) -> list[str]:
+    """Write the same records in each of ``formats``; returns the written paths."""
+    return [write_dataset(records, dir_, stem, fmt=fmt, columns=columns) for fmt in formats]

office365/migration/runner.py

@@ -17,7 +17,6 @@ from __future__ import annotations
 from pathlib import Path
 from typing import TYPE_CHECKING, Callable, Iterable
 
-from office365.migration._util import emit_progress
 from office365.migration.adapters import DataTarget, resolve_dest
 from office365.migration.base import (
     ConflictResolution,
@@ -28,11 +27,80 @@ from office365.migration.base import (
     MigrationStats,
 )
 from office365.migration.checkpoint import Checkpoint
+from office365.runtime.operations import emit_progress
 
 if TYPE_CHECKING:
     from office365.runtime.operations import Progress
 
 
+def _assert_fidelity_supported(options: MigrationOptions, source, target: DataTarget) -> None:
+    """Reject fidelity flags the adapter pair cannot honor.
+
+    ``preserve_versions`` always needs the server-side Migration API (REST can't
+    restore version history). ``preserve_timestamps`` / ``preserve_permissions``
+    are applied client-side on a best-effort basis, but only when the adapters
+    expose the matching optional hooks — failing fast beats silently migrating
+    without the requested fidelity.
+    """
+    if options.preserve_versions:
+        raise NotImplementedError(
+            "preserve_versions is not supported by the client-side runner: REST cannot restore "
+            "version history. Use the server-side Migration API (MigrationServerJob) or set "
+            "preserve_versions to False."
+        )
+    if options.preserve_timestamps and not callable(getattr(target, "apply_timestamps", None)):
+        raise NotImplementedError(
+            "preserve_timestamps is not supported by this target: it does not implement apply_timestamps(item)."
+        )
+    if options.preserve_permissions and not (
+        callable(getattr(source, "read_permissions", None)) and callable(getattr(target, "apply_permissions", None))
+    ):
+        raise NotImplementedError(
+            "preserve_permissions is not supported by this adapter pair: it needs "
+            "source.read_permissions(item) and target.apply_permissions(item, permissions)."
+        )
+
+
+def _apply_fidelity(source, target: DataTarget, item: MigrationItem, options: MigrationOptions) -> None:
+    """Best-effort fidelity after a successful write (timestamps / ACLs).
+
+    Failures propagate (and are captured per-item by the caller) so a requested
+    fidelity that couldn't be applied is visible and the item stays resumable.
+    """
+    if options.preserve_timestamps:
+        _call_optional(target, "apply_timestamps", item)
+    if options.preserve_permissions:
+        permissions = source.read_permissions(item)
+        if permissions:
+            _call_optional(target, "apply_permissions", item, permissions)
+
+
+class _Watermark:
+    """Incremental watermark — the highest source ``modified`` migrated so far.
+
+    Persisted in ``Checkpoint.source_watermark``: a resumed incremental run skips
+    every item at or below it, so only new/changed items are re-scanned.
+    """
+
+    def __init__(self, checkpoint: Checkpoint) -> None:
+        self._checkpoint = checkpoint
+        self._value = checkpoint.source_watermark
+
+    @property
+    def value(self) -> str | None:
+        return self._value
+
+    def is_stale(self, item: MigrationItem) -> bool:
+        """Whether the item is at or below the watermark (already migrated)."""
+        return self._value is not None and item.modified is not None and item.modified <= self._value
+
+    def advance(self, item: MigrationItem) -> None:
+        """Raise the watermark to the item's ``modified`` when it is newer."""
+        if item.modified is not None and (self._value is None or item.modified > self._value):
+            self._value = item.modified
+            self._checkpoint.source_watermark = item.modified
+
+
 class MigrationRunner:
     """Executes migration items between a source and a target adapter."""
 
@@ -47,16 +115,20 @@ class MigrationRunner:
         progress: Callable[["Progress"], None] | None = None,
         stop_event: Callable[[], bool] | None = None,
     ) -> MigrationStats:
+        _assert_fidelity_supported(options, source, target)
+        watermark = _Watermark(checkpoint) if options.incremental else None
         parallel = (
             options.concurrency > 1
             and hasattr(target, "write_many")
             and options.conflict_resolution != ConflictResolution.RENAME
         )
         if parallel:
-            stats = self._run_parallel(source, target, items, options, checkpoint, checkpoint_path, progress, stop_event)
+            stats = self._run_parallel(
+                source, target, items, options, checkpoint, checkpoint_path, progress, stop_event, watermark
+            )
         else:
             stats = self._run_sequential(
-                source, target, items, options, checkpoint, checkpoint_path, progress, stop_event
+                source, target, items, options, checkpoint, checkpoint_path, progress, stop_event, watermark
             )
         _call_optional(target, "commit", options)
 
@@ -78,6 +150,7 @@ class MigrationRunner:
         checkpoint_path: str | Path | None,
         progress: Callable[["Progress"], None] | None,
         stop_event: Callable[[], bool] | None,
+        watermark: _Watermark | None = None,
     ) -> MigrationStats:
         items = list(items)
         stats = MigrationStats(total=len(items))
@@ -90,12 +163,19 @@ class MigrationRunner:
             if callable(stop_event) and stop_event():
                 checkpoint.phase = MigrationPhase.PAUSED
                 break
+            if watermark is not None and watermark.is_stale(item):
+                checkpoint.record(item, ItemStatus.SKIPPED)
+                stats.skipped += 1
+                self._report_progress(progress, stats, item)
+                continue
             checkpoint.record(item, ItemStatus.IN_PROGRESS)
             try:
                 if self._migrate(source, target, item, options):
                     checkpoint.record(item, ItemStatus.DONE)
                     stats.success += 1
                     stats.bytes_transferred += item.size_bytes
+                    if watermark is not None:
+                        watermark.advance(item)
                 else:
                     checkpoint.record(item, ItemStatus.SKIPPED)
                     stats.skipped += 1
@@ -119,11 +199,17 @@ class MigrationRunner:
         checkpoint_path: str | Path | None,
         progress: Callable[["Progress"], None] | None,
         stop_event: Callable[[], bool] | None,
+        watermark: _Watermark | None = None,
     ) -> MigrationStats:
         items = list(items)
         stats = MigrationStats(total=len(items))
         chunk: list[MigrationItem] = []
 
+        def _skip(item: MigrationItem) -> None:
+            checkpoint.record(item, ItemStatus.SKIPPED)
+            stats.skipped += 1
+            self._report_progress(progress, stats, item)
+
         def _flush() -> None:
             if not chunk:
                 return
@@ -137,9 +223,19 @@ class MigrationRunner:
                     checkpoint.record(item, ItemStatus.FAILED)
                     stats.errors += 1
                 else:
-                    checkpoint.record(item, ItemStatus.DONE)
-                    stats.success += 1
-                    stats.bytes_transferred += item.size_bytes
+                    try:
+                        _apply_fidelity(source, target, item, options)
+                    except Exception as e:  # noqa: BLE001 — per-item errors are captured, not fatal
+                        item.error = str(e)
+                        item.error_code = type(e).__name__
+                        checkpoint.record(item, ItemStatus.FAILED)
+                        stats.errors += 1
+                    else:
+                        checkpoint.record(item, ItemStatus.DONE)
+                        stats.success += 1
+                        stats.bytes_transferred += item.size_bytes
+                        if watermark is not None:
+                            watermark.advance(item)
                 self._report_progress(progress, stats, item)
             chunk.clear()
             if checkpoint_path is not None:
@@ -154,11 +250,12 @@ class MigrationRunner:
             if callable(stop_event) and stop_event():
                 checkpoint.phase = MigrationPhase.PAUSED
                 break
+            if (watermark is not None and watermark.is_stale(item)) or _outside_window(item, options):
+                _skip(item)
+                continue
             checkpoint.record(item, ItemStatus.IN_PROGRESS)
             if options.incremental and _target_up_to_date(source, target, item):
-                checkpoint.record(item, ItemStatus.SKIPPED)
-                stats.skipped += 1
-                self._report_progress(progress, stats, item)
+                _skip(item)
                 continue
             if options.conflict_resolution == ConflictResolution.SKIP and target.exists(item):
                 checkpoint.record(item, ItemStatus.SKIPPED)
@@ -178,7 +275,9 @@ class MigrationRunner:
 
     @staticmethod
     def _migrate(source, target: DataTarget, item: MigrationItem, options: MigrationOptions) -> bool:
-        """Move one item; returns ``False`` when skipped (conflict/incremental)."""
+        """Move one item; returns ``False`` when skipped (conflict/incremental/window)."""
+        if _outside_window(item, options):
+            return False
         if options.incremental and _target_up_to_date(source, target, item):
             return False
         if options.conflict_resolution == ConflictResolution.SKIP and target.exists(item):
@@ -187,7 +286,15 @@ class MigrationRunner:
         item.dest_path = dest
         payload = source.read(item)
         target.write(item, payload)
+        _apply_fidelity(source, target, item, options)
+        return True
+
+
+def _outside_window(item: MigrationItem, options: MigrationOptions) -> bool:
+    """Whether the item is outside the ``created_after``/``modified_after`` window (skip it)."""
+    if options.modified_after and item.modified is not None and item.modified <= options.modified_after:
         return True
+    return bool(options.created_after and item.created is not None and item.created <= options.created_after)
 
 
 def _target_up_to_date(source, target: DataTarget, item: MigrationItem) -> bool:

office365/migration/server_job.py

@@ -2,26 +2,68 @@
 
 The SharePoint Migration API runs large migrations server-side: content is
 packaged and uploaded to Azure Storage, then an *ingestion job* ingests it. This
-wrapper submits such a job (``Site.create_migration_ingestion_job``) and polls
-its status with a ``Progress`` hook.
+wrapper submits such a job and polls its status with a ``Progress`` hook.
 
-The status source is abstracted (``status_fn``) so callers can poll whichever
-endpoint reports the job — e.g. the Azure report queue, or the
-Graph ``SharePointMigrationJob`` progress events.
+Status can be read two ways:
+
+- ``GetMigrationJobProgress`` — the recommended API; use :meth:`status_fn` as the
+  ``monitor`` status source (or :meth:`progress` directly);
+- a caller-supplied ``status_fn`` — e.g. an Azure queue or the Graph
+  ``SharePointMigrationJobProgressEvent`` stream.
 """
 
 from __future__ import annotations
 
+import json
 import time
 import uuid
 from collections.abc import Callable
 from typing import TYPE_CHECKING
 
-from office365.migration._util import emit_progress
+from office365.runtime.operations import emit_progress
 
 if TYPE_CHECKING:
     from office365.runtime.operations import Progress
 
+__all__ = ["MigrationServerJob", "job_errors", "parse_progress_events"]
+
+_TERMINAL = ("succeeded", "completed", "failed", "cancelled")
+
+# Migration API progress events -> monitor statuses (JobEnd resolved separately).
+_EVENT_STATUS = {
+    "JobQueued": "queued",
+    "JobStart": "processing",
+    "JobProgress": "processing",
+    "JobError": "processing",
+}
+
+
+def parse_progress_events(events: list[dict]) -> tuple[str, int, int | None]:
+    """Reduce accumulated Migration API progress events to ``(status, done, total)``.
+
+    Pass the events accumulated across polls (they are append-only), so the status
+    stays monotonic even when a poll returns no new events.
+    """
+    status = "queued"
+    done = 0
+    total: int | None = None
+    for event in events:
+        kind = event.get("Event")
+        if kind == "JobEnd":
+            status = "failed" if int(event.get("TotalErrors") or 0) else "succeeded"
+        elif kind in _EVENT_STATUS:
+            status = _EVENT_STATUS[kind]
+        if event.get("ObjectsProcessed") is not None:
+            done = int(event["ObjectsProcessed"])
+        if event.get("TotalExpectedSPObjects") is not None:
+            total = int(event["TotalExpectedSPObjects"])
+    return status, done, total
+
+
+def job_errors(events: list[dict]) -> list[dict]:
+    """The ``JobError`` events from a progress log (each carries ``Message``/``ErrorType``/``Url``)."""
+    return [event for event in events if event.get("Event") == "JobError"]
+
 
 class MigrationServerJob:
     """A server-side migration job: submit an ingestion job, then monitor it."""
@@ -34,16 +76,16 @@ class MigrationServerJob:
         g_web_id,
         azure_container_source_uri: str,
         azure_container_manifest_uri: str,
-        azure_queue_report_uri: str,
+        azure_queue_report_uri: str | None = None,
         ingestion_task_key: str | None = None,
     ) -> str:
-        """Submit an ingestion job and return its job id.
+        """Submit an ingestion job for the staged package and return its job id.
 
         Args:
             g_web_id: Identifier of the destination web.
-            azure_container_source_uri: Azure container URI holding the data.
-            azure_container_manifest_uri: Azure container URI holding the manifest.
-            azure_queue_report_uri: Azure queue URI receiving progress reports.
+            azure_container_source_uri: Content container URI (with SAS token).
+            azure_container_manifest_uri: Manifest container URI (with SAS token).
+            azure_queue_report_uri: Optional Azure queue URI receiving progress reports.
             ingestion_task_key: Optional task key (a UUID is generated when omitted).
         """
         result = self._site.create_migration_ingestion_job(
@@ -55,10 +97,79 @@ class MigrationServerJob:
         )
         return result.execute_query().value
 
+    def submit_encrypted(
+        self,
+        g_web_id,
+        azure_container_source_uri: str,
+        azure_container_manifest_uri: str,
+        aes256_cbc_key: str | bytes,
+        azure_queue_report_uri: str | None = None,
+    ) -> str:
+        """Submit an ingestion job for an AES-256-CBC encrypted package.
+
+        Required for SharePoint-provided containers; use the ``EncryptionKey`` from
+        ``Site.provision_migration_containers`` (a base64 string, passed through).
+        """
+        result = self._site.create_migration_job_encrypted(
+            g_web_id=g_web_id,
+            azure_container_source_uri=azure_container_source_uri,
+            azure_container_manifest_uri=azure_container_manifest_uri,
+            aes256_cbc_key=aes256_cbc_key,
+            azure_queue_report_uri=azure_queue_report_uri,
+        )
+        return result.execute_query().value
+
+    def progress(self, job_id: str, next_token: str = "0") -> tuple[list[dict], str]:
+        """Fetch a page of progress events and the next paging token.
+
+        Args:
+            job_id: The migration job id.
+            next_token: Paging token; use ``"0"`` for the initial request.
+
+        Returns:
+            ``(events, next_token)`` — events are decoded from the JSON log strings.
+        """
+        result = self._site.get_migration_job_progress(job_id, next_token).execute_query()
+        value = result.value
+        events = [json.loads(line) for line in (value.Logs or [])]
+        return events, value.NextToken or next_token
+
+    def all_events(self, job_id: str) -> list[dict]:
+        """All progress events for a job (paged until the token stops advancing).
+
+        For a completed job the API is idempotent per token, so this collects the
+        full lifecycle — including any ``JobError`` entries.
+        """
+        events: list[dict] = []
+        token = "0"
+        for _ in range(100):  # bound: a page with no new events ends the loop
+            page, next_token = self.progress(job_id, token)
+            events.extend(page)
+            if not page or next_token == token:
+                break
+            token = next_token
+        return events
+
+    def errors(self, job_id: str) -> list[dict]:
+        """The ``JobError`` events for a job (message, type, url)."""
+        return job_errors(self.all_events(job_id))
+
+    def status_fn(self) -> Callable[[str], tuple[str, int, int | None]]:
+        """A ``monitor`` status function backed by ``GetMigrationJobProgress``."""
+        state: dict = {"token": "0", "events": []}
+
+        def _status(job_id: str) -> tuple[str, int, int | None]:
+            events, token = self.progress(job_id, state["token"])
+            state["token"] = token
+            state["events"].extend(events)
+            return parse_progress_events(state["events"])
+
+        return _status
+
     def monitor(
         self,
         job_id: str,
-        status_fn: Callable[[str], tuple[str, int, int | None]],
+        status_fn: Callable[[str], tuple[str, int, int | None]] | None = None,
         interval: float = 5,
         timeout: float = 1800,
         progress: Callable[["Progress"], None] | None = None,
@@ -68,7 +179,7 @@ class MigrationServerJob:
         Args:
             job_id: The migration job id.
             status_fn: Callable returning ``(status, done, total)`` for a job id.
-              ``done``/``total`` feed the ``Progress`` hook (total may be None).
+              Defaults to a ``GetMigrationJobProgress``-backed reader.
             interval: Seconds between polls.
             timeout: Maximum seconds to wait before raising ``TimeoutError``.
             progress: Optional hook fired per poll with a ``Progress`` snapshot.
@@ -79,11 +190,12 @@ class MigrationServerJob:
         Raises:
             TimeoutError: When the job doesn't finish within ``timeout`` seconds.
         """
+        status_fn = status_fn or self.status_fn()
         elapsed = 0.0
         while elapsed < timeout:
             status, done, total = status_fn(job_id)
             emit_progress(progress, done=done, total=total, stage="migrating")
-            if status.lower() in ("succeeded", "completed", "failed", "cancelled"):
+            if status.lower() in _TERMINAL:
                 return status
             time.sleep(interval)
             elapsed += interval