Office365-REST-Python-Client 3.0.0__py3-none-any.whl → 3.2.0__py3-none-any.whl

office365/migration/assessment/scanners/base.py

@@ -0,0 +1,123 @@
+"""Assessment scanner base — options, the shared flag helper, and the run contract."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Generic, Optional, TypeVar
+
+from office365.migration.assessment.containers import ScanContainer
+from office365.migration.assessment.issue import AssessmentIssue
+from office365.migration.assessment.report import AssessmentReport
+from office365.runtime.limits import Limit
+
+PayloadT = TypeVar("PayloadT")
+RecordT = TypeVar("RecordT")
+
+
+@dataclass(frozen=True)
+class ScanTarget(Generic[PayloadT]):
+    """A loaded payload handed to a scan — its container, data, and location.
+
+    ``location`` is derived once by the walker (no hand-built paths in scans).
+    For ``SITE`` scans ``entity`` is the product walker-built site summary.
+    """
+
+    container: ScanContainer
+    entity: PayloadT
+    location: str = ""
+
+
+@dataclass
+class AssessmentOptions:
+    """Product-agnostic scanner options.
+
+    Product packages subclass this to seed the thresholds and the ``limits``
+    mapping from their catalog (e.g. ``SharePointAssessmentOptions``).
+    ``disabled_scans`` mirrors SMAT's ScanDef.json ``Enabled`` flag: a scan
+    listed here does not run and its data is not collected.
+    """
+
+    max_path_length: Optional[int] = None
+    max_name_length: Optional[int] = None
+    invalid_chars: set[str] = field(default_factory=set)
+    large_file_bytes: Optional[int] = None
+    large_excel_bytes: Optional[int] = None
+    long_onedrive_url: Optional[int] = None
+    list_view_threshold: Optional[int] = None
+    index_threshold: Optional[int] = None
+    max_list_items: Optional[int] = None
+    lookup_joins: Optional[int] = None
+    unique_scopes: Optional[int] = None
+    recommended_unique_scopes: Optional[int] = None
+    large_site_threshold_gb: Optional[float] = None
+    strip_field_attrs: set[str] = field(default_factory=set)
+    approval_workflow_fields: set[str] = field(default_factory=set)
+    disabled_scans: set[str] = field(default_factory=lambda: {"permissions"})
+    include_site_admins: bool = False
+    system_field_names: set[str] = field(default_factory=set)
+    #: Maps each threshold field to its source :class:`Limit` (seeded by the
+    #: product) so findings can reference the authoritative limit.
+    limits: dict[str, Limit] = field(default_factory=dict)
+
+    def limit(self, name: str) -> Limit:
+        """The source :class:`Limit` for a threshold field (from ``limits``)."""
+        limit = self.limits.get(name)
+        if limit is None:
+            value = getattr(self, name, None)
+            if value is None:
+                raise ValueError(f"no limit or value configured for {name!r}")
+            limit = Limit(name, int(value))
+        return limit
+
+
+class BaseScanner(Generic[RecordT]):
+    """A focused pre-migration check scoped to a container.
+
+    Scanners implement a single :meth:`run` over the data the walker loads for
+    their container. ``record_type`` marks report scans (whose dataclass fields
+    ARE the detail-report columns); issue scanners only ``flag``. ``finalize``
+    is reserved for ``SITE`` report scans that assemble a row after the walk
+    settles.
+    """
+
+    category: str = "general"  # issue-category label (AssessmentIssue.category)
+    scan_name: str = ""  # report name for scans that emit records (e.g. "LargeSites")
+
+    # Which list-items projection this scan consumes. ITEMS container scans
+    # normally share the default load; a scan needing a different projection
+    # (e.g. paged unique-permission items) overrides this.
+    items_load: str = "default"
+
+    # Typed report row for scans that emit SMAT-style detail records. Its
+    # dataclass fields ARE the report columns (SMAT headers), so ``columns``
+    # and the CSV/JSON export stay trivial.
+    record_type: type[RecordT] | None = None
+
+    def __init__(self, options: AssessmentOptions | None = None) -> None:
+        self.options = options or AssessmentOptions()
+        self.records: list[RecordT] = []
+
+    @property
+    def columns(self) -> tuple[str, ...]:
+        """SMAT detail-report columns (``<ScanName>-detail.csv`` header)."""
+        fields = getattr(self.record_type, "__dataclass_fields__", None)
+        return tuple(fields) if fields else ()
+
+    def flag(
+        self,
+        report: AssessmentReport,
+        severity: str,
+        location: str,
+        message: str,
+        suggestion: str = "",
+        risk_code: str = "",
+        limit: Optional[Limit] = None,
+    ) -> None:
+        report.issues.append(AssessmentIssue(severity, self.category, location, message, suggestion, risk_code, limit))
+
+    def run(self, target: ScanTarget[Any], report: AssessmentReport) -> None:
+        """Inspect a loaded container payload (target.entity) and flag / record."""
+        raise NotImplementedError
+
+    def finalize(self, report: AssessmentReport) -> None:
+        """All scans for the collection have settled — build rows / flag issues."""

office365/migration/base.py

@@ -1,12 +1,15 @@
 """
-Shared types, enums, and the MigrationQuery base class.
+Shared types and enums for the migration toolkit.
 """
 
 from __future__ import annotations
 
-from dataclasses import dataclass, field
+from dataclasses import asdict, dataclass, field, fields
 from enum import Enum
 
+from office365.runtime.limits import DEFAULT_BATCH_SIZE
+from office365.runtime.operations import OperationStats
+
 
 class ConflictResolution(str, Enum):
     SKIP = "skip"
@@ -28,6 +31,35 @@ class ExportFormat(str, Enum):
     DELTA = "delta"
 
 
+class ItemStatus(str, Enum):
+    """Per-item status, persisted in the checkpoint for resumable migrations."""
+
+    PENDING = "pending"
+    IN_PROGRESS = "in_progress"
+    DONE = "done"
+    SKIPPED = "skipped"
+    FAILED = "failed"
+
+
+class MigrationPhase(str, Enum):
+    """Lifecycle of a migration job (scan -> plan -> run -> monitor).
+
+    ``paused`` / ``failed`` / ``cancelled`` all retain a persisted checkpoint, so
+    the job can be resumed from where it stopped.
+    """
+
+    CREATED = "created"
+    ASSESSING = "assessing"
+    PLANNING = "planning"
+    RUNNING = "running"
+    PAUSED = "paused"
+    VERIFYING = "verifying"
+    COMPLETED = "completed"
+    COMPLETED_WITH_ERRORS = "completed_with_errors"
+    FAILED = "failed"
+    CANCELLED = "cancelled"
+
+
 @dataclass
 class MigrationItem:
     """Unit of work flowing through the migration pipeline."""
@@ -36,16 +68,30 @@ class MigrationItem:
     dest_path: str
     size_bytes: int = 0
     item_type: str = "file"
-    status: str = "pending"
-    error: Exception | None = None
+    status: ItemStatus = ItemStatus.PENDING
+    error: str | None = None  # failure message (SPMT "Message")
+    error_code: str | None = None  # failure category/code (e.g. exception type name)
+    modified: str | None = None  # source last-modified (ISO-8601), for incremental
+    created: str | None = None  # source created (ISO-8601)
+    author_id: int | None = None  # source author user id (system ``AuthorId``)
+    editor_id: int | None = None  # source last-modified-by user id (system ``EditorId``)
+
+
+@dataclass
+class PermissionEntry:
+    """A single principal → role-names grant, used for best-effort ACL preservation.
+
+    Produced by :meth:`DataSource.read_permissions` and consumed by
+    :meth:`DataTarget.apply_permissions`; role names are the target's role
+    definition names (e.g. ``"Read"``, ``"Edit"``, ``"Full Control"``).
+    """
+
+    principal_name: str
+    roles: list[str] = field(default_factory=list)
 
 
 @dataclass
-class MigrationStats:
-    total: int = 0
-    success: int = 0
-    skipped: int = 0
-    errors: int = 0
+class MigrationStats(OperationStats):
     bytes_transferred: int = 0
 
     def summary(self) -> str:
@@ -60,9 +106,37 @@ class MigrationStats:
 @dataclass
 class MigrationOptions:
     conflict_resolution: ConflictResolution = ConflictResolution.SKIP
-    preserve_timestamps: bool = True
+    incremental: bool = False  # skip items at/below the persisted watermark (and target-newer ones)
+    # Fidelity flags. ``preserve_timestamps`` and ``preserve_permissions`` are
+    # applied client-side on a best-effort basis when the adapters support it
+    # (see ``DataSource.read_permissions`` / ``DataTarget.apply_timestamps`` /
+    # ``DataTarget.apply_permissions``); REST cannot restore version history, so
+    # ``preserve_versions`` requires the server-side Migration API
+    # (``MigrationServerJob``). Defaults are False so they are honest no-ops
+    # rather than silently doing nothing.
+    preserve_timestamps: bool = False
     preserve_permissions: bool = False
     preserve_versions: bool = False
     include_patterns: list[str] = field(default_factory=list)
     exclude_patterns: list[str] = field(default_factory=list)
-    batch_size: int = 100
+    # Date window (ISO-8601): skip items created/modified at or before these
+    created_after: str | None = None
+    modified_after: str | None = None
+    batch_size: int = DEFAULT_BATCH_SIZE
+    concurrency: int = 1  # parallel workers for bulk file upload / batched writes
+
+
+def item_to_dict(item: MigrationItem) -> dict:
+    """Serialize a ``MigrationItem`` to a JSON-safe dict (enum/error by value)."""
+    data = asdict(item)
+    data["status"] = item.status.value
+    data["error"] = str(item.error) if item.error else None
+    return data
+
+
+def item_from_dict(data: dict) -> MigrationItem:
+    """Rebuild a ``MigrationItem`` (missing fields fall back to dataclass defaults)."""
+    kwargs = {f.name: data[f.name] for f in fields(MigrationItem) if f.name in data}
+    if "status" in kwargs:
+        kwargs["status"] = ItemStatus(kwargs["status"])
+    return MigrationItem(**kwargs)

office365/migration/checkpoint.py

@@ -0,0 +1,76 @@
+"""Migration checkpoint — the persisted run state enabling resume.
+
+A checkpoint records the job's phase, an optional source watermark (for
+incremental migrations), and the per-item status for every item in the
+manifest. The runner persists it after each batch, so an interrupted or failed
+run can resume by re-driving only ``pending`` / ``failed`` / ``in_progress``
+items — a checkpointed, idempotent, at-least-once migration.
+"""
+
+from __future__ import annotations
+
+import json
+import uuid
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from office365.migration.base import ItemStatus, MigrationItem, MigrationPhase
+from office365.runtime.converters.scalars import utc_now_iso
+
+
+@dataclass
+class Checkpoint:
+    """Run-state snapshot for a migration job."""
+
+    run_id: str
+    phase: MigrationPhase = MigrationPhase.CREATED
+    updated_at: str = ""
+    source_watermark: str | None = None
+    items: dict[str, str] = field(default_factory=dict)  # dest_path -> ItemStatus.value
+
+    @classmethod
+    def create(cls, run_id: str | None = None) -> "Checkpoint":
+        return cls(
+            run_id=run_id or uuid.uuid4().hex[:12],
+            phase=MigrationPhase.CREATED,
+            updated_at=utc_now_iso(),
+        )
+
+    def status_of(self, item: MigrationItem) -> ItemStatus:
+        return ItemStatus(self.items.get(item.dest_path, ItemStatus.PENDING.value))
+
+    def record(self, item: MigrationItem, status: ItemStatus) -> None:
+        self.items[item.dest_path] = status.value
+        item.status = status
+        self.updated_at = utc_now_iso()
+
+    def done_count(self) -> int:
+        return sum(1 for v in self.items.values() if v == ItemStatus.DONE.value)
+
+    def save(self, path: str | Path) -> None:
+        """Persist the checkpoint as JSON."""
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(
+                {
+                    "run_id": self.run_id,
+                    "phase": self.phase.value,
+                    "updated_at": self.updated_at,
+                    "source_watermark": self.source_watermark,
+                    "items": self.items,
+                },
+                f,
+                indent=2,
+            )
+
+    @classmethod
+    def load(cls, path: str | Path) -> "Checkpoint":
+        """Load a checkpoint previously persisted with :meth:`save`."""
+        with open(path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        return cls(
+            run_id=data["run_id"],
+            phase=MigrationPhase(data.get("phase", MigrationPhase.CREATED.value)),
+            updated_at=data.get("updated_at", ""),
+            source_watermark=data.get("source_watermark"),
+            items=data.get("items", {}),
+        )

office365/migration/job.py

@@ -0,0 +1,209 @@
+"""Migration job — a single client-side copy between source and target.
+
+Lifecycle: ``assess -> plan -> run -> verify``, with pause/resume
+from a persisted checkpoint. The job is platform-agnostic — it works against any
+``DataSource`` / ``DataTarget`` adapter pair. (For server-side Azure ingestion,
+see :class:`MigrationServerJob` instead.)
+"""
+
+from __future__ import annotations
+
+import threading
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Callable, cast
+
+from office365.migration.base import MigrationOptions, MigrationPhase, MigrationStats
+from office365.migration.checkpoint import Checkpoint
+from office365.migration.manifest import Manifest
+from office365.migration.runner import MigrationRunner
+
+if TYPE_CHECKING:
+    from office365.runtime.operations import Progress
+
+
+class MigrationJob:
+    """A resumable migration between a source and a target adapter."""
+
+    def __init__(
+        self,
+        source,
+        target,
+        options: MigrationOptions | None = None,
+        run_id: str | None = None,
+        manifest_path: str | Path | None = None,
+        checkpoint_path: str | Path | None = None,
+    ) -> None:
+        self._source = source
+        self._target = target
+        self._options = options or MigrationOptions()
+        self._manifest = Manifest()
+        self._checkpoint = Checkpoint.create(run_id)
+        self._stats = MigrationStats()
+        self._manifest_path = manifest_path
+        self._checkpoint_path = checkpoint_path
+        self._runner = MigrationRunner()
+        self._stop_event = threading.Event()
+        self._cancel_requested = False
+        self._assess_hook: Any = None
+        self._started_at: datetime | None = None
+        self._finished_at: datetime | None = None
+
+    # ── Configuration ────────────────────────────────────────────
+
+    def with_assessor(self, hook) -> "MigrationJob":
+        """Attach a pre-flight assessment (a ``MigrationAssessor`` or a callable).
+
+        The hook runs during :meth:`assess` and is the "scan" phase of the job.
+        """
+        self._assess_hook = hook
+        return self
+
+    # ── State ────────────────────────────────────────────────────
+
+    @property
+    def phase(self) -> MigrationPhase:
+        return self._checkpoint.phase
+
+    @property
+    def stats(self) -> MigrationStats:
+        return self._stats
+
+    @property
+    def manifest(self) -> Manifest:
+        return self._manifest
+
+    @property
+    def checkpoint(self) -> Checkpoint:
+        return self._checkpoint
+
+    @property
+    def source_label(self) -> str:
+        """A human-readable label for the source (adapter type / root)."""
+        label = getattr(self._source, "label", None)
+        return str(label()) if callable(label) else type(self._source).__name__
+
+    @property
+    def target_label(self) -> str:
+        label = getattr(self._target, "label", None)
+        return str(label()) if callable(label) else type(self._target).__name__
+
+    @property
+    def started_at(self) -> datetime | None:
+        return self._started_at
+
+    @property
+    def finished_at(self) -> datetime | None:
+        return self._finished_at
+
+    @property
+    def duration(self) -> float | None:
+        """Wall-clock duration of the last run, in seconds."""
+        if self._started_at is None or self._finished_at is None:
+            return None
+        return (self._finished_at - self._started_at).total_seconds()
+
+    # ── Lifecycle ────────────────────────────────────────────────
+
+    def assess(self, progress: Callable[["Progress"], None] | None = None) -> object | None:
+        """Run the pre-migration assessment (the "scan" phase).
+
+        Accepts a ``MigrationAssessor`` or a callable attached via
+        :meth:`with_assessor`; ``progress`` is forwarded when the hook is a
+        ``MigrationAssessor``.
+        """
+        self._checkpoint.phase = MigrationPhase.ASSESSING
+        self._save_state()
+        hook = self._assess_hook
+        assess: Any = getattr(hook, "assess", None)
+        if callable(assess):
+            result = cast(Any, assess(progress=progress))
+            return result.execute_query().value
+        if callable(hook):
+            return hook()
+        return None
+
+    def plan(self, progress: Callable[["Progress"], None] | None = None) -> Manifest:
+        """Enumerate the source into a persisted manifest."""
+        self._checkpoint.phase = MigrationPhase.PLANNING
+        self._manifest = Manifest.from_source(self._source, progress)
+        self._save_state()
+        return self._manifest
+
+    def run(self, progress: Callable[["Progress"], None] | None = None) -> MigrationStats:
+        """Execute the migration (resumable; re-drives pending/failed items)."""
+        self._checkpoint.phase = MigrationPhase.RUNNING
+        self._started_at = datetime.now(timezone.utc)
+        self._stats = self._runner.run(
+            self._source,
+            self._target,
+            self._manifest.items,
+            self._options,
+            self._checkpoint,
+            self._checkpoint_path,
+            progress,
+            stop_event=self._stop_event.is_set,
+        )
+        self._finished_at = datetime.now(timezone.utc)
+        if self._checkpoint.phase == MigrationPhase.PAUSED and self._cancel_requested:
+            self._checkpoint.phase = MigrationPhase.CANCELLED
+            self._save_state()
+        return self._stats
+
+    def pause(self) -> None:
+        """Request a clean stop at the next batch boundary (checkpoint saved)."""
+        self._stop_event.set()
+
+    def cancel(self) -> None:
+        """Request cancellation; the job is marked ``cancelled`` and stays resumable."""
+        self._cancel_requested = True
+        self._stop_event.set()
+
+    def resume(self, progress: Callable[["Progress"], None] | None = None) -> MigrationStats:
+        """Reload persisted state (if any) and continue from the last checkpoint."""
+        self._load_state()
+        self._stop_event.clear()
+        self._cancel_requested = False
+        return self.run(progress)
+
+    def status(self) -> MigrationPhase:
+        return self.phase
+
+    def verify(self, spot_checks: int = 20):
+        """Reconcile source vs target after migration (counts + checksum spot-checks)."""
+        from office365.migration.validators import verify as _verify
+
+        self._checkpoint.phase = MigrationPhase.VERIFYING
+        report = _verify(self._source, self._target, self._manifest, spot_checks)
+        self._checkpoint.phase = MigrationPhase.COMPLETED if report.ok else MigrationPhase.COMPLETED_WITH_ERRORS
+        return report
+
+    def export_reports(self, output_dir: str | Path):
+        """Write Summary/Item/Failure reports (CSV + JSON) for the last run.
+
+        Reuses the migration state (stats, manifest, checkpoint) through the
+        records form — see :mod:`office365.migration.report`.
+
+        Args:
+            output_dir: Directory to write the report files into.
+
+        Returns:
+            List of written file paths.
+        """
+        from office365.migration.report import export_reports as _export_reports
+
+        return _export_reports(self, output_dir)
+
+    # ── Helpers ──────────────────────────────────────────────────
+
+    def _save_state(self) -> None:
+        if self._manifest_path is not None:
+            self._manifest.save(self._manifest_path)
+        if self._checkpoint_path is not None:
+            self._checkpoint.save(self._checkpoint_path)
+
+    def _load_state(self) -> None:
+        if self._manifest_path is not None and Path(self._manifest_path).exists():
+            self._manifest = Manifest.load(self._manifest_path)
+        if self._checkpoint_path is not None and Path(self._checkpoint_path).exists():
+            self._checkpoint = Checkpoint.load(self._checkpoint_path)

office365/migration/manifest.py

@@ -0,0 +1,71 @@
+"""Migration manifest — the persisted plan of items to move.
+
+The manifest is the "what to migrate": a serializable list of
+:class:`MigrationItem` (source/dest path, size, type). It is produced by
+enumerating a :class:`DataSource` and persisted as JSON so a run can be
+planned, audited, and resumed.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Callable, Iterator
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from office365.migration.base import ItemStatus, MigrationItem, item_from_dict, item_to_dict
+
+if TYPE_CHECKING:
+    from office365.runtime.operations import Progress
+
+
+class Manifest:
+    """A serializable plan of ``MigrationItem`` units."""
+
+    def __init__(self, items: list[MigrationItem] | None = None) -> None:
+        self._items: list[MigrationItem] = list(items or [])
+
+    @classmethod
+    def from_source(
+        cls,
+        source,
+        progress: Callable[["Progress"], None] | None = None,
+    ) -> "Manifest":
+        """Enumerate a source into a manifest (with per-item progress)."""
+        return cls(source.list_items(progress))
+
+    def add(self, item: MigrationItem) -> "Manifest":
+        self._items.append(item)
+        return self
+
+    @property
+    def items(self) -> list[MigrationItem]:
+        return self._items
+
+    @property
+    def pending(self) -> list[MigrationItem]:
+        return [i for i in self._items if i.status == ItemStatus.PENDING]
+
+    @property
+    def failed(self) -> list[MigrationItem]:
+        return [i for i in self._items if i.status == ItemStatus.FAILED]
+
+    def by_dest(self) -> dict[str, MigrationItem]:
+        return {i.dest_path: i for i in self._items}
+
+    def __len__(self) -> int:
+        return len(self._items)
+
+    def __iter__(self) -> Iterator[MigrationItem]:
+        return iter(self._items)
+
+    def save(self, path: str | Path) -> None:
+        """Persist the manifest as a JSON array of items."""
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump([item_to_dict(i) for i in self._items], f, indent=2)
+
+    @classmethod
+    def load(cls, path: str | Path) -> "Manifest":
+        """Load a manifest previously persisted with :meth:`save`."""
+        with open(path, "r", encoding="utf-8") as f:
+            return cls([item_from_dict(d) for d in json.load(f)])

office365/migration/outlook/__init__.py

@@ -0,0 +1,20 @@
+"""Outlook (mail) assessment for the migration toolkit.
+
+Reuses the shared assessment model (containers, scans, reports, export); the
+product-specific part is the mailbox walker (:class:`MailboxAssessor`) and its
+scanners (e.g. the ``MailFolders`` folder inventory).
+"""
+
+from office365.migration.outlook.assessor import MailboxAssessor
+from office365.migration.outlook.registry import OUTLOOK_SCANS, outlook_scan_pairs
+from office365.migration.outlook.scanner import MailboxFolderRecord, MailboxFolderScan, MailFolderData, OutlookOptions
+
+__all__ = [
+    "OUTLOOK_SCANS",
+    "MailFolderData",
+    "MailboxAssessor",
+    "MailboxFolderRecord",
+    "MailboxFolderScan",
+    "OutlookOptions",
+    "outlook_scan_pairs",
+]