Office365-REST-Python-Client 3.0.0__py3-none-any.whl → 3.2.0__py3-none-any.whl

office365/migration/server_job.py

@@ -0,0 +1,202 @@
+"""Server-side migration job — submit + monitor.
+
+The SharePoint Migration API runs large migrations server-side: content is
+packaged and uploaded to Azure Storage, then an *ingestion job* ingests it. This
+wrapper submits such a job and polls its status with a ``Progress`` hook.
+
+Status can be read two ways:
+
+- ``GetMigrationJobProgress`` — the recommended API; use :meth:`status_fn` as the
+  ``monitor`` status source (or :meth:`progress` directly);
+- a caller-supplied ``status_fn`` — e.g. an Azure queue or the Graph
+  ``SharePointMigrationJobProgressEvent`` stream.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+import uuid
+from collections.abc import Callable
+from typing import TYPE_CHECKING
+
+from office365.runtime.operations import emit_progress
+
+if TYPE_CHECKING:
+    from office365.runtime.operations import Progress
+
+__all__ = ["MigrationServerJob", "job_errors", "parse_progress_events"]
+
+_TERMINAL = ("succeeded", "completed", "failed", "cancelled")
+
+# Migration API progress events -> monitor statuses (JobEnd resolved separately).
+_EVENT_STATUS = {
+    "JobQueued": "queued",
+    "JobStart": "processing",
+    "JobProgress": "processing",
+    "JobError": "processing",
+}
+
+
+def parse_progress_events(events: list[dict]) -> tuple[str, int, int | None]:
+    """Reduce accumulated Migration API progress events to ``(status, done, total)``.
+
+    Pass the events accumulated across polls (they are append-only), so the status
+    stays monotonic even when a poll returns no new events.
+    """
+    status = "queued"
+    done = 0
+    total: int | None = None
+    for event in events:
+        kind = event.get("Event")
+        if kind == "JobEnd":
+            status = "failed" if int(event.get("TotalErrors") or 0) else "succeeded"
+        elif kind in _EVENT_STATUS:
+            status = _EVENT_STATUS[kind]
+        if event.get("ObjectsProcessed") is not None:
+            done = int(event["ObjectsProcessed"])
+        if event.get("TotalExpectedSPObjects") is not None:
+            total = int(event["TotalExpectedSPObjects"])
+    return status, done, total
+
+
+def job_errors(events: list[dict]) -> list[dict]:
+    """The ``JobError`` events from a progress log (each carries ``Message``/``ErrorType``/``Url``)."""
+    return [event for event in events if event.get("Event") == "JobError"]
+
+
+class MigrationServerJob:
+    """A server-side migration job: submit an ingestion job, then monitor it."""
+
+    def __init__(self, site) -> None:
+        self._site = site
+
+    def submit(
+        self,
+        g_web_id,
+        azure_container_source_uri: str,
+        azure_container_manifest_uri: str,
+        azure_queue_report_uri: str | None = None,
+        ingestion_task_key: str | None = None,
+    ) -> str:
+        """Submit an ingestion job for the staged package and return its job id.
+
+        Args:
+            g_web_id: Identifier of the destination web.
+            azure_container_source_uri: Content container URI (with SAS token).
+            azure_container_manifest_uri: Manifest container URI (with SAS token).
+            azure_queue_report_uri: Optional Azure queue URI receiving progress reports.
+            ingestion_task_key: Optional task key (a UUID is generated when omitted).
+        """
+        result = self._site.create_migration_ingestion_job(
+            g_web_id=g_web_id,
+            azure_container_source_uri=azure_container_source_uri,
+            azure_container_manifest_uri=azure_container_manifest_uri,
+            azure_queue_report_uri=azure_queue_report_uri,
+            ingestion_task_key=ingestion_task_key or uuid.uuid4().hex,
+        )
+        return result.execute_query().value
+
+    def submit_encrypted(
+        self,
+        g_web_id,
+        azure_container_source_uri: str,
+        azure_container_manifest_uri: str,
+        aes256_cbc_key: str | bytes,
+        azure_queue_report_uri: str | None = None,
+    ) -> str:
+        """Submit an ingestion job for an AES-256-CBC encrypted package.
+
+        Required for SharePoint-provided containers; use the ``EncryptionKey`` from
+        ``Site.provision_migration_containers`` (a base64 string, passed through).
+        """
+        result = self._site.create_migration_job_encrypted(
+            g_web_id=g_web_id,
+            azure_container_source_uri=azure_container_source_uri,
+            azure_container_manifest_uri=azure_container_manifest_uri,
+            aes256_cbc_key=aes256_cbc_key,
+            azure_queue_report_uri=azure_queue_report_uri,
+        )
+        return result.execute_query().value
+
+    def progress(self, job_id: str, next_token: str = "0") -> tuple[list[dict], str]:
+        """Fetch a page of progress events and the next paging token.
+
+        Args:
+            job_id: The migration job id.
+            next_token: Paging token; use ``"0"`` for the initial request.
+
+        Returns:
+            ``(events, next_token)`` — events are decoded from the JSON log strings.
+        """
+        result = self._site.get_migration_job_progress(job_id, next_token).execute_query()
+        value = result.value
+        events = [json.loads(line) for line in (value.Logs or [])]
+        return events, value.NextToken or next_token
+
+    def all_events(self, job_id: str) -> list[dict]:
+        """All progress events for a job (paged until the token stops advancing).
+
+        For a completed job the API is idempotent per token, so this collects the
+        full lifecycle — including any ``JobError`` entries.
+        """
+        events: list[dict] = []
+        token = "0"
+        for _ in range(100):  # bound: a page with no new events ends the loop
+            page, next_token = self.progress(job_id, token)
+            events.extend(page)
+            if not page or next_token == token:
+                break
+            token = next_token
+        return events
+
+    def errors(self, job_id: str) -> list[dict]:
+        """The ``JobError`` events for a job (message, type, url)."""
+        return job_errors(self.all_events(job_id))
+
+    def status_fn(self) -> Callable[[str], tuple[str, int, int | None]]:
+        """A ``monitor`` status function backed by ``GetMigrationJobProgress``."""
+        state: dict = {"token": "0", "events": []}
+
+        def _status(job_id: str) -> tuple[str, int, int | None]:
+            events, token = self.progress(job_id, state["token"])
+            state["token"] = token
+            state["events"].extend(events)
+            return parse_progress_events(state["events"])
+
+        return _status
+
+    def monitor(
+        self,
+        job_id: str,
+        status_fn: Callable[[str], tuple[str, int, int | None]] | None = None,
+        interval: float = 5,
+        timeout: float = 1800,
+        progress: Callable[["Progress"], None] | None = None,
+    ) -> str:
+        """Poll a job until it reaches a terminal status.
+
+        Args:
+            job_id: The migration job id.
+            status_fn: Callable returning ``(status, done, total)`` for a job id.
+              Defaults to a ``GetMigrationJobProgress``-backed reader.
+            interval: Seconds between polls.
+            timeout: Maximum seconds to wait before raising ``TimeoutError``.
+            progress: Optional hook fired per poll with a ``Progress`` snapshot.
+
+        Returns:
+            The terminal status.
+
+        Raises:
+            TimeoutError: When the job doesn't finish within ``timeout`` seconds.
+        """
+        status_fn = status_fn or self.status_fn()
+        elapsed = 0.0
+        while elapsed < timeout:
+            status, done, total = status_fn(job_id)
+            emit_progress(progress, done=done, total=total, stage="migrating")
+            if status.lower() in _TERMINAL:
+                return status
+            time.sleep(interval)
+            elapsed += interval
+        raise TimeoutError(f"Migration job {job_id} did not finish within {timeout}s")

office365/migration/session.py

@@ -0,0 +1,188 @@
+"""Migration session — the SPMT-shaped facade over a batch of migrations.
+
+Mirrors the ``Microsoft.SharePoint.MigrationTool.PowerShell`` cmdlets:
+
+| cmdlet | method |
+|---|---|
+| ``Register-SPMTMigration`` | :meth:`MigrationSession.register` |
+| ``Get-SPMTMigration`` | :meth:`MigrationSession.get` |
+| ``Add-SPMTTask`` | :meth:`MigrationSession.add_task` |
+| ``Remove-SPMTTask`` | :meth:`MigrationSession.remove_task` |
+| ``Show-SPMTMigration`` | :meth:`MigrationSession.show` |
+| ``Start-SPMTMigration`` | :meth:`MigrationSession.start` |
+| ``Stop-SPMTMigration`` | :meth:`MigrationSession.stop` |
+| ``Unregister-SPMTMigration`` | :meth:`MigrationSession.unregister` |
+
+A task pairs a source and a target adapter — either supplied directly, or
+declared as a :class:`~office365.migration.tasks.MigrationTask` (resolved against
+the registered ``ClientContext``).
+"""
+
+from __future__ import annotations
+
+import itertools
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from office365.migration.base import MigrationOptions, MigrationPhase
+from office365.migration.job import MigrationJob
+
+if TYPE_CHECKING:
+    from office365.migration.settings import MigrationSettings
+    from office365.migration.tasks import MigrationTask
+
+__all__ = ["MigrationSession", "SessionTask"]
+
+
+def _job_status(job: MigrationJob) -> dict:
+    """Project a job's state — the status form of a session task."""
+    return {
+        "source": job.source_label,
+        "target": job.target_label,
+        "phase": job.phase.value,
+        "stats": {
+            "total": job.stats.total,
+            "success": job.stats.success,
+            "skipped": job.stats.skipped,
+            "errors": job.stats.errors,
+            "bytes_transferred": job.stats.bytes_transferred,
+        },
+    }
+
+
+@dataclass
+class SessionTask:
+    """A registered task: an id, its descriptor (when declared), and its job."""
+
+    id: str
+    job: MigrationJob
+    descriptor: "MigrationTask | None" = None
+
+    def status(self) -> dict:
+        data = _job_status(self.job)
+        data["id"] = self.id
+        if self.descriptor is not None:
+            data["descriptor"] = self.descriptor.label()
+        return data
+
+
+class MigrationSession:
+    """A batch of migration tasks, driven like the SPMT PowerShell cmdlets."""
+
+    def __init__(self, options: MigrationOptions | None = None, context=None, settings=None) -> None:
+        self._options = options
+        self._context = context
+        self._settings = settings
+        self._tasks: list[SessionTask] = []
+        self._ids = itertools.count(1)
+
+    # ── Cmdlets ──────────────────────────────────────────────────
+
+    def register(self, settings: "MigrationSettings | None" = None, context=None) -> "MigrationSession":
+        """Register the session — its settings and the target ``ClientContext``."""
+        if settings is not None:
+            self._settings = settings
+        if context is not None:
+            self._context = context
+        return self
+
+    def get(self) -> "MigrationSession":
+        """The registered session (``Get-SPMTMigration``)."""
+        return self
+
+    def add_task(
+        self,
+        source=None,
+        target=None,
+        *,
+        task: "MigrationTask | None" = None,
+        options: MigrationOptions | None = None,
+        manifest_path=None,
+        checkpoint_path=None,
+        **descriptor,
+    ) -> MigrationJob:
+        """Add a task — adapters directly, a ``MigrationTask``, or task kwargs.
+
+        Descriptor kwargs (``file_share_source`` / ``target_site_url`` /
+        ``target_list`` …) are resolved against the registered context.
+        """
+        if task is None and descriptor:
+            from office365.migration.tasks import MigrationTask
+
+            task = MigrationTask(**descriptor)
+        if task is not None:
+            if self._context is None:
+                raise ValueError("register a ClientContext first (session.register(context=ctx))")
+            from office365.migration.sharepoint.resolvers import resolve_task
+
+            source, target = resolve_task(task, self._context, self._settings)
+        job = MigrationJob(
+            source,
+            target,
+            options if options is not None else self._options,
+            manifest_path=manifest_path,
+            checkpoint_path=checkpoint_path,
+        )
+        self._tasks.append(SessionTask(str(next(self._ids)), job, task))
+        return job
+
+    def remove_task(self, task) -> None:
+        """Remove a task by its ``MigrationJob``, ``SessionTask`` or id."""
+        for entry in list(self._tasks):
+            if task in (entry, entry.job, entry.id):
+                self._tasks.remove(entry)
+                return
+        raise KeyError(task)
+
+    def show(self) -> list[dict]:
+        """The per-task status (``Show-SPMTMigration``)."""
+        return [entry.status() for entry in self._tasks]
+
+    def start(self, progress=None) -> list[dict]:
+        """Start the migration: plan + run every registered task."""
+        statuses: list[dict] = []
+        for entry in self._tasks:
+            job = entry.job
+            if job.phase in (
+                MigrationPhase.COMPLETED,
+                MigrationPhase.COMPLETED_WITH_ERRORS,
+                MigrationPhase.CANCELLED,
+            ):
+                statuses.append(entry.status())
+                continue
+            job.plan(progress)
+            job.run(progress)
+            statuses.append(entry.status())
+        return statuses
+
+    def stop(self) -> None:
+        """Cancel the session (``Stop-SPMTMigration``); tasks stay resumable."""
+        for entry in self._tasks:
+            entry.job.cancel()
+
+    def pause(self) -> None:
+        """Request a clean stop at the next batch boundary (resumable)."""
+        for entry in self._tasks:
+            entry.job.pause()
+
+    def unregister(self) -> None:
+        """Drop all tasks and release the session (``Unregister-SPMTMigration``)."""
+        self._tasks.clear()
+
+    # ── State ────────────────────────────────────────────────────
+
+    @property
+    def tasks(self) -> list[SessionTask]:
+        return self._tasks
+
+    @property
+    def jobs(self) -> list[MigrationJob]:
+        return [entry.job for entry in self._tasks]
+
+    def status(self) -> list[dict]:
+        """The per-task status (alias of :meth:`show`)."""
+        return self.show()
+
+    def verify(self, spot_checks: int = 20):
+        """Reconcile source vs target for every task after migration."""
+        return [entry.job.verify(spot_checks) for entry in self._tasks]

office365/migration/settings.py

@@ -0,0 +1,67 @@
+"""Migration settings — the session-level surface (SPMT ``Register-SPMTMigration``).
+
+Mirrors the parameters of ``Microsoft.SharePoint.MigrationTool.PowerShell``'s
+``Register-SPMTMigration`` cmdlet (snake-cased) and maps the subset the core
+enforces onto :class:`~office365.migration.base.MigrationOptions`. Settings that
+only apply to the SPMT Windows tool (``working_folder``, ``enable_multi_round``,
+...) are kept for parity/documentation and ignored by the core.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from office365.migration.base import MigrationOptions
+
+__all__ = ["MigrationSettings"]
+
+
+@dataclass
+class MigrationSettings:
+    """Session-level migration settings (the ``Register-SPMTMigration`` surface)."""
+
+    migration_type: str = "Content"  # Content | Workflow
+    preserve_permissions: bool = False
+    preserve_inheritance: bool = True
+    preserve_versions: bool = False
+    keep_all_versions: bool = True
+    number_of_versions: int = 10
+    include_hidden_files: bool = True
+    migrate_files_created_after: str | None = None
+    migrate_files_modified_after: str | None = None
+    skip_files_with_extension: list[str] = field(default_factory=list)
+    skip_sites_with_name: str | None = None
+    skip_lists_with_name: str | None = None
+    skip_content_types_with_name: str | None = None
+    replacement_of_invalid_char: str | None = None
+    duplicate_page_behavior: str = "RENAME"
+    automatic_user_mapping: bool = True
+    user_mapping_file: str | None = None
+    # use the server-side Migration API (full fidelity: versions/ACLs) instead of
+    # client-side REST uploads — the SPMT path
+    use_migration_api: bool = False
+    use_custom_azure_storage: bool = False
+    custom_azure_storage_account: str | None = None
+    custom_azure_access_key: str | None = None
+    enable_encryption: bool = True
+    delete_temp_files_when_migration_done: bool = False
+    working_folder: str | None = None
+    enable_multi_round: bool = False
+    scan_only: bool = False
+    migrate_all_site_fields_and_content_types: bool = False
+    migrate_site_settings: str = "PRESERVE_ALL_SETTINGS"
+    migrate_navigation: bool = True
+    migrate_term_groups: bool = True
+    lookup_reference_policy: int = 0
+
+    def to_options(self, **overrides) -> MigrationOptions:
+        """Build :class:`MigrationOptions` from the settings (``overrides`` win)."""
+        values = {
+            "preserve_permissions": self.preserve_permissions,
+            "preserve_versions": self.preserve_versions,
+            "exclude_patterns": [f"*.{extension.lstrip('.')}" for extension in self.skip_files_with_extension],
+            "created_after": self.migrate_files_created_after,
+            "modified_after": self.migrate_files_modified_after,
+        }
+        values.update(overrides)
+        return MigrationOptions(**values)

office365/migration/sharepoint/__init__.py

@@ -0,0 +1,55 @@
+"""SharePoint migration product: assessment + data adapters.
+
+from office365.migration.sharepoint import MigrationAssessor
+from office365.migration.sharepoint.adapters import SharePointLibraryTarget
+"""
+
+from office365.migration.sharepoint.adapters import (
+    SharePointLibrarySource,
+    SharePointLibraryTarget,
+    SharePointListSource,
+    SharePointListTarget,
+)
+from office365.migration.sharepoint.assessor import MigrationAssessor
+from office365.migration.sharepoint.options import SharePointAssessmentOptions
+from office365.migration.sharepoint.registry import (
+    SHAREPOINT_SCANS,
+    get_scan,
+    sharepoint_scan_pairs,
+)
+from office365.migration.sharepoint.scanners import (
+    AssessmentOptions,
+    BaseScanner,
+    FieldScanner,
+    FileScanner,
+    LargeSitesScanner,
+    PathScanner,
+    PermissionScanner,
+    ScanTarget,
+    SiteLockedScanner,
+    SiteScanSummary,
+)
+from office365.migration.sharepoint.tenant_assessor import MigrationTenantAssessor
+
+__all__ = [
+    "AssessmentOptions",
+    "BaseScanner",
+    "FieldScanner",
+    "FileScanner",
+    "LargeSitesScanner",
+    "MigrationAssessor",
+    "MigrationTenantAssessor",
+    "PathScanner",
+    "PermissionScanner",
+    "SHAREPOINT_SCANS",
+    "ScanTarget",
+    "SharePointAssessmentOptions",
+    "SharePointLibrarySource",
+    "SharePointLibraryTarget",
+    "SharePointListSource",
+    "SharePointListTarget",
+    "SiteLockedScanner",
+    "SiteScanSummary",
+    "get_scan",
+    "sharepoint_scan_pairs",
+]