Office365-REST-Python-Client 3.0.0__py3-none-any.whl → 3.2.0__py3-none-any.whl

office365/migration/adapters/filesystem.py

@@ -0,0 +1,177 @@
+"""Filesystem source/target adapters.
+
+A pure-stdlib adapter pair that copies a directory tree — the reference
+implementation and the primary storage for the migration toolkit — plus a
+:class:`JsonFileTarget` for persisting record payloads (one JSON file per item).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+
+from office365.migration.adapters import MigrationProgress
+from office365.migration.base import MigrationItem
+from office365.runtime.converters.json_file import record_to_json
+from office365.runtime.converters.scalars import iso
+from office365.runtime.operations import emit_progress
+
+
+class FileSystemSource:
+    """Enumerates and reads files under a root directory."""
+
+    def __init__(self, root: str | Path) -> None:
+        self._root = Path(root)
+
+    @property
+    def root(self) -> Path:
+        return self._root
+
+    def label(self) -> str:
+        return str(self._root)
+
+    def list_items(self, progress: MigrationProgress = None) -> list[MigrationItem]:
+        items: list[MigrationItem] = []
+        for path in sorted(self._root.rglob("*")):
+            if not path.is_file():
+                continue
+            stat = path.stat()
+            item = MigrationItem(
+                source_path=str(path),
+                dest_path=str(path.relative_to(self._root)).replace("\\", "/"),
+                size_bytes=stat.st_size,
+                modified=iso(datetime.fromtimestamp(stat.st_mtime, timezone.utc)),
+                created=iso(datetime.fromtimestamp(stat.st_ctime, timezone.utc)),
+            )
+            items.append(item)
+            emit_progress(progress, done=len(items), stage="planning", items=[item])
+        return items
+
+    def read(self, item: MigrationItem) -> bytes:
+        return Path(item.source_path).read_bytes()
+
+    def checksum(self, item: MigrationItem) -> str:
+        return _md5(Path(item.source_path))
+
+    def close(self) -> None:
+        pass
+
+
+class FileSystemTarget:
+    """Writes files under a root directory."""
+
+    def __init__(self, root: str | Path, include_folders: bool = False) -> None:
+        self._root = Path(root)
+        self._include_folders = include_folders  # list folder entries in list_paths (verify)
+
+    @property
+    def root(self) -> Path:
+        return self._root
+
+    def label(self) -> str:
+        return str(self._root)
+
+    def exists(self, item: MigrationItem) -> bool:
+        return (self._root / item.dest_path).exists()
+
+    def write(self, item: MigrationItem, payload: object) -> None:
+        dest = self._root / item.dest_path
+        if item.item_type == "folder":
+            dest.mkdir(parents=True, exist_ok=True)
+            return
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        dest.write_bytes(payload if isinstance(payload, bytes) else str(payload).encode("utf-8"))
+
+    def list_paths(self) -> list[str]:
+        paths = []
+        for p in self._root.rglob("*"):
+            rel = str(p.relative_to(self._root)).replace("\\", "/")
+            if p.is_dir():
+                if self._include_folders:
+                    paths.append(f"{rel}/")
+            else:
+                paths.append(rel)
+        return paths
+
+    def modified(self, item: MigrationItem) -> str:
+        """Last-modified of the target file, for incremental migration."""
+        return iso(datetime.fromtimestamp((self._root / item.dest_path).stat().st_mtime, timezone.utc))
+
+    def checksum(self, item: MigrationItem) -> str:
+        return _md5(self._root / item.dest_path)
+
+    def commit(self, options=None) -> None:
+        pass
+
+    def close(self) -> None:
+        pass
+
+
+class JsonFileSource:
+    """Reads record payloads (one JSON file per item) — the import counterpart
+    of :class:`JsonFileTarget`. Enables records round-trips through the filesystem.
+    """
+
+    def __init__(self, root: str | Path) -> None:
+        self._root = Path(root)
+
+    def label(self) -> str:
+        return str(self._root)
+
+    def list_items(self, progress: MigrationProgress = None) -> list[MigrationItem]:
+        items: list[MigrationItem] = []
+        for path in sorted(self._root.rglob("*.json")):
+            item = MigrationItem(
+                source_path=str(path),
+                dest_path=path.relative_to(self._root).with_suffix("").as_posix(),
+                item_type="record",
+            )
+            items.append(item)
+            emit_progress(progress, done=len(items), stage="planning", items=[item])
+        return items
+
+    def read(self, item: MigrationItem) -> object:
+        with open(item.source_path, "r", encoding="utf-8") as f:
+            return json.load(f)
+
+    def checksum(self, item: MigrationItem) -> str:
+        return _md5(Path(item.source_path))
+
+    def close(self) -> None:
+        pass
+
+
+class JsonFileTarget(FileSystemTarget):
+    """Persists record payloads (dicts) as one JSON file per item.
+
+    An interchange format for migrating tabular data (SharePoint lists,
+    PostgreSQL tables, ...) through the filesystem — a stepping stone for
+    S3/object-storage targets.
+    """
+
+    def _path(self, item: MigrationItem) -> Path:
+        return self._root / f"{item.dest_path}.json"
+
+    def exists(self, item: MigrationItem) -> bool:
+        return self._path(item).exists()
+
+    def write(self, item: MigrationItem, payload: object) -> None:
+        dest = self._path(item)
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        dest.write_text(record_to_json(payload), encoding="utf-8")
+
+    def list_paths(self) -> list[str]:
+        return [p.relative_to(self._root).with_suffix("").as_posix() for p in self._root.rglob("*.json")]
+
+    def checksum(self, item: MigrationItem) -> str:
+        return _md5(self._path(item))
+
+
+def _md5(path: Path) -> str:
+    digest = hashlib.md5()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()

office365/migration/assessment/__init__.py

@@ -0,0 +1,27 @@
+"""Pre-migration assessment core — SMAT-style modular scans, scoped to containers.
+
+Product-specific scans live in the product packages
+(``office365.migration.sharepoint``, ``office365.migration.outlook``).
+"""
+
+from office365.migration.assessment.containers import ScanContainer
+from office365.migration.assessment.registry import ScanDefinition, scan_pairs
+from office365.migration.assessment.report import AssessmentReport, ScanReport
+from office365.migration.assessment.runner import ScanRunner
+from office365.migration.assessment.scanners import (
+    AssessmentOptions,
+    BaseScanner,
+    ScanTarget,
+)
+
+__all__ = [
+    "AssessmentOptions",
+    "AssessmentReport",
+    "BaseScanner",
+    "ScanContainer",
+    "ScanDefinition",
+    "ScanReport",
+    "ScanRunner",
+    "ScanTarget",
+    "scan_pairs",
+]

office365/migration/assessment/containers.py

@@ -0,0 +1,24 @@
+from enum import Enum
+
+
+class ScanContainer(Enum):
+    """Where a scan runs in the product object tree — the dispatch + report container.
+
+    Follows the container hierarchy (``TENANT -> SITE -> WEB -> LIST`` with
+    ``FIELDS`` / ``ITEMS`` / ``FILES`` as list sub-resources) for SharePoint;
+    Outlook adds ``MAILBOX -> MAIL_FOLDER``. Scans declare the container(s)
+    they consume; the per-product assessor walker loads each container's data
+    once and dispatches to the matching scans.
+    """
+
+    TENANT = "tenant"  # reserved — tenant assessor (site enumeration)
+    SITE = "site"  # the site collection
+    WEB = "web"  # subsite
+    LIST = "list"  # list / library container
+    FIELDS = "fields"  # a list's column schema
+    ITEMS = "items"  # list items
+    FILES = "files"  # list files / folders
+
+    # Outlook (mail)
+    MAILBOX = "mailbox"  # the mailbox root
+    MAIL_FOLDER = "mail_folder"  # a mail folder

office365/migration/assessment/export.py

@@ -0,0 +1,51 @@
+"""Assessment report export — write the issues and per-scan detail reports.
+
+Uses the report's records form (:meth:`AssessmentReport.to_records`) and the
+per-scan detail records (:attr:`AssessmentReport.scan_reports`), so the report
+model stays format-agnostic while the caller gets the downloadable
+assessment. Scan detail files land in a ``ScannerReports`` subfolder, mirroring
+SMAT's output layout (``<ScanName>-detail.csv``).
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from office365.migration.assessment.report import AssessmentReport
+from office365.migration.report_io import write_formats
+
+_COLUMNS = ["severity", "category", "location", "message", "suggestion"]
+
+
+def export_assessment(report: AssessmentReport, output_dir: str | Path) -> list[str]:
+    """Write the aggregate issues plus each scan's detail report.
+
+    Args:
+        report: The assessment result.
+        output_dir: Directory to write the files into.
+
+    Returns:
+        List of written file paths.
+    """
+    written = write_formats(report.to_records(), output_dir, "AssessmentReport", columns=_COLUMNS)
+    written += export_scan_reports(report, output_dir)
+    return written
+
+
+def export_scan_reports(report: AssessmentReport, output_dir: str | Path) -> list[str]:
+    """Write each scan's SMAT-style detail report (``ScannerReports/<Scan>-detail``).
+
+    Only scans that produced rows are written. Returns the written paths.
+    """
+    written: list[str] = []
+    for scan in report.scan_reports.values():
+        if not scan.records:
+            continue
+        written += write_formats(
+            scan.to_records(),
+            os.path.join(output_dir, "ScannerReports"),
+            f"{scan.name}-detail",
+            columns=list(scan.columns),
+        )
+    return written

office365/migration/assessment/issue.py

@@ -1,4 +1,9 @@
+from __future__ import annotations
+
 from dataclasses import dataclass
+from typing import Optional
+
+from office365.runtime.limits import Limit
 
 
 @dataclass
@@ -8,3 +13,17 @@ class AssessmentIssue:
     location: str  # list/folder/field path
     message: str
     suggestion: str = ""
+    risk_code: str = ""  # product-specific scan-assessment code (SPMT), when applicable
+    limit: Optional[Limit] = None  # the authoritative (product-agnostic) limit, when applicable
+
+
+#: SPMT scan-assessment risk codes the library can emit (a subset — the ones tied
+#: to SharePoint service limits). See
+#: https://learn.microsoft.com/en-us/sharepointmigration/spmt-scan-risk-codes
+RISK_CODES: dict[str, str] = {
+    "LIST_VIEW_EXCEED_LIMIT": "The list view shows more items than the list view threshold.",
+    "ITEM_COUNT_EXCEED_INDEX_LIMIT": "Item count is too large to create a column index.",
+    "ITEM_COUNT_EXCEED_LIMIT": "Item count exceeds the per-list maximum.",
+    "LIST_VIEW_LOOKUP_EXCEED_LIMIT": "Lookup/people/managed-metadata columns exceed the list view lookup threshold.",
+    "UNIQUE_PERMISSION_EXCEED_LIMIT": "Unique permissions per list exceed the supported/recommended limit.",
+}

office365/migration/assessment/registry.py

@@ -0,0 +1,60 @@
+"""Scan registry — the ScanDef.json analog, keyed by container.
+
+Core declares the :class:`ScanDefinition` shape; each product owns its scan
+list and helpers (e.g. ``office365.migration.sharepoint.registry``). Each entry
+mirrors SMAT's ``ScanDef.json`` shape:
+``{Name, Scanner, ReportCategoryType(=container), Enabled}``.
+Disabling a scan (``disabled_scans``) skips it and stops the walker collecting
+its container's data.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from office365.migration.assessment.containers import ScanContainer
+from office365.migration.assessment.scanners.base import AssessmentOptions, BaseScanner
+
+
+@dataclass
+class ScanDefinition:
+    """One scan: name, implementation, and the container it consumes.
+
+    ``tenant_only`` marks scans that need the tenant site-property bag (e.g.
+    LockedSites reads ``SiteProperties.LockState``) and therefore run in the
+    TENANT walker, not the single-site walker.
+    """
+
+    name: str
+    scanner: type[BaseScanner[Any]]
+    container: ScanContainer
+    enabled: bool = True
+    tenant_only: bool = False
+
+
+def scan_pairs(
+    scans: list[ScanDefinition],
+    options: AssessmentOptions | None = None,
+    tenant_scope: bool = False,
+    container: ScanContainer | None = None,
+) -> list[tuple[ScanDefinition, BaseScanner[Any]]]:
+    """The enabled ``(definition, scanner)`` pairs from ``scans``, in registry order.
+
+    Args:
+        scans: The product's scan definitions.
+        options: Assessment options (``disabled_scans`` applied).
+        tenant_scope: Include ``tenant_only`` scans (the TENANT walker).
+        container: Restrict to scans consuming this container.
+    """
+    options = options or AssessmentOptions()
+    pairs: list[tuple[ScanDefinition, BaseScanner[Any]]] = []
+    for definition in scans:
+        if not (definition.enabled and definition.name not in options.disabled_scans):
+            continue
+        if definition.tenant_only and not tenant_scope:
+            continue
+        if container is not None and definition.container is not container:
+            continue
+        pairs.append((definition, definition.scanner(options)))
+    return pairs

office365/migration/assessment/report.py

@@ -1,11 +1,62 @@
 from __future__ import annotations
 
+import csv
+import io
+import json
+import uuid
 from dataclasses import dataclass, field
-from pathlib import Path
+from datetime import datetime
+from typing import TYPE_CHECKING, Any, Callable, Generic, TypeVar, cast
 
+from office365.migration.assessment.containers import ScanContainer
 from office365.migration.assessment.issue import AssessmentIssue
 from office365.runtime.client_value import ClientValue
 
+if TYPE_CHECKING:
+    from office365.migration.assessment.scanners.base import BaseScanner
+
+RecordT = TypeVar("RecordT")
+
+
+@dataclass
+class ScanReport(Generic[RecordT]):
+    """SMAT-style detail report produced by one scan (``<Scan>-detail.csv``).
+
+    ``records`` are typed values (each scan declares its own record type, whose
+    dataclass fields are the SMAT column headers), so the neutral records form
+    and the CSV/JSON projections are trivial.
+    """
+
+    name: str
+    container: ScanContainer
+    columns: tuple[str, ...]
+    records: list[RecordT]
+
+    def to_records(self) -> list[dict]:
+        """Project rows to plain dicts keyed by the SMAT columns (``None`` -> ``n/a``)."""
+        rows = []
+        for record in self.records:
+            row = {}
+            for column in self.columns:
+                value = getattr(record, column, None)
+                if isinstance(value, datetime):
+                    value = value.isoformat()
+                row[column] = value if value is not None else "n/a"
+            rows.append(row)
+        return rows
+
+    def to_json(self) -> str:
+        """Compact JSON of the report rows."""
+        return json.dumps(self.to_records(), indent=2)
+
+    def to_csv(self) -> str:
+        """CSV of the report rows, header from ``columns``."""
+        buffer = io.StringIO()
+        writer = csv.DictWriter(buffer, fieldnames=list(self.columns), restval="n/a")
+        writer.writeheader()
+        writer.writerows(self.to_records())
+        return buffer.getvalue()
+
 
 @dataclass
 class AssessmentReport(ClientValue):
@@ -13,54 +64,125 @@ class AssessmentReport(ClientValue):
 
     # Inventory
     total_lists: int = 0
+    total_webs: int = 0
     total_files: int = 0
     total_size_gb: float = 0.0
 
+    # Scans that could not run (e.g. access denied) — counts are not meaningful
+    lists_skipped: bool = False
+    webs_skipped: bool = False
+
+    # SMAT ScanID — unique identifier of this assessment run
+    scan_id: str = ""
+
     # Issues
     issues: list[AssessmentIssue] = field(default_factory=list)
 
+    # Per-scan SMAT-style detail reports, keyed by scan name (finalized on read)
+    _scan_reports: dict[str, ScanReport[Any]] = field(default_factory=dict, init=False, repr=False)
+
+    # Lazy finalize — scans assemble their detail rows once the deferred batch
+    # has settled (post ``execute_query``); the first consumer triggers it.
+    _finalizer: Callable[[], None] | None = field(default=None, init=False, repr=False)
+    _finalized: bool = field(default=False, init=False, repr=False)
+
+    @classmethod
+    def new(cls) -> "AssessmentReport":
+        """A fresh report with a unique SMAT ScanID."""
+        return cls(scan_id=str(uuid.uuid4()))
+
+    def attach_finalizer(self, fn: Callable[[], None]) -> None:
+        """Register the one-time hook that assembles per-scan detail reports."""
+        self._finalizer = fn
+
+    def _ensure_finalized(self) -> None:
+        if not self._finalized:
+            if self._finalizer is not None:
+                self._finalizer()
+            self._finalized = True
+
+    def add_scan_report(self, scanner: "BaseScanner[Any]", container: ScanContainer) -> None:
+        """Store a scan's typed detail rows (no-op when the scan produced none)."""
+        if not scanner.records:
+            return
+        self._scan_reports[scanner.scan_name] = ScanReport(
+            name=scanner.scan_name,
+            container=container,
+            columns=scanner.columns,
+            records=scanner.records,
+        )
+
+    def add_access_issue(self, location: str, error: Exception) -> None:
+        """Record a warning for an area that could not be read (skipped, not fatal)."""
+        self.issues.append(AssessmentIssue("warning", "access", location, f"skipped — {error}"))
+
+    @property
+    def scan_reports(self) -> dict[str, ScanReport[Any]]:
+        self._ensure_finalized()
+        return self._scan_reports
+
+    def scan_report(self, scanner: type["BaseScanner[RecordT]"]) -> ScanReport[RecordT]:
+        """Typed access to a scan's detail report by its scanner class."""
+        return cast(ScanReport[RecordT], self.scan_reports[scanner.scan_name])
+
     @property
     def blockers(self) -> list[AssessmentIssue]:
+        self._ensure_finalized()
         return [i for i in self.issues if i.severity == "blocker"]
 
     @property
     def warnings(self) -> list[AssessmentIssue]:
+        self._ensure_finalized()
         return [i for i in self.issues if i.severity == "warning"]
 
+    @property
+    def info(self) -> list[AssessmentIssue]:
+        self._ensure_finalized()
+        return [i for i in self.issues if i.severity == "info"]
+
     @property
     def is_ready(self) -> bool:
         return len(self.blockers) == 0
 
     def summary(self) -> str:
-        lines = [
-            f"Lists: {self.total_lists} | Files: {self.total_files} | Size: {self.total_size_gb:.2f}GB",
-            f"Blockers: {len(self.blockers)} | Warnings: {len(self.warnings)}",
-        ]
-        if self.blockers:
-            lines.append("\nBLOCKERS (must fix before migration):")
-            for b in self.blockers:
-                lines.append(f"  ✗ [{b.category}] {b.location}: {b.message}")
-                if b.suggestion:
-                    lines.append(f"    → {b.suggestion}")
-        return "\n".join(lines)
-
-    def to_dataframe(self):
-        try:
-            import pandas as pd
-        except ImportError:
-            raise ImportError("pip install office365-rest-python-client[pandas]") from None
-        return pd.DataFrame(
-            [
-                {
-                    "severity": i.severity,
-                    "category": i.category,
-                    "location": i.location,
-                    "message": i.message,
-                    "suggestion": i.suggestion,
-                }
-                for i in self.issues
-            ]
+        """A concise one-liner — the detailed issue list is the caller's rendering."""
+        self._ensure_finalized()
+        webs = "n/a" if self.webs_skipped else self.total_webs
+        lists = "n/a" if self.lists_skipped else self.total_lists
+        status = "ready" if self.is_ready else "blocked"
+        scans = " | ".join(f"{name}: {len(sr.records)}" for name, sr in sorted(self.scan_reports.items()))
+        line = (
+            f"Webs: {webs} | Lists: {lists} | Files: {self.total_files} | "
+            f"Size: {self.total_size_gb:.2f}GB | Blockers: {len(self.blockers)} | "
+            f"Warnings: {len(self.warnings)} | {status}"
         )
+        return f"{line}\nScans: {scans}" if scans else line
+
+    def to_records(self) -> list[dict]:
+        """Project the issues into plain records — the pipeline's neutral form.
 
-    def to_excel(self, path: str | Path) -> None:
-        self.to_dataframe().to_excel(path, index=False)
+        Useful for exporting the assessment (CSV/JSON) without coupling the
+        report model to a specific format.
+        """
+        self._ensure_finalized()
+        return [
+            {
+                "severity": issue.severity,
+                "category": issue.category,
+                "location": issue.location,
+                "message": issue.message,
+                "suggestion": issue.suggestion,
+                "risk_code": issue.risk_code,
+            }
+            for issue in self.issues
+        ]
+
+    @property
+    def by_risk_code(self) -> dict[str, int]:
+        """Issue counts per SPMT risk code (issues without a code are excluded)."""
+        self._ensure_finalized()
+        counts: dict[str, int] = {}
+        for issue in self.issues:
+            if issue.risk_code:
+                counts[issue.risk_code] = counts.get(issue.risk_code, 0) + 1
+        return counts

office365/migration/assessment/runner.py

@@ -0,0 +1,57 @@
+"""Scan runner — the shared report/dispatch engine behind every assessor.
+
+An assessor only walks its product's object tree and hands loaded payloads to
+the runner; the runner owns the :class:`AssessmentReport`, filters the enabled
+scans per container, flags access failures, and assembles the per-scan detail
+reports once the walk settles. This keeps the walkers (deferred SharePoint,
+eager Outlook) thin and their report behavior identical.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from office365.migration.assessment.containers import ScanContainer
+from office365.migration.assessment.registry import ScanDefinition
+from office365.migration.assessment.report import AssessmentReport
+from office365.migration.assessment.scanners.base import BaseScanner, ScanTarget
+
+
+class ScanRunner:
+    """Holds the report + enabled ``(definition, scanner)`` pairs."""
+
+    def __init__(
+        self,
+        pairs: list[tuple[ScanDefinition, BaseScanner[Any]]],
+        report: AssessmentReport | None = None,
+    ) -> None:
+        self.pairs = pairs
+        self.report = report or AssessmentReport.new()
+
+    def scanners(self, container: ScanContainer, items_load: str | None = None) -> list[BaseScanner[Any]]:
+        """Enabled scans for a container (optionally a specific items projection)."""
+        return [
+            scanner
+            for definition, scanner in self.pairs
+            if definition.container is container and (items_load is None or scanner.items_load == items_load)
+        ]
+
+    def dispatch(self, container: ScanContainer, entity: Any, location: str = "", items_load: str | None = None) -> None:
+        """Run the matching scans over a loaded payload."""
+        target = ScanTarget(container, entity, location)
+        for scanner in self.scanners(container, items_load):
+            scanner.run(target, self.report)
+
+    def run_site_scans(self, summary: Any) -> None:
+        """Run the SITE-container scans over the walker-aggregated summary."""
+        self.dispatch(ScanContainer.SITE, summary, getattr(summary, "site_url", "") or "")
+
+    def flag_access(self, location: str, error: Exception) -> None:
+        """Record a skipped area as an ``access`` warning."""
+        self.report.add_access_issue(location, error)
+
+    def collect(self) -> None:
+        """Finalize every scan and store the ones that produced rows."""
+        for definition, scanner in self.pairs:
+            scanner.finalize(self.report)
+            self.report.add_scan_report(scanner, definition.container)

office365/migration/assessment/scanners/__init__.py

@@ -0,0 +1,13 @@
+"""Assessment scanner base — options, the shared flag helper, and the run contract."""
+
+from office365.migration.assessment.scanners.base import (
+    AssessmentOptions,
+    BaseScanner,
+    ScanTarget,
+)
+
+__all__ = [
+    "AssessmentOptions",
+    "BaseScanner",
+    "ScanTarget",
+]