qbvisor 0.3.0__py3-none-any.whl

qbvisor/_backup/reader.py

@@ -0,0 +1,263 @@
+"""Open, verify, and analyze completed application backups."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+from ..backup import (
+    ApplicationBackup,
+    BackupArtifact,
+    BackupManifest,
+    BackupTable,
+    BackupVerification,
+)
+from ..exceptions import BackupIntegrityError
+
+
+def open_backup(path: str | Path) -> ApplicationBackup:
+    root = Path(path).expanduser().resolve()
+    manifest_path = root / "manifest.json"
+    try:
+        payload = json.loads(manifest_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as error:
+        raise ValueError(f"Could not read backup manifest at {manifest_path}") from error
+    if not isinstance(payload, dict):
+        raise ValueError("backup manifest must contain a JSON object")
+    return ApplicationBackup(path=root, manifest=BackupManifest.from_dict(payload))
+
+
+def _hash_file(path: Path) -> tuple[str, int]:
+    digest = hashlib.sha256()
+    size = 0
+    with path.open("rb") as stream:
+        while chunk := stream.read(1024 * 1024):
+            digest.update(chunk)
+            size += len(chunk)
+    return digest.hexdigest(), size
+
+
+def _json_line_count(path: Path, issues: list[str]) -> int:
+    count = 0
+    with path.open(encoding="utf-8") as stream:
+        for line_number, line in enumerate(stream, start=1):
+            try:
+                payload = json.loads(line)
+            except json.JSONDecodeError:
+                issues.append(f"{path.name} contains invalid JSON at line {line_number}")
+                continue
+            if not isinstance(payload, dict):
+                issues.append(f"{path.name} contains a non-object at line {line_number}")
+            count += 1
+    return count
+
+
+def _json_item_count(path: Path, artifact: BackupArtifact, issues: list[str]) -> int | None:
+    if artifact.kind in {"records", "attachment-index"}:
+        return _json_line_count(path, issues)
+    if artifact.item_count is None:
+        return None
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        issues.append(f"{artifact.path} is not valid JSON")
+        return None
+    if artifact.kind == "reports" and isinstance(payload, dict):
+        payload = payload.get("details")
+    if not isinstance(payload, list):
+        issues.append(f"{artifact.path} does not contain the expected array")
+        return None
+    return len(payload)
+
+
+def _verify_table_counts(backup: ApplicationBackup, issues: list[str]) -> None:
+    artifacts = {artifact.path: artifact for artifact in backup.manifest.artifacts}
+    for table in backup.manifest.tables:
+        table_artifacts = [artifacts[path] for path in table.artifacts]
+        record_artifacts = [artifact for artifact in table_artifacts if artifact.kind == "records"]
+        index_artifacts = [
+            artifact for artifact in table_artifacts if artifact.kind == "attachment-index"
+        ]
+        if len(record_artifacts) != 1 or record_artifacts[0].item_count != table.record_count:
+            issues.append(f"table {table.id} record count does not match its records artifact")
+        if len(index_artifacts) != 1 or index_artifacts[0].item_count != table.attachment_count:
+            issues.append(f"table {table.id} attachment count does not match its index")
+
+
+def _verify_attachment_indexes(backup: ApplicationBackup, issues: list[str]) -> None:
+    attachments = {
+        artifact.path: artifact
+        for artifact in backup.manifest.artifacts
+        if artifact.kind == "attachment"
+    }
+    indexed_paths: set[str] = set()
+    for artifact in backup.manifest.artifacts:
+        if artifact.kind != "attachment-index":
+            continue
+        path = backup.path / artifact.path
+        if not path.is_file():
+            continue
+        table_prefix = artifact.path.removesuffix("attachments.jsonl") + "attachments/"
+        try:
+            with path.open(encoding="utf-8") as stream:
+                for line_number, line in enumerate(stream, start=1):
+                    try:
+                        entry = json.loads(line)
+                    except json.JSONDecodeError:
+                        continue
+                    if not isinstance(entry, dict):
+                        continue
+                    referenced_path = entry.get("path")
+                    if not isinstance(referenced_path, str):
+                        issues.append(
+                            f"{artifact.path} line {line_number} has an invalid attachment path"
+                        )
+                        continue
+                    if not referenced_path.startswith(table_prefix):
+                        issues.append(
+                            f"{artifact.path} line {line_number} references another table"
+                        )
+                        continue
+                    referenced = attachments.get(referenced_path)
+                    if referenced is None:
+                        issues.append(
+                            f"{artifact.path} line {line_number} references a missing attachment"
+                        )
+                        continue
+                    if referenced_path in indexed_paths:
+                        issues.append(f"attachment is indexed more than once: {referenced_path}")
+                    indexed_paths.add(referenced_path)
+                    if (
+                        entry.get("sha256") != referenced.sha256
+                        or entry.get("bytes") != referenced.bytes
+                    ):
+                        issues.append(
+                            f"{artifact.path} line {line_number} has incorrect attachment integrity"
+                        )
+        except (OSError, UnicodeDecodeError):
+            # The general artifact pass already reports unreadable index content.
+            continue
+    missing = set(attachments) - indexed_paths
+    if missing:
+        issues.append(f"attachment artifacts are not indexed: {', '.join(sorted(missing))}")
+
+
+def verify_backup(backup: ApplicationBackup) -> BackupVerification:
+    root = backup.path
+    issues: list[str] = []
+    if not root.is_dir() or root.is_symlink():
+        raise BackupIntegrityError((f"backup root is not a regular directory: {root}",))
+    try:
+        disk_manifest = open_backup(root).manifest
+    except ValueError as error:
+        issues.append(str(error))
+    else:
+        if disk_manifest != backup.manifest:
+            issues.append("manifest.json changed after this backup was opened")
+
+    expected_paths = {artifact.path for artifact in backup.manifest.artifacts} | {"manifest.json"}
+    actual_paths: set[str] = set()
+    for path in root.rglob("*"):
+        relative = path.relative_to(root).as_posix()
+        if path.is_symlink():
+            issues.append(f"backup contains a symbolic link: {relative}")
+        elif path.is_file():
+            actual_paths.add(relative)
+    missing = expected_paths - actual_paths
+    unexpected = actual_paths - expected_paths
+    if missing:
+        issues.append(f"backup files are missing: {', '.join(sorted(missing))}")
+    if unexpected:
+        issues.append(f"backup contains untracked files: {', '.join(sorted(unexpected))}")
+
+    total_bytes = 0
+    for artifact in backup.manifest.artifacts:
+        path = root / artifact.path
+        if not path.is_file() or path.is_symlink():
+            continue
+        try:
+            digest, size = _hash_file(path)
+        except OSError as error:
+            issues.append(f"artifact could not be read: {artifact.path} ({error})")
+            continue
+        total_bytes += size
+        if digest != artifact.sha256:
+            issues.append(f"artifact hash does not match: {artifact.path}")
+        if size != artifact.bytes:
+            issues.append(f"artifact byte count does not match: {artifact.path}")
+        try:
+            item_count = _json_item_count(path, artifact, issues)
+        except (OSError, UnicodeDecodeError) as error:
+            issues.append(f"artifact content could not be read: {artifact.path} ({error})")
+            continue
+        if item_count is not None and item_count != artifact.item_count:
+            issues.append(f"artifact item count does not match: {artifact.path}")
+
+    _verify_table_counts(backup, issues)
+    _verify_attachment_indexes(backup, issues)
+    if issues:
+        raise BackupIntegrityError(tuple(issues))
+    return BackupVerification(
+        artifact_count=len(backup.manifest.artifacts),
+        total_bytes=total_bytes,
+    )
+
+
+def _resolve_table(backup: ApplicationBackup, value: str) -> BackupTable:
+    matches = [
+        table
+        for table in backup.manifest.tables
+        if table.id == value or table.name.casefold() == value.casefold()
+    ]
+    if not matches:
+        available = ", ".join(table.name for table in backup.manifest.tables)
+        raise KeyError(f"Table {value!r} is not in this backup. Available: {available}")
+    if len(matches) > 1:
+        raise KeyError(f"Table name {value!r} is ambiguous; use a table ID")
+    return matches[0]
+
+
+def table_dataframe(backup: ApplicationBackup, table: str) -> pd.DataFrame:
+    selected = _resolve_table(backup, table)
+    fields_path = backup.path / f"tables/{selected.id}/fields.json"
+    records_path = backup.path / f"tables/{selected.id}/records.jsonl"
+    fields = json.loads(fields_path.read_text(encoding="utf-8"))
+    if not isinstance(fields, list) or not all(isinstance(field, dict) for field in fields):
+        raise ValueError(f"Captured fields for table {selected.id} are invalid")
+    columns: list[tuple[str, str]] = []
+    for field in fields:
+        field_id = field.get("id")
+        label = field.get("label")
+        if (
+            not isinstance(field_id, int)
+            or isinstance(field_id, bool)
+            or not isinstance(label, str)
+        ):
+            raise ValueError(f"Captured field metadata for table {selected.id} is invalid")
+        columns.append((str(field_id), label))
+    labels = [label for _, label in columns]
+    if len(set(labels)) != len(labels):
+        raise ValueError(f"Captured field labels for table {selected.id} are not unique")
+
+    rows: list[dict[str, Any]] = []
+    known_ids = {field_id for field_id, _ in columns}
+    with records_path.open(encoding="utf-8") as stream:
+        for line_number, line in enumerate(stream, start=1):
+            record = json.loads(line)
+            if not isinstance(record, dict):
+                raise ValueError(f"Captured record at line {line_number} is not an object")
+            unknown = set(record) - known_ids
+            if unknown:
+                raise ValueError(
+                    f"Captured record at line {line_number} contains unknown fields: {unknown}"
+                )
+            row: dict[str, Any] = {}
+            for field_id, label in columns:
+                cell = record.get(field_id)
+                row[label] = cell.get("value") if isinstance(cell, dict) else None
+            rows.append(row)
+    return pd.DataFrame.from_records(rows, columns=labels)

qbvisor/_backup/records.py

@@ -0,0 +1,83 @@
+"""Stream complete table records into deterministic JSON Lines artifacts."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from dataclasses import dataclass
+from typing import Any
+
+from .._records.pagination import (
+    RECORD_ID_FIELD_ID,
+    RecordQueryClient,
+    iter_record_pages_by_id,
+)
+from .schema import CapturedSchema, CapturedTable
+from .workspace import BackupWorkspace
+
+RecordClient = RecordQueryClient
+
+
+@dataclass(frozen=True, slots=True)
+class CapturedRecordTable:
+    id: str
+    record_count: int
+    artifact: str
+
+
+@dataclass(frozen=True, slots=True)
+class CapturedRecords:
+    tables: tuple[CapturedRecordTable, ...]
+
+
+def _field_ids(table: CapturedTable) -> tuple[int, ...]:
+    field_ids: list[int] = []
+    for field in table.fields:
+        field_id = field.get("id")
+        if not isinstance(field_id, int) or isinstance(field_id, bool):
+            raise ValueError(f"Quickbase field in table {table.id} is missing a valid id")
+        field_ids.append(field_id)
+    if len(set(field_ids)) != len(field_ids):
+        raise ValueError(f"Quickbase returned duplicate field IDs for table {table.id}")
+    if RECORD_ID_FIELD_ID not in field_ids:
+        raise ValueError(f"Quickbase table {table.id} does not include Record ID# field 3")
+    return tuple(field_ids)
+
+
+def _iter_table_records(
+    client: RecordClient,
+    table: CapturedTable,
+    page_size: int,
+) -> Iterator[dict[str, Any]]:
+    select_fields = _field_ids(table)
+    for page in iter_record_pages_by_id(
+        client,
+        table.id,
+        select_fields=select_fields,
+        page_size=page_size,
+    ):
+        yield from page
+
+
+def capture_records(
+    client: RecordClient,
+    schema: CapturedSchema,
+    workspace: BackupWorkspace,
+    *,
+    page_size: int,
+) -> CapturedRecords:
+    """Stream all fields and records for every captured table."""
+    captured: list[CapturedRecordTable] = []
+    for table in schema.tables:
+        artifact = workspace.write_json_lines(
+            f"tables/{table.id}/records.jsonl",
+            "records",
+            _iter_table_records(client, table, page_size),
+        )
+        captured.append(
+            CapturedRecordTable(
+                id=table.id,
+                record_count=artifact.item_count or 0,
+                artifact=artifact.path,
+            )
+        )
+    return CapturedRecords(tables=tuple(captured))

qbvisor/_backup/schema.py

@@ -0,0 +1,143 @@
+"""Capture application and table schema without reshaping Quickbase metadata."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+from .workspace import BackupWorkspace
+
+
+class SchemaClient(Protocol):
+    def _ids(self, app_name: str, table_name: None = None) -> tuple[str, None]: ...
+
+    def get_app(self, app_name: str) -> dict[str, Any]: ...
+
+    def get_app_events(self, app_name: str) -> list[dict[str, Any]]: ...
+
+    def get_app_roles(self, app_name: str) -> list[dict[str, Any]]: ...
+
+    def get_tables_for_app(self, app_name: str) -> list[dict[str, Any]]: ...
+
+    def get_table(self, app_name: str, table_name: str) -> dict[str, Any]: ...
+
+    def get_fields_for_table(self, app_name: str, table_name: str) -> list[dict[str, Any]]: ...
+
+    def get_all_relationships(self, app_name: str, table_name: str) -> list[dict[str, Any]]: ...
+
+    def get_reports_for_table(self, app_name: str, table_name: str) -> list[dict[str, Any]]: ...
+
+    def get_report(self, app_name: str, table_name: str, report_id: int) -> dict[str, Any]: ...
+
+
+@dataclass(frozen=True, slots=True)
+class CapturedTable:
+    id: str
+    name: str
+    fields: tuple[dict[str, Any], ...]
+    artifacts: tuple[str, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class CapturedSchema:
+    app_id: str
+    app_name: str
+    tables: tuple[CapturedTable, ...]
+    artifacts: tuple[str, ...]
+
+
+def _required_identifier(payload: dict[str, Any], key: str, resource: str) -> str:
+    value = payload.get(key)
+    if not isinstance(value, str) or not value:
+        raise ValueError(f"Quickbase {resource} is missing a valid {key}")
+    return value
+
+
+def _report_id(summary: dict[str, Any], table_id: str) -> int:
+    value = summary.get("id")
+    if isinstance(value, int) and not isinstance(value, bool):
+        return value
+    if isinstance(value, str) and value.isdigit():
+        return int(value)
+    raise ValueError(f"Quickbase report in table {table_id} is missing a valid id")
+
+
+def capture_schema(
+    client: SchemaClient,
+    app_name: str,
+    workspace: BackupWorkspace,
+) -> CapturedSchema:
+    """Capture raw application metadata and every table's schema artifacts."""
+    app_id, _ = client._ids(app_name)
+    app = client.get_app(app_name)
+    response_app_id = app.get("id")
+    if response_app_id is not None and response_app_id != app_id:
+        raise ValueError(f"Quickbase returned app {response_app_id!r} while backing up {app_id!r}")
+    resolved_app_name = _required_identifier(app, "name", "application")
+    events = client.get_app_events(app_name)
+    roles = client.get_app_roles(app_name)
+    table_summaries = client.get_tables_for_app(app_name)
+
+    artifact_paths = [
+        workspace.write_json("app.json", "application", app).path,
+        workspace.write_json("events.json", "events", events, item_count=len(events)).path,
+        workspace.write_json("roles.json", "roles", roles, item_count=len(roles)).path,
+    ]
+    captured_tables: list[CapturedTable] = []
+
+    for summary in table_summaries:
+        table_id = _required_identifier(summary, "id", "table summary")
+        table_name = _required_identifier(summary, "name", f"table {table_id}")
+        details = client.get_table(app_name, table_id)
+        response_table_id = details.get("id")
+        if response_table_id is not None and response_table_id != table_id:
+            raise ValueError(
+                f"Quickbase returned table {response_table_id!r} while backing up {table_id!r}"
+            )
+        fields = client.get_fields_for_table(app_name, table_id)
+        relationships = client.get_all_relationships(app_name, table_id)
+        report_summaries = client.get_reports_for_table(app_name, table_id)
+        reports = [
+            client.get_report(app_name, table_id, _report_id(report, table_id))
+            for report in report_summaries
+        ]
+
+        prefix = f"tables/{table_id}"
+        table_artifacts = (
+            workspace.write_json(
+                f"{prefix}/table.json",
+                "table",
+                {"summary": summary, "details": details},
+            ).path,
+            workspace.write_json(
+                f"{prefix}/fields.json", "fields", fields, item_count=len(fields)
+            ).path,
+            workspace.write_json(
+                f"{prefix}/relationships.json",
+                "relationships",
+                relationships,
+                item_count=len(relationships),
+            ).path,
+            workspace.write_json(
+                f"{prefix}/reports.json",
+                "reports",
+                {"summaries": report_summaries, "details": reports},
+                item_count=len(reports),
+            ).path,
+        )
+        artifact_paths.extend(table_artifacts)
+        captured_tables.append(
+            CapturedTable(
+                id=table_id,
+                name=table_name,
+                fields=tuple(fields),
+                artifacts=table_artifacts,
+            )
+        )
+
+    return CapturedSchema(
+        app_id=app_id,
+        app_name=resolved_app_name,
+        tables=tuple(captured_tables),
+        artifacts=tuple(artifact_paths),
+    )

qbvisor/_backup/workflow.py

@@ -0,0 +1,184 @@
+"""Atomic orchestration for complete application backups."""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+from datetime import UTC, datetime
+from importlib.metadata import PackageNotFoundError, version
+from pathlib import Path
+from typing import Any, Protocol
+from uuid import uuid4
+
+from ..backup import ApplicationBackup, BackupManifest, BackupOptions, BackupTable
+from ..exceptions import BackupConsistencyError, QuickbaseResponseError
+from ..transport import QuickBaseTransport
+from .attachments import AttachmentClient, capture_attachments
+from .records import RecordClient, capture_records
+from .schema import SchemaClient, capture_schema
+from .workspace import BackupWorkspace
+
+
+class BackupClient(SchemaClient, RecordClient, AttachmentClient, Protocol):
+    transport: QuickBaseTransport
+
+    def records_modified_since(
+        self,
+        app_name: str,
+        table_name: str,
+        after: datetime | str,
+        *,
+        field_list: Any = None,
+        include_details: bool = False,
+    ) -> dict[str, Any]: ...
+
+
+def _timestamp(value: datetime) -> str:
+    return value.astimezone(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
+
+
+def _package_version() -> str:
+    try:
+        return version("qbvisor")
+    except PackageNotFoundError:
+        return "0+unknown"
+
+
+def _changed_tables(
+    client: BackupClient,
+    app_name: str,
+    table_ids: tuple[str, ...],
+    started_at: str,
+) -> tuple[str, ...]:
+    changed: list[str] = []
+    for table_id in table_ids:
+        response = client.records_modified_since(app_name, table_id, started_at)
+        count = response.get("count")
+        if not isinstance(count, int) or isinstance(count, bool) or count < 0:
+            raise QuickbaseResponseError(
+                "POST",
+                "records/modifiedSince",
+                expected="non-negative integer count",
+                actual=type(count).__name__,
+            )
+        if count:
+            changed.append(table_id)
+    return tuple(changed)
+
+
+def _write_manifest(root: Path, manifest: BackupManifest) -> None:
+    destination = root / "manifest.json"
+    content = (
+        json.dumps(
+            manifest.to_dict(),
+            ensure_ascii=False,
+            indent=2,
+            sort_keys=True,
+            allow_nan=False,
+        )
+        + "\n"
+    ).encode("utf-8")
+    temporary = destination.with_name(f".{destination.name}.{uuid4().hex}.tmp")
+    try:
+        temporary.write_bytes(content)
+        os.replace(temporary, destination)
+    finally:
+        temporary.unlink(missing_ok=True)
+
+
+def create_application_backup(
+    client: BackupClient,
+    app_name: str,
+    output_dir: str | Path,
+    *,
+    options: BackupOptions,
+) -> ApplicationBackup:
+    """Capture, inventory, and atomically publish one complete application backup."""
+    if not isinstance(options, BackupOptions):
+        raise ValueError("options must be BackupOptions")
+    app_id, _ = client._ids(app_name)
+    if not app_id.isalnum():
+        raise ValueError(f"Quickbase returned an unsafe application ID: {app_id!r}")
+
+    started = datetime.now(UTC)
+    started_at = _timestamp(started)
+    snapshot_id = str(uuid4())
+    output_root = Path(output_dir).expanduser().resolve()
+    output_root.mkdir(parents=True, exist_ok=True)
+    staging = output_root / f".qbvisor-{snapshot_id}.tmp"
+    final = output_root / (f"{app_id}-{started.strftime('%Y%m%dT%H%M%SZ')}-{snapshot_id}")
+    staging.mkdir(exist_ok=False)
+    workspace = BackupWorkspace(staging)
+
+    try:
+        schema = capture_schema(client, app_name, workspace)
+        records = capture_records(
+            client,
+            schema,
+            workspace,
+            page_size=options.page_size,
+        )
+        attachments = capture_attachments(
+            client,
+            schema,
+            workspace,
+            mode=options.attachment_versions,
+            max_concurrency=options.max_attachment_concurrency,
+        )
+        table_ids = tuple(table.id for table in schema.tables)
+        changed_tables = _changed_tables(client, app_name, table_ids, started_at)
+        if changed_tables and options.fail_on_changes:
+            raise BackupConsistencyError(changed_tables)
+
+        records_by_table = {table.id: table for table in records.tables}
+        attachments_by_table = {table.id: table for table in attachments.tables}
+        artifacts = tuple(sorted(workspace.artifacts, key=lambda artifact: artifact.path))
+        backup_tables: list[BackupTable] = []
+        for table in schema.tables:
+            table_records = records_by_table[table.id]
+            table_attachments = attachments_by_table[table.id]
+            prefix = f"tables/{table.id}/"
+            table_artifacts = tuple(
+                artifact.path for artifact in artifacts if artifact.path.startswith(prefix)
+            )
+            backup_tables.append(
+                BackupTable(
+                    id=table.id,
+                    name=table.name,
+                    record_count=table_records.record_count,
+                    attachment_count=table_attachments.attachment_count,
+                    artifacts=table_artifacts,
+                )
+            )
+
+        realm = client.transport.realm_hostname
+        if not isinstance(realm, str) or not realm:
+            raise ValueError("Quickbase transport does not have a realm hostname")
+        manifest = BackupManifest(
+            snapshot_id=snapshot_id,
+            source_realm=realm,
+            source_app_id=schema.app_id,
+            source_app_name=schema.app_name,
+            qbvisor_version=_package_version(),
+            started_at=started_at,
+            completed_at=_timestamp(datetime.now(UTC)),
+            options=options,
+            consistent=not changed_tables,
+            changed_tables=changed_tables,
+            tables=tuple(backup_tables),
+            artifacts=artifacts,
+        )
+        _write_manifest(staging, manifest)
+        if final.exists():
+            raise FileExistsError(f"Backup destination already exists: {final}")
+        os.replace(staging, final)
+        return ApplicationBackup(path=final, manifest=manifest)
+    except Exception as error:
+        try:
+            shutil.rmtree(staging)
+        except FileNotFoundError:
+            pass
+        except OSError as cleanup_error:
+            error.add_note(f"Could not remove incomplete backup {staging}: {cleanup_error}")
+        raise