qbvisor 0.3.0__py3-none-any.whl

qbvisor/__init__.py

@@ -0,0 +1,141 @@
+"""Public package interface for the qbvisor Quickbase SDK."""
+
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+from ._version import __version__
+
+# Try to load .env from common locations
+possible_paths = [
+    Path(__file__).resolve().parents[2] / ".env",  # repo root if src/qbvisor/
+    Path(__file__).resolve().parents[1] / ".env",  # one level up
+    Path.cwd() / ".env",  # where the script is run from
+]
+
+for dotenv_path in possible_paths:
+    if dotenv_path.exists():
+        load_dotenv(dotenv_path=dotenv_path)
+        break
+
+from .backup import (
+    BACKUP_FORMAT,
+    BACKUP_FORMAT_VERSION,
+    ApplicationBackup,
+    AttachmentVersionMode,
+    BackupArtifact,
+    BackupArtifactKind,
+    BackupManifest,
+    BackupOptions,
+    BackupTable,
+    BackupVerification,
+)
+from .client import QuickBaseClient
+from .exceptions import (
+    BackupConsistencyError,
+    BackupIntegrityError,
+    QuickbaseBatchError,
+    QuickbaseConfigurationError,
+    QuickbaseConnectionError,
+    QuickbaseError,
+    QuickbaseHTTPError,
+    QuickbaseRateLimitError,
+    QuickbaseResponseError,
+    QuickbaseSchemaApplyError,
+    QuickbaseSchemaConflictError,
+    QuickbaseSchemaLockError,
+    QuickbaseSchemaStalePlanError,
+    QuickbaseSchemaStateError,
+    QuickbaseTimeoutError,
+)
+from .helpers import (
+    ensure_temp_dir,
+    generate_timestamped_folder,
+    sanitize_filenames,
+    summarize_file_sizes,
+)
+from .log_runner import LoggingConfigurator, get_logger
+from .models import RelationshipAccumulation, RelationshipSummary
+from .query_helper import QueryHelper
+from .schema import (
+    SCHEMA_STATE_FORMAT,
+    SCHEMA_STATE_FORMAT_VERSION,
+    AppSpec,
+    FieldSpec,
+    FormulaFieldType,
+    FormulaSpec,
+    RelationshipSpec,
+    SchemaAction,
+    SchemaApplyResult,
+    SchemaAttributeChange,
+    SchemaChange,
+    SchemaPlan,
+    SchemaResourceKind,
+    SchemaState,
+    SchemaStateAction,
+    StateResource,
+    SummaryFieldSpec,
+    TableSpec,
+)
+from .transport import QuickBaseTransport, RetryPolicy
+
+# Expose file download utilities directly on the client
+
+__all__ = [
+    "__version__",
+    "QuickBaseClient",
+    "BACKUP_FORMAT",
+    "BACKUP_FORMAT_VERSION",
+    "ApplicationBackup",
+    "AttachmentVersionMode",
+    "BackupArtifact",
+    "BackupArtifactKind",
+    "BackupManifest",
+    "BackupOptions",
+    "BackupTable",
+    "BackupVerification",
+    "BackupConsistencyError",
+    "BackupIntegrityError",
+    "QuickBaseTransport",
+    "RetryPolicy",
+    "QuickbaseError",
+    "QuickbaseBatchError",
+    "QuickbaseConfigurationError",
+    "QuickbaseConnectionError",
+    "QuickbaseTimeoutError",
+    "QuickbaseHTTPError",
+    "QuickbaseRateLimitError",
+    "QuickbaseResponseError",
+    "QuickbaseSchemaApplyError",
+    "QuickbaseSchemaConflictError",
+    "QuickbaseSchemaLockError",
+    "QuickbaseSchemaStateError",
+    "QuickbaseSchemaStalePlanError",
+    "QueryHelper",
+    "SCHEMA_STATE_FORMAT",
+    "SCHEMA_STATE_FORMAT_VERSION",
+    "AppSpec",
+    "FieldSpec",
+    "FormulaFieldType",
+    "FormulaSpec",
+    "RelationshipSpec",
+    "SchemaAction",
+    "SchemaApplyResult",
+    "SchemaAttributeChange",
+    "SchemaChange",
+    "SchemaPlan",
+    "SchemaResourceKind",
+    "SchemaState",
+    "SchemaStateAction",
+    "StateResource",
+    "SummaryFieldSpec",
+    "TableSpec",
+    "RelationshipAccumulation",
+    "RelationshipSummary",
+    "sanitize_filenames",
+    "ensure_temp_dir",
+    "generate_timestamped_folder",
+    "summarize_file_sizes",
+    "LoggingConfigurator",
+    "get_logger",
+]

qbvisor/_attachments.py

@@ -0,0 +1,83 @@
+"""Validation helpers for attachment metadata returned by record queries."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from .exceptions import QuickbaseResponseError
+
+
+@dataclass(frozen=True, slots=True)
+class LatestAttachment:
+    """The latest downloadable version of a file attachment."""
+
+    version_number: int
+    file_name: str
+
+
+def latest_attachment(
+    value: Any,
+    *,
+    table_id: str,
+    record_id: int,
+    field_id: int,
+) -> LatestAttachment | None:
+    """Return validated latest-version metadata, or ``None`` for an empty file cell."""
+    location = f"{table_id}/{record_id}/{field_id}"
+    if value in (None, ""):
+        return None
+    if not isinstance(value, dict):
+        raise QuickbaseResponseError(
+            "POST",
+            "records/query",
+            expected="file attachment value object",
+            actual=f"{type(value).__name__} at {location}",
+        )
+
+    versions = value.get("versions")
+    if versions is None:
+        return None
+    if not isinstance(versions, list) or not all(isinstance(version, dict) for version in versions):
+        raise QuickbaseResponseError(
+            "POST",
+            "records/query",
+            expected="file attachment versions array",
+            actual=f"{type(versions).__name__} at {location}",
+        )
+    if not versions:
+        return None
+
+    numbered: list[tuple[int, dict[str, Any]]] = []
+    for version in versions:
+        number = version.get("versionNumber")
+        if not isinstance(number, int) or isinstance(number, bool) or number < 1:
+            raise QuickbaseResponseError(
+                "POST",
+                "records/query",
+                expected="positive integer attachment versionNumber",
+                actual=f"{number!r} at {location}",
+            )
+        numbered.append((number, version))
+
+    if len({number for number, _ in numbered}) != len(numbered):
+        raise QuickbaseResponseError(
+            "POST",
+            "records/query",
+            expected="unique attachment versionNumber values",
+            actual=f"duplicates at {location}",
+        )
+
+    version_number, latest = max(numbered, key=lambda item: item[0])
+    file_name = latest.get("fileName") or value.get("fileName")
+    if file_name is None or file_name == "":
+        file_name = f"fid{field_id}_v{version_number}.bin"
+    elif not isinstance(file_name, str):
+        raise QuickbaseResponseError(
+            "POST",
+            "records/query",
+            expected="string attachment fileName",
+            actual=f"{type(file_name).__name__} at {location}",
+        )
+
+    return LatestAttachment(version_number=version_number, file_name=file_name)

qbvisor/_backup/__init__.py

@@ -0,0 +1,23 @@
+"""Private implementation for application backup workflows."""
+
+from .attachments import (
+    CapturedAttachments,
+    CapturedAttachmentTable,
+    capture_attachments,
+)
+from .records import CapturedRecords, CapturedRecordTable, capture_records
+from .schema import CapturedSchema, CapturedTable, capture_schema
+from .workspace import BackupWorkspace
+
+__all__ = [
+    "BackupWorkspace",
+    "CapturedAttachments",
+    "CapturedAttachmentTable",
+    "CapturedRecords",
+    "CapturedRecordTable",
+    "CapturedSchema",
+    "CapturedTable",
+    "capture_attachments",
+    "capture_records",
+    "capture_schema",
+]

qbvisor/_backup/attachments.py

@@ -0,0 +1,303 @@
+"""Archive file attachment versions referenced by captured table records."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from collections.abc import Callable, Iterator
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+from ..async_transport import AsyncQuickBaseTransport
+from ..backup import AttachmentVersionMode, BackupArtifact
+from ..exceptions import QuickbaseResponseError
+from ..helpers import sanitize_filenames
+from ..transport import QuickBaseTransport
+from .schema import CapturedSchema, CapturedTable
+from .workspace import BackupWorkspace, JsonLinesArtifactWriter
+
+
+class AttachmentClient(Protocol):
+    transport: QuickBaseTransport
+
+
+class AsyncFileTransport(Protocol):
+    async def __aenter__(self) -> AsyncFileTransport: ...
+
+    async def __aexit__(self, *_: object) -> None: ...
+
+    async def get_file(self, path: str) -> bytes: ...
+
+
+@dataclass(frozen=True, slots=True)
+class AttachmentJob:
+    table_id: str
+    record_id: int
+    field_id: int
+    version_number: int
+    file_name: str
+    path: str
+    metadata: dict[str, Any]
+
+
+@dataclass(frozen=True, slots=True)
+class CapturedAttachmentTable:
+    id: str
+    attachment_count: int
+    index_artifact: str
+
+
+@dataclass(frozen=True, slots=True)
+class CapturedAttachments:
+    tables: tuple[CapturedAttachmentTable, ...]
+
+
+def _safe_filename(value: str, field_id: int, version_number: int) -> str:
+    clean = sanitize_filenames(value)
+    clean = "".join(character for character in clean if " " <= character != "\x7f")
+    clean = clean.strip().strip(".")
+    if not clean:
+        clean = f"fid{field_id}_v{version_number}.bin"
+    while len(clean.encode("utf-8")) > 200:
+        clean = clean[:-1]
+    return clean
+
+
+def _file_field_ids(table: CapturedTable) -> tuple[int, ...]:
+    field_ids: list[int] = []
+    for field in table.fields:
+        if field.get("fieldType") != "file":
+            continue
+        field_id = field.get("id")
+        if not isinstance(field_id, int) or isinstance(field_id, bool):
+            raise ValueError(f"Quickbase file field in table {table.id} is missing a valid id")
+        field_ids.append(field_id)
+    return tuple(field_ids)
+
+
+def _record_id(record: dict[str, Any], table_id: str, line_number: int) -> int:
+    cell = record.get("3")
+    value = cell.get("value") if isinstance(cell, dict) else None
+    if not isinstance(value, int) or isinstance(value, bool) or value < 1:
+        raise ValueError(
+            f"records.jsonl for table {table_id} has an invalid Record ID# at line {line_number}"
+        )
+    return value
+
+
+def _selected_versions(
+    versions: Any,
+    mode: AttachmentVersionMode,
+    *,
+    table_id: str,
+    record_id: int,
+    field_id: int,
+) -> tuple[dict[str, Any], ...]:
+    if not isinstance(versions, list) or not all(isinstance(version, dict) for version in versions):
+        raise QuickbaseResponseError(
+            "POST",
+            "records/query",
+            expected="file attachment versions array",
+            actual=type(versions).__name__,
+        )
+    numbered: list[tuple[int, dict[str, Any]]] = []
+    for version in versions:
+        number = version.get("versionNumber")
+        if not isinstance(number, int) or isinstance(number, bool) or number < 1:
+            raise ValueError(
+                f"Attachment version for {table_id}/{record_id}/{field_id} has an invalid number"
+            )
+        numbered.append((number, version))
+    numbered.sort(key=lambda item: item[0])
+    if len({number for number, _ in numbered}) != len(numbered):
+        raise ValueError(
+            f"Attachment versions for {table_id}/{record_id}/{field_id} contain duplicates"
+        )
+    if mode == "latest" and numbered:
+        numbered = [numbered[-1]]
+    return tuple(version for _, version in numbered)
+
+
+def _attachment_jobs(
+    table: CapturedTable,
+    workspace: BackupWorkspace,
+    mode: AttachmentVersionMode,
+) -> Iterator[AttachmentJob]:
+    if mode == "none":
+        return
+    field_ids = _file_field_ids(table)
+    if not field_ids:
+        return
+    records_path = workspace.root / "tables" / table.id / "records.jsonl"
+    with records_path.open(encoding="utf-8") as records:
+        for line_number, line in enumerate(records, start=1):
+            try:
+                record = json.loads(line)
+            except json.JSONDecodeError as error:
+                raise ValueError(
+                    f"records.jsonl for table {table.id} is invalid at line {line_number}"
+                ) from error
+            if not isinstance(record, dict):
+                raise ValueError(f"records.jsonl for table {table.id} must contain objects")
+            record_id = _record_id(record, table.id, line_number)
+            for field_id in field_ids:
+                cell = record.get(str(field_id))
+                value = cell.get("value") if isinstance(cell, dict) else None
+                if value in (None, ""):
+                    continue
+                if not isinstance(value, dict):
+                    raise QuickbaseResponseError(
+                        "POST",
+                        "records/query",
+                        expected="file attachment value object",
+                        actual=type(value).__name__,
+                    )
+                versions = _selected_versions(
+                    value.get("versions", []),
+                    mode,
+                    table_id=table.id,
+                    record_id=record_id,
+                    field_id=field_id,
+                )
+                for version in versions:
+                    version_number = int(version["versionNumber"])
+                    raw_name = version.get("fileName") or value.get("fileName")
+                    file_name = (
+                        raw_name
+                        if isinstance(raw_name, str) and raw_name
+                        else f"fid{field_id}_v{version_number}.bin"
+                    )
+                    safe_name = _safe_filename(file_name, field_id, version_number)
+                    path = (
+                        f"tables/{table.id}/attachments/{record_id}/"
+                        f"{field_id}/{version_number}/{safe_name}"
+                    )
+                    yield AttachmentJob(
+                        table_id=table.id,
+                        record_id=record_id,
+                        field_id=field_id,
+                        version_number=version_number,
+                        file_name=file_name,
+                        path=path,
+                        metadata=version,
+                    )
+
+
+def _index_entry(job: AttachmentJob, artifact: BackupArtifact) -> dict[str, Any]:
+    return {
+        "record_id": job.record_id,
+        "field_id": job.field_id,
+        "version_number": job.version_number,
+        "file_name": job.file_name,
+        "path": artifact.path,
+        "sha256": artifact.sha256,
+        "bytes": artifact.bytes,
+        "metadata": job.metadata,
+    }
+
+
+async def _download_job(
+    transport: AsyncFileTransport,
+    workspace: BackupWorkspace,
+    job: AttachmentJob,
+) -> tuple[AttachmentJob, BackupArtifact]:
+    endpoint = f"files/{job.table_id}/{job.record_id}/{job.field_id}/{job.version_number}"
+    content = await transport.get_file(endpoint)
+    return job, workspace.write_bytes(job.path, "attachment", content)
+
+
+async def _capture_table_attachments(
+    transport: AsyncFileTransport,
+    table: CapturedTable,
+    workspace: BackupWorkspace,
+    mode: AttachmentVersionMode,
+    max_concurrency: int,
+    index_writer: JsonLinesArtifactWriter,
+) -> int:
+    jobs = iter(_attachment_jobs(table, workspace, mode))
+    count = 0
+    while True:
+        batch: list[AttachmentJob] = []
+        for _ in range(max_concurrency):
+            try:
+                batch.append(next(jobs))
+            except StopIteration:
+                break
+        if not batch:
+            return count
+        results = await asyncio.gather(*(_download_job(transport, workspace, job) for job in batch))
+        for job, artifact in results:
+            index_writer.write(_index_entry(job, artifact))
+            count += 1
+
+
+async def _capture_all_attachments(
+    client: AttachmentClient,
+    schema: CapturedSchema,
+    workspace: BackupWorkspace,
+    mode: AttachmentVersionMode,
+    max_concurrency: int,
+    transport_factory: Callable[[QuickBaseTransport], AsyncFileTransport],
+) -> CapturedAttachments:
+    captured: list[CapturedAttachmentTable] = []
+    async with transport_factory(client.transport) as transport:
+        for table in schema.tables:
+            index_path = f"tables/{table.id}/attachments.jsonl"
+            with workspace.json_lines_writer(index_path, "attachment-index") as writer:
+                count = await _capture_table_attachments(
+                    transport,
+                    table,
+                    workspace,
+                    mode,
+                    max_concurrency,
+                    writer,
+                )
+            captured.append(
+                CapturedAttachmentTable(
+                    id=table.id,
+                    attachment_count=count,
+                    index_artifact=writer.artifact.path,
+                )
+            )
+    return CapturedAttachments(tables=tuple(captured))
+
+
+def capture_attachments(
+    client: AttachmentClient,
+    schema: CapturedSchema,
+    workspace: BackupWorkspace,
+    *,
+    mode: AttachmentVersionMode,
+    max_concurrency: int,
+    transport_factory: Callable[[QuickBaseTransport], AsyncFileTransport] = AsyncQuickBaseTransport,
+) -> CapturedAttachments:
+    """Archive selected attachment versions with bounded concurrent downloads."""
+    if mode not in {"all", "latest", "none"}:
+        raise ValueError(f"Unsupported attachment version mode: {mode}")
+    if max_concurrency < 1:
+        raise ValueError("max_concurrency must be at least 1")
+    if mode != "none":
+        return asyncio.run(
+            _capture_all_attachments(
+                client,
+                schema,
+                workspace,
+                mode,
+                max_concurrency,
+                transport_factory,
+            )
+        )
+
+    captured: list[CapturedAttachmentTable] = []
+    for table in schema.tables:
+        index_path = f"tables/{table.id}/attachments.jsonl"
+        with workspace.json_lines_writer(index_path, "attachment-index") as writer:
+            pass
+        captured.append(
+            CapturedAttachmentTable(
+                id=table.id,
+                attachment_count=0,
+                index_artifact=writer.artifact.path,
+            )
+        )
+    return CapturedAttachments(tables=tuple(captured))