redfetch 1.3.0__py3-none-any.whl

redfetch/sync_executor.py

@@ -0,0 +1,164 @@
+"""Concurrent download executor for the planned sync."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable, Callable
+
+import httpx
+
+from redfetch import download
+from redfetch.sync_types import (
+    DesiredInstallTarget,
+    DesiredSet,
+    ExecutionPlan,
+    ExecutionResult,
+    ExecutionResultItem,
+    PlannedAction,
+    PlanReason,
+    RemoteResourceState,
+    RemoteSnapshot,
+    ResultOutcome,
+    SyncEvent,
+    SyncEventCallback,
+)
+
+DOWNLOAD_CONCURRENCY = 6
+
+_INSTANT_OUTCOMES = {"skip": "skipped", "block": "blocked", "untrack": "untracked"}
+
+DownloadSuccessHook = Callable[
+    [DesiredInstallTarget, PlannedAction, RemoteResourceState], Awaitable[None]
+]
+
+
+def _make_item(
+    action: PlannedAction,
+    outcome: ResultOutcome,
+    *,
+    reason: PlanReason | None = None,
+    version: int | None = None,
+    error: str | None = None,
+) -> ExecutionResultItem:
+    """Fill out a result record for the resource. Auto-copies identity fields so callers only pass what varies."""
+    return ExecutionResultItem(
+        target_key=action.target_key,
+        resource_id=action.resource_id,
+        outcome=outcome,
+        reason=reason or action.reason,
+        written_version=version,
+        error_detail=error,
+    )
+
+
+async def _do_download(
+    client: httpx.AsyncClient, action: PlannedAction,
+) -> tuple[bool, str | None]:
+    """Perform one download attempt, returning success and any error."""
+    try:
+        ok = await download.download_install_target_async(
+            client=client,
+            resource_id=action.resource_id,
+            download_url=action.artifact.download_url,
+            filename=action.artifact.filename,
+            file_hash=action.artifact.file_hash,
+            folder_path=action.resolved_path,
+            should_flatten=action.flatten,
+            protected_files=action.protected_files,
+        )
+    except asyncio.CancelledError:
+        raise
+    except Exception as exc:
+        return False, str(exc)
+    return ok, (None if ok else "download failed")
+
+
+async def execute_plan(
+    *,
+    headers: dict,
+    desired_set: DesiredSet,
+    remote_snapshot: RemoteSnapshot,
+    execution_plan: ExecutionPlan,
+    on_event: SyncEventCallback | None = None,
+    on_download_success: DownloadSuccessHook | None = None,
+) -> ExecutionResult:
+    """Execute planned actions (skips, downloads, etc.) and build the result report."""
+    result = ExecutionResult(items={})
+    satisfied: set[str] = set()
+
+    def emit(event: SyncEvent) -> None:
+        if on_event:
+            on_event(event)
+
+    for action in execution_plan.actions.values():
+        outcome = _INSTANT_OUTCOMES.get(action.action)
+        if outcome is None:
+            continue
+        version = action.remote_version if action.action == "skip" else None
+        result.items[action.target_key] = _make_item(action, outcome, version=version)
+        if action.action == "skip":
+            satisfied.add(action.target_key)
+        emit(("done", action.resource_id, outcome))
+
+    download_actions = {
+        k: a for k, a in execution_plan.actions.items() if a.action == "download"
+    }
+    if not download_actions:
+        return result
+
+    sem = asyncio.Semaphore(DOWNLOAD_CONCURRENCY)
+    gate: dict[str, asyncio.Event] = {k: asyncio.Event() for k in download_actions}
+
+    async def run_one(key: str, client: httpx.AsyncClient) -> None:
+        action = download_actions[key]
+        parent = action.parent_target_key
+        try:
+            if parent and parent in gate:
+                await gate[parent].wait()
+
+            if parent and parent not in satisfied:
+                result.items[key] = _make_item(action, "blocked", reason="parent_failed")
+                emit(("done", action.resource_id, "blocked"))
+                return
+
+            if action.artifact is None or action.resolved_path is None:
+                result.items[key] = _make_item(
+                    action, "error", error="missing artifact or resolved path",
+                )
+                emit(("done", action.resource_id, "error"))
+                return
+
+            async with sem:
+                emit(("start", action.resource_id, action.title))
+                ok, error_detail = await _do_download(client, action)
+
+            if ok:
+                result.items[key] = _make_item(
+                    action, "downloaded", version=action.remote_version,
+                )
+                satisfied.add(key)
+                if on_download_success:
+                    await on_download_success(
+                        desired_set.install_targets[key],
+                        action,
+                        remote_snapshot.resources[action.resource_id],
+                    )
+                emit(("done", action.resource_id, "downloaded"))
+            else:
+                result.items[key] = _make_item(action, "error", error=error_detail)
+                emit(("done", action.resource_id, "error"))
+        finally:
+            gate[key].set()
+
+    async with httpx.AsyncClient(
+        headers=headers,
+        http2=True,
+        timeout=60.0,
+        limits=httpx.Limits(
+            max_connections=DOWNLOAD_CONCURRENCY,
+            max_keepalive_connections=DOWNLOAD_CONCURRENCY,
+        ),
+    ) as client:
+        await asyncio.gather(*(run_one(k, client) for k in download_actions))
+
+    return result

redfetch/sync_planner.py

@@ -0,0 +1,196 @@
+"""Decides download/skip/block actions for the planned sync."""
+
+from __future__ import annotations
+
+from redfetch import config
+from redfetch.sync_discovery import (
+    is_special_resource,
+    resolve_dependency_path,
+    resolve_root_path,
+)
+from redfetch.sync_types import (
+    ActionType,
+    DesiredInstallTarget,
+    DesiredSet,
+    ExecutionPlan,
+    LocalInstallState,
+    LocalSnapshot,
+    PlannedAction,
+    PlanReason,
+    RemoteResourceState,
+    RemoteSnapshot,
+    parse_target_key,
+    target_depth,
+)
+
+
+BLOCKING_STATUSES = {"access_denied", "no_files", "multiple_files", "not_found", "fetch_error"}
+
+
+def _desired_targets_in_order(desired_set: DesiredSet) -> list[DesiredInstallTarget]:
+    """Sort targets so parents are always planned before their children."""
+    return sorted(
+        desired_set.install_targets.values(),
+        key=lambda target: (target_depth(target.target_key), target.target_key),
+    )
+
+
+def _cycle_block_keys(desired_set: DesiredSet) -> set[str]:
+    """Block targets that would cause circular dependency loops."""
+    cycle_keys: set[str] = set()
+    for target in desired_set.install_targets.values():
+        segments = parse_target_key(target.target_key)
+        if len(segments) == len(set(segments)):
+            continue
+        repeated_resource_id = segments[-1]
+        first_index = segments[:-1].index(repeated_resource_id)
+        first_blocked_prefix = first_index + 2
+        for prefix_length in range(first_blocked_prefix, len(segments) + 1):
+            candidate_key = f"/{'/'.join(segments[:prefix_length])}/"
+            if candidate_key in desired_set.install_targets:
+                cycle_keys.add(candidate_key)
+    return cycle_keys
+
+
+def _resolve_target_path(
+    target: DesiredInstallTarget,
+    *,
+    parent_action: PlannedAction | None,
+    remote_state: RemoteResourceState | None,
+    settings_env: str,
+) -> tuple[str | None, str | None]:
+    """Figure out where on disk this target should be installed."""
+    if target.resolved_path is not None:
+        return target.resolved_path, target.subfolder
+
+    if target.target_kind == "root":
+        category_id = target.category_id
+        if category_id is None and remote_state is not None:
+            category_id = remote_state.category_id
+        if not is_special_resource(target.resource_id, settings_env):
+            if category_id is None or category_id not in config.CATEGORY_MAP:
+                return None, target.subfolder
+        return resolve_root_path(target.resource_id, category_id, settings_env), None
+
+    if parent_action is None or not parent_action.resolved_path or target.parent_id is None:
+        return None, target.subfolder
+    resolved_path, subfolder = resolve_dependency_path(
+        parent_action.resolved_path,
+        target.parent_id,
+        target.resource_id,
+        settings_env,
+    )
+    return resolved_path, (target.subfolder or subfolder)
+
+
+def _install_context_changed(
+    local_state: LocalInstallState | None,
+    *,
+    resolved_path: str | None,
+    subfolder: str | None,
+    target: DesiredInstallTarget,
+) -> bool:
+    """Check if path, subfolder, or install settings have changed since the last sync."""
+    if local_state is None:
+        return False
+    return (
+        local_state.resolved_path != resolved_path
+        or local_state.subfolder != subfolder
+        or local_state.flatten != target.flatten
+        or local_state.protected_files != target.protected_files
+    )
+
+
+def _decide_action(
+    target: DesiredInstallTarget,
+    *,
+    local_state: LocalInstallState | None,
+    remote_state: RemoteResourceState | None,
+    parent_action: PlannedAction | None,
+    cycle_block_keys: set[str],
+    resolved_path: str | None,
+    subfolder: str | None,
+) -> tuple[ActionType, PlanReason]:
+    """Decide whether a single target should be downloaded, skipped, or blocked, and why."""
+    if target.target_key in cycle_block_keys:
+        return "block", "dependency_cycle"
+    if target.parent_target_key and (
+        parent_action is None or parent_action.action in {"block", "untrack"}
+    ):
+        return "block", "parent_blocked"
+    if target.discovery_block:
+        return "block", target.discovery_block
+    if remote_state is None:
+        return "block", "fetch_error"
+    if remote_state.status in BLOCKING_STATUSES:
+        return "block", remote_state.status  # type: ignore[return-value]
+    if resolved_path is None and target.target_kind == "root":
+        return "block", "unknown_category"
+    if _install_context_changed(
+        local_state, resolved_path=resolved_path, subfolder=subfolder, target=target,
+    ):
+        return "download", "install_context_changed"
+    if (
+        local_state is not None
+        and remote_state.version_id is not None
+        and local_state.version_local == remote_state.version_id
+    ):
+        return "skip", "already_current"
+    if local_state is None or local_state.version_local is None:
+        return "download", "not_installed"
+    return "download", "outdated"
+
+
+def build_execution_plan(
+    *,
+    desired_set: DesiredSet,
+    remote_snapshot: RemoteSnapshot,
+    local_snapshot: LocalSnapshot,
+    settings_env: str,
+) -> ExecutionPlan:
+    """Main planner entry point: decide the action for every target, then mark untracked leftovers."""
+    actions: dict[str, PlannedAction] = {}
+    cycle_keys = _cycle_block_keys(desired_set)
+
+    for target in _desired_targets_in_order(desired_set):
+        local_state = local_snapshot.install_targets.get(target.target_key)
+        remote_state = remote_snapshot.resources.get(target.resource_id)
+        parent_action = actions.get(target.parent_target_key) if target.parent_target_key else None
+        resolved_path, subfolder = _resolve_target_path(
+            target,
+            parent_action=parent_action,
+            remote_state=remote_state,
+            settings_env=settings_env,
+        )
+        action, reason = _decide_action(
+            target,
+            local_state=local_state,
+            remote_state=remote_state,
+            parent_action=parent_action,
+            cycle_block_keys=cycle_keys,
+            resolved_path=resolved_path,
+            subfolder=subfolder,
+        )
+        actions[target.target_key] = PlannedAction.from_desired(
+            target,
+            action=action,
+            reason=reason,
+            title=remote_state.title if remote_state else target.title,
+            category_id=remote_state.category_id if remote_state else target.category_id,
+            remote_version=remote_state.version_id if remote_state else None,
+            artifact=remote_state.artifact if remote_state and action == "download" else None,
+            resolved_path=resolved_path,
+            subfolder=subfolder,
+        )
+
+    if desired_set.mode == "full":
+        local_scope = local_snapshot.install_targets.values()
+    else:
+        local_scope = local_snapshot.roots_in_closure(desired_set.requested_root_ids)
+
+    for local_state in local_scope:
+        if local_state.target_key in actions:
+            continue
+        actions[local_state.target_key] = PlannedAction.untrack_from_local(local_state)
+
+    return ExecutionPlan(actions=actions)

redfetch/sync_remote.py

@@ -0,0 +1,182 @@
+"""Fetches info about resources from the website."""
+
+from __future__ import annotations
+
+from typing import NamedTuple
+
+import httpx
+
+from redfetch import api
+from redfetch import net
+from redfetch.sync_discovery import payload_category_id, payload_title, payload_version_id
+from redfetch.sync_types import (
+    DesiredInstallTarget,
+    DesiredSet,
+    LocalSnapshot,
+    RemoteArtifact,
+    RemoteResourceState,
+    RemoteSnapshot,
+)
+
+
+def _normalize_hash(raw_hash: str | None) -> str | None:
+    """Sanitize the xenforo file hash to a standard format."""
+    if not raw_hash:
+        return None
+    cleaned = str(raw_hash).strip().lower()
+    if len(cleaned) == 32 and all(char in "0123456789abcdef" for char in cleaned):
+        return cleaned
+    return None
+
+
+class _PayloadDetails(NamedTuple):
+    """Bundle of fields extracted from the API for building a RemoteResourceState."""
+    title: str | None
+    category_id: int | None
+    version_id: int | None
+    artifact: RemoteArtifact | None
+
+
+def _extract_artifact(payload: dict) -> RemoteArtifact | None:
+    """Extract downloadable file info."""
+    current_files = payload.get("current_files")
+    if not isinstance(current_files, list) or len(current_files) != 1:
+        return None
+    file_info = current_files[0]
+    if file_info.get("id") is None or not file_info.get("filename") or not file_info.get("download_url"):
+        return None
+    return RemoteArtifact(
+        file_id=int(file_info["id"]),
+        filename=str(file_info["filename"]),
+        download_url=str(file_info["download_url"]),
+        file_hash=_normalize_hash(file_info.get("hash")),
+    )
+
+
+def _payload_details(payload: dict | None) -> _PayloadDetails:
+    """Bundle all fields extracted from a raw API payload."""
+    if not payload:
+        return _PayloadDetails(None, None, None, None)
+    return _PayloadDetails(
+        title=payload_title(payload),
+        category_id=payload_category_id(payload),
+        version_id=payload_version_id(payload),
+        artifact=_extract_artifact(payload),
+    )
+
+
+def _needs_live_check(
+    *,
+    desired_targets: list[DesiredInstallTarget],
+    local_snapshot: LocalSnapshot,
+    manifest_details: _PayloadDetails | None,
+) -> bool:
+    """Return True if the resource needs an update, or data is incomplete, or config has changed."""
+    if manifest_details is None or manifest_details.version_id is None or manifest_details.artifact is None:
+        return True
+
+    for target in desired_targets:
+        local_state = local_snapshot.install_targets.get(target.target_key)
+        if local_state is None:
+            return True
+        if local_state.version_local is None:
+            return True
+        if local_state.version_local != manifest_details.version_id:
+            return True
+        if (
+            local_state.resolved_path != target.resolved_path
+            or local_state.subfolder != target.subfolder
+            or local_state.flatten != target.flatten
+            or local_state.protected_files != target.protected_files
+        ):
+            return True
+
+    return False
+
+
+def _blocked_state(
+    resource_id: str,
+    *,
+    status: str,
+    manifest_details: _PayloadDetails | None,
+    live_title: str | None = None,
+    live_category_id: int | None = None,
+) -> RemoteResourceState:
+    """Build a RemoteResourceState for a resource that is blocked."""
+    return RemoteResourceState(
+        resource_id=resource_id,
+        title=(manifest_details.title if manifest_details else None) or live_title,
+        category_id=(manifest_details.category_id if manifest_details and manifest_details.category_id is not None else live_category_id),
+        version_id=manifest_details.version_id if manifest_details else None,
+        status=status,
+        artifact=None,
+        source_note="manifest" if manifest_details else "live_access_only",
+    )
+
+
+async def fetch_remote_snapshot(
+    *,
+    client: httpx.AsyncClient,
+    desired_set: DesiredSet,
+    local_snapshot: LocalSnapshot,
+) -> RemoteSnapshot:
+    """Assemble the remote half of the sync picture: version, status, and download info per resource."""
+    manifest = await net.fetch_manifest_cached(client)
+    manifest_resources = manifest.get("resources", {}) or {}
+
+    remote_resources: dict[str, RemoteResourceState] = {}
+    ids_needing_live_check: list[str] = []
+    manifest_cache: dict[str, _PayloadDetails | None] = {}
+
+    for resource_id in desired_set.resource_ids:
+        manifest_entry = manifest_resources.get(resource_id)
+        manifest_details = _payload_details(manifest_entry) if manifest_entry else None
+        manifest_cache[resource_id] = manifest_details
+        desired_targets = desired_set.resource_targets(resource_id)
+        if _needs_live_check(
+            desired_targets=desired_targets,
+            local_snapshot=local_snapshot,
+            manifest_details=manifest_details,
+        ):
+            ids_needing_live_check.append(resource_id)
+            continue
+
+        remote_resources[resource_id] = RemoteResourceState(
+            resource_id=resource_id,
+            title=manifest_details.title,
+            category_id=manifest_details.category_id,
+            version_id=manifest_details.version_id,
+            status="manifest_current",
+            artifact=manifest_details.artifact,
+            source_note="manifest_only",
+        )
+
+    if ids_needing_live_check:
+        live_records = await api.fetch_resource_records_batch(client, ids_needing_live_check)
+        for record in live_records:
+            resource_id = record.resource_id
+            manifest_details = manifest_cache.get(resource_id)
+            live_title = payload_title(record.resource)
+            live_category_id = payload_category_id(record.resource)
+
+            if record.status != "downloadable":
+                remote_resources[resource_id] = _blocked_state(
+                    resource_id,
+                    status=record.status,
+                    manifest_details=manifest_details,
+                    live_title=live_title,
+                    live_category_id=live_category_id,
+                )
+                continue
+
+            remote_resources[resource_id] = RemoteResourceState(
+                resource_id=resource_id,
+                title=(manifest_details.title if manifest_details else None) or live_title,
+                category_id=(manifest_details.category_id if manifest_details and manifest_details.category_id is not None else live_category_id),
+                version_id=manifest_details.version_id if manifest_details else None,
+                status="downloadable",
+                artifact=manifest_details.artifact if manifest_details else None,
+                source_note="manifest_plus_access_check" if manifest_details else "live_access_only",
+            )
+
+    return RemoteSnapshot(resources=remote_resources)