redfetch 1.3.0__py3-none-any.whl

redfetch/sync.py

@@ -0,0 +1,261 @@
+from __future__ import annotations
+
+import asyncio
+
+import httpx
+
+from redfetch import config
+from redfetch import store
+from redfetch import sync_discovery
+from redfetch import sync_executor
+from redfetch import sync_planner
+from redfetch import sync_remote
+from redfetch.sync_types import (
+    ExecutionPlan,
+    ExecutionResult,
+    PLAN_REASON_META,
+    PreparedSync,
+    ReasonInfo,
+    SyncEventCallback,
+    reason_message,
+)
+
+
+_sync_lock: asyncio.Lock | None = None
+_DEFAULT_REASON = ReasonInfo(message="")
+
+
+def _print_plan_summary(
+    execution_plan: ExecutionPlan,
+    resource_ids: list[str] | None = None,
+) -> None:
+    is_full_sync = resource_ids is None
+    all_blocked = [
+        action for action in execution_plan.actions.values()
+        if action.action == "block"
+    ]
+    if is_full_sync:
+        quiet = [a for a in all_blocked if PLAN_REASON_META.get(a.reason, _DEFAULT_REASON).quiet]
+        blocked = [a for a in all_blocked if not PLAN_REASON_META.get(a.reason, _DEFAULT_REASON).quiet]
+    else:
+        quiet = []
+        blocked = all_blocked
+
+    counts = execution_plan.action_counts()
+    print(f"Resources in scope: >>> {len(execution_plan.actions)} <<<")
+    print(f"Resources to download: >>> {counts.get('download', 0)} <<<")
+    if blocked:
+        print(f"Resources blocked: >>> {len(blocked)} <<<")
+        for action in blocked:
+            label = action.title or action.resource_id
+            print(f"  - {label} (id={action.resource_id}): {reason_message(action.reason)}")
+
+    summary_buckets: dict[str, int] = {}
+    for action in quiet:
+        label = PLAN_REASON_META[action.reason].summary_label
+        if label:
+            summary_buckets[label] = summary_buckets.get(label, 0) + 1
+    for label, count in summary_buckets.items():
+        print(f"{label}: >>> {count} <<<")
+
+    if counts.get("untrack", 0):
+        print(f"Resources to untrack: >>> {counts.get('untrack', 0)} <<<")
+
+
+def _print_failure_detail(
+    execution_plan: ExecutionPlan,
+    resource_ids: list[str],
+) -> None:
+    """Print a hint when a targeted sync fails with zero scoped actions (resource not in scope at all)."""
+    requested = {str(rid) for rid in resource_ids}
+    has_scoped_actions = any(
+        action.root_resource_id in requested
+        for action in execution_plan.actions.values()
+    )
+    if not has_scoped_actions:
+        print(
+            f"No valid resources found for IDs: {resource_ids}. "
+            "Are you in the right server env? Did you opt_in in your settings.local.toml?"
+        )
+
+
+def _run_succeeded(
+    *,
+    execution_plan: ExecutionPlan,
+    execution_result: ExecutionResult,
+    resource_ids: list[str] | None,
+) -> bool:
+    """Targeted sync succeeds when there are no errors, at least one explicit root exists, and no target in the requested closure is blocked."""
+    if execution_result.has_errors():
+        return False
+
+    if resource_ids is None:
+        return True
+
+    requested_root_ids = {str(resource_id) for resource_id in resource_ids}
+    scoped_actions = [
+        action
+        for action in execution_plan.actions.values()
+        if action.root_resource_id in requested_root_ids
+    ]
+    if not scoped_actions:
+        return False
+
+    if not any(action.explicit_root for action in scoped_actions):
+        return False
+
+    for action in scoped_actions:
+        item = execution_result.items.get(action.target_key)
+        if item is None or item.outcome == "blocked":
+            return False
+    return True
+
+
+async def prepare_sync(
+    db_path: str,
+    headers: dict,
+    resource_ids: list[str] | None = None,
+) -> PreparedSync:
+    """Run discovery -> remote snapshot -> planning, returning the plan without executing it."""
+    settings_env = config.settings.ENV
+    local_snapshot = await store.load_local_snapshot(db_path)
+
+    async with httpx.AsyncClient(
+        headers=headers,
+        http2=True,
+        timeout=30.0,
+        limits=httpx.Limits(max_connections=10, max_keepalive_connections=5),
+    ) as client:
+        desired_set = await sync_discovery.discover_desired_set(
+            client=client,
+            resource_ids=resource_ids,
+            settings_env=settings_env,
+        )
+        remote_snapshot = await sync_remote.fetch_remote_snapshot(
+            client=client,
+            desired_set=desired_set,
+            local_snapshot=local_snapshot,
+        )
+
+    execution_plan = sync_planner.build_execution_plan(
+        desired_set=desired_set,
+        remote_snapshot=remote_snapshot,
+        local_snapshot=local_snapshot,
+        settings_env=settings_env,
+    )
+
+    return PreparedSync(
+        desired_set=desired_set,
+        remote_snapshot=remote_snapshot,
+        local_snapshot=local_snapshot,
+        execution_plan=execution_plan,
+    )
+
+
+async def sync(
+    db_path: str,
+    headers: dict,
+    resource_ids: list[str] | None = None,
+    on_event: SyncEventCallback | None = None,
+) -> bool:
+    """Discover, plan, and execute a sync run against the API."""
+    prepared = await prepare_sync(db_path, headers, resource_ids=resource_ids)
+    desired_set = prepared.desired_set
+    remote_snapshot = prepared.remote_snapshot
+    local_snapshot = prepared.local_snapshot
+    execution_plan = prepared.execution_plan
+
+    _print_plan_summary(execution_plan, resource_ids=resource_ids)
+    if on_event:
+        on_event(("total", len(execution_plan.actions), None))
+
+    execution_result = await sync_executor.execute_plan(
+        headers=headers,
+        desired_set=desired_set,
+        remote_snapshot=remote_snapshot,
+        execution_plan=execution_plan,
+        on_event=on_event,
+        on_download_success=lambda target, action, remote: store.record_download_success(
+            db_path,
+            target=target,
+            action=action,
+            remote_state=remote,
+        ),
+    )
+
+    try:
+        await store.record_installed_state(
+            db_path,
+            desired_set=desired_set,
+            remote_snapshot=remote_snapshot,
+            local_snapshot=local_snapshot,
+            execution_plan=execution_plan,
+            execution_result=execution_result,
+        )
+    except Exception as exc:
+        print(f"Warning: failed to record sync state: {exc}")
+
+    success = _run_succeeded(
+        execution_plan=execution_plan,
+        execution_result=execution_result,
+        resource_ids=resource_ids,
+    )
+
+    if execution_result.has_errors():
+        errored_items = [
+            item
+            for item in execution_result.items.values()
+            if item.outcome == "error"
+        ]
+        if errored_items:
+            print("One or more resources failed to download.")
+            for item in errored_items:
+                detail = f": {item.error_detail}" if item.error_detail else ""
+                print(f"  - {item.resource_id}{detail}")
+    elif resource_ids is not None and not success:
+        _print_failure_detail(execution_plan, resource_ids)
+    elif any(item.outcome == "downloaded" for item in execution_result.items.values()):
+        print("All resources downloaded successfully.")
+    else:
+        print("All resources are up-to-date; no downloads were necessary.")
+
+    return success
+
+
+async def run_sync(
+    db_path: str,
+    headers: dict,
+    resource_ids: list[str] | None = None,
+    on_event: SyncEventCallback | None = None,
+    navmesh_override: bool | None = None,
+) -> bool:
+    """Top-level entry point: run the sync pipeline under a global lock, then navmesh if applicable."""
+    global _sync_lock
+    if _sync_lock is None:
+        _sync_lock = asyncio.Lock()
+
+    try:
+        async with _sync_lock:
+            result = await sync(
+                db_path,
+                headers,
+                resource_ids=resource_ids,
+                on_event=on_event,
+            )
+
+            if resource_ids is None:
+                from redfetch import navmesh
+
+                navmesh_ok = await navmesh.sync_navmeshes(
+                    db_path,
+                    headers,
+                    on_event=on_event,
+                    override=navmesh_override,
+                )
+                if not navmesh_ok:
+                    print("navmesh sync encountered errors")
+
+            return result
+    except (KeyboardInterrupt, asyncio.CancelledError):
+        print("Download cancelled by user.")
+        return False

redfetch/sync_discovery.py

@@ -0,0 +1,352 @@
+"""Builds the desired install-target set from resources that are watched, licensed, special, etc."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+import httpx
+
+from redfetch import api
+from redfetch import config
+from redfetch import utils
+from redfetch.sync_types import (
+    DesiredInstallTarget,
+    DesiredSet,
+    make_child_target_key,
+    make_root_target_key,
+    parse_target_key,
+)
+
+
+# ---------------------------------------------------------------------------
+# Path resolution helpers
+# ---------------------------------------------------------------------------
+
+def is_special_resource(resource_id: str, settings_env: str) -> bool:
+    """Check if a resource is configured in SPECIAL_RESOURCES for the given environment."""
+    return str(resource_id) in config.settings.from_env(settings_env).SPECIAL_RESOURCES
+
+
+def _get_dependency_details(parent_id: str, resource_id: str, settings_env: str) -> dict[str, Any]:
+    settings_for_env = config.settings.from_env(settings_env)
+    parent_settings = settings_for_env.SPECIAL_RESOURCES.get(str(parent_id), {})
+    dependencies = parent_settings.get("dependencies", {})
+    return dependencies.get(str(resource_id), {})
+
+
+def resolve_dependency_path(parent_target_path: str, parent_id: str, resource_id: str, settings_env: str) -> tuple[str, str | None]:
+    dependency_details = _get_dependency_details(parent_id, resource_id, settings_env)
+    subfolder = dependency_details.get("subfolder", "") or ""
+    resolved_path = os.path.join(parent_target_path, subfolder) if subfolder else parent_target_path
+    return os.path.normpath(resolved_path), (subfolder or None)
+
+
+def _get_protected_files(resource_id: str, settings_env: str) -> list[str]:
+    settings_for_env = config.settings.from_env(settings_env)
+    protected = getattr(settings_for_env, "PROTECTED_FILES_BY_RESOURCE", {})
+    return list(protected.get(str(resource_id), []))
+
+
+def resolve_root_path(resource_id: str, category_id: int | None, settings_env: str) -> str:
+    settings_for_env = config.settings.from_env(settings_env)
+    download_folder = os.path.normpath(settings_for_env.DOWNLOAD_FOLDER) if settings_for_env.DOWNLOAD_FOLDER else ""
+
+    special_resource = settings_for_env.SPECIAL_RESOURCES.get(str(resource_id))
+    special_destination = utils.resolve_special_destination(special_resource, download_folder)
+    if special_destination:
+        return special_destination
+
+    category_name = config.CATEGORY_MAP.get(category_id or -1, "")
+
+    if category_name:
+        category_paths = getattr(settings_for_env, "CATEGORY_PATHS", None) or {}
+        override = category_paths.get(category_name)
+        if override:
+            if os.path.isabs(override):
+                return os.path.normpath(override)
+            return os.path.normpath(os.path.join(download_folder, override))
+
+    base_path = download_folder
+    vvmq_id = utils.get_current_vvmq_id(settings_env)
+    if vvmq_id:
+        vvmq_resource = settings_for_env.SPECIAL_RESOURCES.get(vvmq_id)
+        vvmq_destination = utils.resolve_special_destination(vvmq_resource, download_folder)
+        if vvmq_destination:
+            base_path = vvmq_destination
+
+    if category_name:
+        return os.path.normpath(os.path.join(base_path, category_name))
+    return base_path
+
+
+def _get_flatten(resource_id: str, parent_id: str | None, settings_env: str) -> bool:
+    if parent_id:
+        dependency_details = _get_dependency_details(parent_id, resource_id, settings_env)
+        if "flatten" in dependency_details:
+            return bool(dependency_details["flatten"])
+
+    settings_for_env = config.settings.from_env(settings_env)
+    special_resource = settings_for_env.SPECIAL_RESOURCES.get(str(resource_id), {})
+    return bool(special_resource.get("flatten", False))
+
+
+# ---------------------------------------------------------------------------
+# Discovery logic
+# ---------------------------------------------------------------------------
+
+@dataclass
+class _RootSpec:
+    """Tracks why a resource was selected for sync and its raw API payload."""
+    sources: set[str] = field(default_factory=set)
+    payload: dict | None = None
+    discovery_block: str | None = None
+
+
+def payload_title(payload: dict | None) -> str | None:
+    if not payload:
+        return None
+    title = payload.get("title")
+    return str(title) if title is not None else None
+
+
+def payload_category_id(payload: dict | None) -> int | None:
+    if not payload:
+        return None
+    category = payload.get("Category") or payload.get("category") or {}
+    raw = category.get("parent_category_id") or payload.get("parent_category_id")
+    if raw is None:
+        return None
+    return int(raw)
+
+
+def payload_version_id(payload: dict | None) -> int | None:
+    """Only the manifest endpoint provides this; the live XenForo API does not."""
+    if not payload:
+        return None
+    raw = payload.get("version_id")
+    return int(raw) if raw is not None else None
+
+
+def _category_allowed_in_env(category_id: int | None, settings_env: str) -> bool:
+    """Category 11 (plugins) is excluded from TEST and EMU for now."""
+    if category_id is None:
+        return False
+    if category_id not in config.CATEGORY_MAP:
+        return False
+    if category_id == 11 and settings_env.upper() in {"TEST", "EMU"}:
+        return False
+    return True
+
+
+def _root_sources_for_full_sync(
+    watched_resources: list[dict],
+    licenses: list[dict],
+    settings_env: str,
+) -> dict[str, _RootSpec]:
+    """Collect every resource that qualifies for a full sync and record how each one qualified."""
+    specs: dict[str, _RootSpec] = {}
+
+    for payload in watched_resources:
+        category_id = payload_category_id(payload)
+        if not _category_allowed_in_env(category_id, settings_env):
+            continue
+        resource_id = str(payload["resource_id"])
+        spec = specs.setdefault(resource_id, _RootSpec())
+        spec.sources.add("watching")
+        spec.payload = payload
+
+    licenses_by_resource: dict[str, list[tuple[dict, bool]]] = {}
+    for license_info in licenses:
+        if not license_info.get("active", False):
+            continue
+        end_date = license_info.get("end_date", 0)
+        is_expired = end_date != 0 and end_date < time.time()
+        payload = license_info.get("resource") or {}
+        category_id = payload_category_id(payload)
+        if not _category_allowed_in_env(category_id, settings_env):
+            continue
+        resource_id = str(payload["resource_id"])
+        licenses_by_resource.setdefault(resource_id, []).append((payload, is_expired))
+
+    for resource_id, resource_licenses in licenses_by_resource.items():
+        spec = specs.setdefault(resource_id, _RootSpec())
+        spec.sources.add("licensed")
+        valid_license = next(
+            (lic_payload for lic_payload, is_expired in resource_licenses if not is_expired),
+            None,
+        )
+        if valid_license is None:
+            spec.discovery_block = "license_expired"
+            spec.payload = resource_licenses[0][0]
+        else:
+            spec.discovery_block = None
+            spec.payload = valid_license
+
+    settings_for_env = config.settings.from_env(settings_env)
+    for resource_id, resource_info in settings_for_env.SPECIAL_RESOURCES.items():
+        if resource_info.get("opt_in", False):
+            specs.setdefault(str(resource_id), _RootSpec()).sources.add("special")
+
+    return specs
+
+
+def _root_sources_for_targeted(resource_ids: list[str], settings_env: str) -> dict[str, _RootSpec]:
+    """builds the resource list for a single resource sync, since they can have dependencies."""
+    settings_for_env = config.settings.from_env(settings_env)
+    specs: dict[str, _RootSpec] = {}
+    for resource_id in resource_ids:
+        rid = str(resource_id)
+        spec = _RootSpec(sources={"explicit"})
+        resource_info = settings_for_env.SPECIAL_RESOURCES.get(rid, {})
+        if resource_info.get("opt_in", False):
+            spec.sources.add("special")
+        specs[rid] = spec
+    return specs
+
+
+def _add_root_target(
+    desired_set: DesiredSet,
+    *,
+    resource_id: str,
+    sources: set[str],
+    payload: dict | None,
+    settings_env: str,
+) -> DesiredInstallTarget:
+    """Resolve path, category, flatten, and protected files for a root resource and add it to the plan."""
+    category_id = payload_category_id(payload)
+    resolved_path = None
+    settings_for_env = config.settings.from_env(settings_env)
+    special_resource = settings_for_env.SPECIAL_RESOURCES.get(str(resource_id))
+    has_own_destination = bool(
+        special_resource and (special_resource.get("custom_path") or special_resource.get("default_path"))
+    )
+    if has_own_destination or category_id is not None:
+        resolved_path = resolve_root_path(resource_id, category_id, settings_env)
+    target = DesiredInstallTarget(
+        target_key=make_root_target_key(resource_id),
+        resource_id=resource_id,
+        parent_id=None,
+        parent_target_key=None,
+        root_resource_id=resource_id,
+        target_kind="root",
+        sources=set(sources),
+        title=payload_title(payload),
+        category_id=category_id,
+        resolved_path=resolved_path,
+        subfolder=None,
+        flatten=_get_flatten(resource_id, None, settings_env),
+        protected_files=_get_protected_files(resource_id, settings_env),
+        explicit_root="explicit" in sources,
+    )
+    return desired_set.add_target(target)
+
+
+def _add_dependency_target(
+    desired_set: DesiredSet,
+    *,
+    parent_target: DesiredInstallTarget,
+    resource_id: str,
+    settings_env: str,
+) -> DesiredInstallTarget:
+    """Build an install target for a dependency, deriving its path from the parent target."""
+    if parent_target.resolved_path:
+        resolved_path, subfolder = resolve_dependency_path(
+            parent_target.resolved_path, parent_target.resource_id, resource_id, settings_env,
+        )
+    else:
+        resolved_path, subfolder = None, None
+
+    target = DesiredInstallTarget(
+        target_key=make_child_target_key(parent_target.target_key, resource_id),
+        resource_id=resource_id,
+        parent_id=parent_target.resource_id,
+        parent_target_key=parent_target.target_key,
+        root_resource_id=parent_target.root_resource_id,
+        target_kind="dependency",
+        sources={"dependency"},
+        title=None,
+        category_id=None,
+        resolved_path=resolved_path,
+        subfolder=subfolder,
+        flatten=_get_flatten(resource_id, parent_target.resource_id, settings_env),
+        protected_files=_get_protected_files(resource_id, settings_env),
+        explicit_root=False,
+    )
+    return desired_set.add_target(target)
+
+
+def _expand_dependencies(
+    desired_set: DesiredSet,
+    *,
+    parent_target: DesiredInstallTarget,
+    settings_env: str,
+) -> None:
+    """Recursively add all opt-in dependencies of a parent target to the desired set."""
+    settings_for_env = config.settings.from_env(settings_env)
+    parent_settings = settings_for_env.SPECIAL_RESOURCES.get(parent_target.resource_id, {})
+    dependencies = parent_settings.get("dependencies", {})
+
+    for dependency_id, dependency_info in dependencies.items():
+        info = dependency_info or {}
+        if not info.get("opt_in", False):
+            continue
+        dependency_id_str = str(dependency_id)
+        child_target = _add_dependency_target(
+            desired_set,
+            parent_target=parent_target,
+            resource_id=dependency_id_str,
+            settings_env=settings_env,
+        )
+        if dependency_id_str in parse_target_key(parent_target.target_key):
+            # Keep the repeated edge once so planning can block the cycle explicitly.
+            continue
+        _expand_dependencies(
+            desired_set,
+            parent_target=child_target,
+            settings_env=settings_env,
+        )
+
+
+async def discover_desired_set(
+    *,
+    client: httpx.AsyncClient,
+    resource_ids: list[str] | None,
+    settings_env: str,
+) -> DesiredSet:
+    """Main entry point for discovery: figure out everything that needs to be installed and where it goes."""
+    if resource_ids is None:
+        watched_resources, licenses = await asyncio.gather(
+            api.fetch_watched_resources(client),
+            api.fetch_licenses(client),
+        )
+        root_specs = _root_sources_for_full_sync(watched_resources, licenses, settings_env)
+        desired_set = DesiredSet(mode="full")
+    else:
+        normalized_ids = [str(resource_id) for resource_id in resource_ids]
+        root_specs = _root_sources_for_targeted(normalized_ids, settings_env)
+        desired_set = DesiredSet(
+            mode="targeted",
+            requested_root_ids=set(normalized_ids),
+        )
+
+    for resource_id, spec in root_specs.items():
+        root_target = _add_root_target(
+            desired_set,
+            resource_id=resource_id,
+            sources=spec.sources,
+            payload=spec.payload,
+            settings_env=settings_env,
+        )
+        if spec.discovery_block:
+            root_target.discovery_block = spec.discovery_block
+        _expand_dependencies(
+            desired_set,
+            parent_target=root_target,
+            settings_env=settings_env,
+        )
+
+    return desired_set