typekro 0.34.0 → 0.35.0

package/dist/alchemy/clickhouse-schema/runner.js

@@ -0,0 +1,613 @@
+/**
+ * Convergence logic for the `ClickHouseSchema` alchemy resource: fingerprinting,
+ * `clickhouse-client` command rendering, Ready-pod selection and statement execution.
+ *
+ * Kept free of alchemy and of `@kubernetes/client-node` so it can be exercised whole
+ * against an injected {@link ClickHouseExecutor}.
+ */
+import { createHash } from 'node:crypto';
+import { getComponentLogger } from '../../core/logging/index.js';
+import { ClickHouseSchemaError, parseClickHouseErrorCode, parseClickHouseExceptionName, redactClickHouseOutput, } from './types.js';
+/** The Altinity CHI server container name (see `clickHouseInstallation`'s pod template). */
+export const DEFAULT_CLICKHOUSE_CONTAINER = 'clickhouse';
+/** ClickHouse's native protocol port, which `clickhouse-client` speaks. */
+export const DEFAULT_CLICKHOUSE_PORT = 9000;
+export const DEFAULT_CLICKHOUSE_USER = 'default';
+export const DEFAULT_CLICKHOUSE_DATABASE = 'default';
+/**
+ * Default environment variable read for the password INSIDE the container.
+ *
+ * This is the convention every in-repo ClickHouse workload already uses: the ClickStack
+ * retention CronJob reads `CLICKHOUSE_PASSWORD`, and `clickHouseS3BackupCronJob` sets
+ * that same variable from the credentials Secret. `clickhouse-client` also honours
+ * `CLICKHOUSE_PASSWORD` natively, but this resource does NOT depend on that: it passes
+ * `--password "${VAR:-}"` explicitly through `sh -c`, which behaves identically for a
+ * custom `passwordEnv` and on images whose client build predates the native support.
+ */
+export const DEFAULT_CLICKHOUSE_PASSWORD_ENV = 'CLICKHOUSE_PASSWORD';
+export const DEFAULT_WAIT_FOR_POD_TIMEOUT_MS = 120_000;
+export const DEFAULT_STATEMENT_TIMEOUT_MS = 300_000;
+export const DEFAULT_MAX_ATTEMPTS = 3;
+export const DEFAULT_BACKOFF_MS = 1_000;
+/**
+ * How many times a `fanout` apply re-lists and applies to newly appeared pods before it
+ * gives up and fails. Three covers the ordinary races — a scale-out or a replacement
+ * landing mid-apply — without letting a cluster that is genuinely churning stretch one
+ * converge indefinitely.
+ */
+export const DEFAULT_MAX_RECONCILE_PASSES = 3;
+const POD_POLL_INTERVAL_MS = 2_000;
+export const defaultRuntimeDeps = {
+    now: () => Date.now(),
+    sleep: (ms) => new Promise((resolve) => {
+        setTimeout(resolve, ms);
+    }),
+};
+/** Single-quote a word for `sh -c`, closing and reopening around any embedded quote. */
+function shellQuote(value) {
+    return `'${value.replaceAll("'", `'\\''`)}'`;
+}
+export function resolveDatabase(config) {
+    return config.client?.database ?? DEFAULT_CLICKHOUSE_DATABASE;
+}
+export function resolveContainer(config) {
+    return config.target.container ?? DEFAULT_CLICKHOUSE_CONTAINER;
+}
+/**
+ * The `sh -c` command run inside the server container.
+ *
+ * The password is expanded from the container's OWN environment (`"${VAR:-}"`, a POSIX
+ * default expansion so an unset variable means the empty password the stock Altinity
+ * `default` user has) and is therefore never in props, never in alchemy state, never in
+ * this process's memory, and never echoed. `exec` replaces the shell so the client is
+ * the container's direct child and receives the statement on stdin unbuffered.
+ *
+ * `--host 127.0.0.1`: the exec already landed inside a server pod, so the connection
+ * never leaves it. There is no port-forward and no network path from the runner.
+ */
+export function renderClickHouseCommand(config) {
+    const user = config.client?.user ?? DEFAULT_CLICKHOUSE_USER;
+    const port = config.client?.port ?? DEFAULT_CLICKHOUSE_PORT;
+    const passwordEnv = config.client?.passwordEnv ?? DEFAULT_CLICKHOUSE_PASSWORD_ENV;
+    const settings = Object.entries(config.settings ?? {}).map(([name, value]) => `--${name}=${shellQuote(String(value))}`);
+    const client = [
+        'exec clickhouse-client',
+        '--host 127.0.0.1',
+        `--port ${port}`,
+        `--user ${shellQuote(user)}`,
+        `--database ${shellQuote(resolveDatabase(config))}`,
+        // POSIX default-value expansion: an unset variable means the empty password.
+        `--password "\${${passwordEnv}:-}"`,
+        ...settings,
+    ].join(' ');
+    return ['sh', '-c', client];
+}
+/**
+ * sha256 over the ordered statements, the settings, the client configuration and the
+ * execution model.
+ *
+ * The TARGET is deliberately NOT part of the fingerprint — it describes where to reach
+ * the server, not what is applied — so {@link needsApply} compares it separately and a
+ * re-pointed resource re-applies even though its DDL is byte-identical.
+ *
+ * The EXECUTION MODEL is part of it, because switching from `onCluster` to `fanout`
+ * changes which servers the identical statements reached, and that is a change to what is
+ * applied even though the SQL is untouched.
+ */
+export function computeFingerprint(config) {
+    const canonical = JSON.stringify({
+        statements: config.statements,
+        settings: Object.entries(config.settings ?? {})
+            .map(([name, value]) => [name, String(value)])
+            .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)),
+        client: {
+            user: config.client?.user ?? DEFAULT_CLICKHOUSE_USER,
+            passwordEnv: config.client?.passwordEnv ?? DEFAULT_CLICKHOUSE_PASSWORD_ENV,
+            database: resolveDatabase(config),
+            port: config.client?.port ?? DEFAULT_CLICKHOUSE_PORT,
+        },
+        execution: config.execution,
+    });
+    return createHash('sha256').update(canonical).digest('hex');
+}
+function runtimeDeps(context) {
+    return context.deps ?? defaultRuntimeDeps;
+}
+/**
+ * Whether a converge must (re-)run the statements against this target.
+ *
+ * Three things are compared, and the third is the one that is easy to forget: the
+ * fingerprint (what is applied), the target strings (where, within a cluster), and the
+ * CLUSTER ITSELF. Namespace, selector and container are just strings — `telemetry` +
+ * `chi=orders` names a pod in staging exactly as well as it names one in production — so
+ * without the cluster identity, re-pointing a resource at a second cluster matches the
+ * recorded target, matches the fingerprint, and silently applies nothing there.
+ *
+ * The live pod SET is deliberately not compared here: it needs an API call, so
+ * {@link applyClickHouseSchema} checks it separately and only under `fanout`, by NAME AND
+ * UID ({@link samePodIdentitySet}) so a same-name replacement is not invisible.
+ */
+export function needsApply(config, previous, clusterId) {
+    if (!previous)
+        return true;
+    if (previous.fingerprint !== computeFingerprint(config))
+        return true;
+    if (previous.clusterId !== clusterId)
+        return true;
+    return (JSON.stringify({
+        namespace: previous.target.namespace,
+        podSelector: previous.target.podSelector,
+        container: previous.target.container,
+    }) !==
+        JSON.stringify({
+            namespace: config.target.namespace,
+            podSelector: config.target.podSelector,
+            container: config.target.container,
+        }));
+}
+function selectorText(config) {
+    return Object.entries(config.target.podSelector)
+        .map(([key, value]) => `${key}=${value}`)
+        .join(',');
+}
+/**
+ * A pod whose container list is empty is a pod the transport could not describe, not a
+ * pod without containers; exec'ing into it and letting the API server object is a better
+ * failure than refusing it here on missing information.
+ */
+function hasContainer(pod, container) {
+    return pod.containers.length === 0 || pod.containers.includes(container);
+}
+/** Phases a pod never leaves. It cannot become Ready, so waiting for one is waiting forever. */
+const TERMINAL_PHASES = new Set(['Succeeded', 'Failed']);
+/**
+ * Whether a matching pod belongs to the set this converge is responsible for.
+ *
+ * Excluded are pods on their way out (`metadata.deletionTimestamp` set) and pods that have
+ * already finished (`Succeeded`/`Failed`). Both are matched by the selector and neither
+ * will ever serve a statement, so counting them would make `fanout` hang until its budget
+ * expired and then fail on a pod nobody was waiting for. Everything else — Ready, Pending,
+ * Running-but-not-Ready, phase unknown — is in the set and must become Ready.
+ */
+function isMatchingPod(pod) {
+    return pod.terminating !== true && !TERMINAL_PHASES.has(pod.phase ?? '');
+}
+/** `name: Ready` / `name: Pending, not Ready` — what a failure has to name to be actionable. */
+function describePod(pod) {
+    if (pod.ready)
+        return `${pod.name}: Ready`;
+    return `${pod.name}: ${pod.phase ?? 'phase unknown'}, not Ready`;
+}
+function byName(left, right) {
+    return left.name < right.name ? -1 : left.name > right.name ? 1 : 0;
+}
+/**
+ * One `list pods` call, reduced to the matching set and sorted by name.
+ *
+ * Sorted so the recorded pod set, the execution order and every error message are stable
+ * across converges.
+ */
+async function listMatchingPods(context) {
+    const pods = await context.executor.listPods(context.config.target.namespace, context.config.target.podSelector, context.abortSignal);
+    return [...pods].filter(isMatchingPod).sort(byName);
+}
+/**
+ * The mid-run empty set, in the words a reader needs to recognise the race.
+ *
+ * A set that goes empty after this run has already applied to pods is not convergence and
+ * is not "nothing matched": it is the window in which a pod has been deleted and its
+ * replacement has not yet appeared. Naming the count that WAS applied to, and the
+ * successor that would otherwise be missed, is what tells the reader which of the two
+ * empty-set situations they are looking at.
+ */
+function fanoutSetBecameEmptyClause(appliedSoFar) {
+    return (`the matching set became empty after ${appliedSoFar} pod(s) were applied; a same-named ` +
+        `successor would be unapplied`);
+}
+/**
+ * `fanout`: EVERY matching pod, or none at all.
+ *
+ * The rule this enforces is that a `fanout` converge is never partial. Taking whichever
+ * pods happen to be Ready is what makes a StatefulSet mid-rollout — one Ready replica and
+ * two Pending ones — record a successful, fingerprinted, cluster-wide apply after updating
+ * a single server; the remaining replicas then come up with no schema and the fingerprint
+ * says the work is done. So the whole matching set is enumerated first and each pod must
+ * become Ready within `waitForPod.timeoutMs`:
+ *
+ * - a pod that never becomes Ready fails the converge, naming the pods and their phases;
+ * - a pod without the requested container fails IMMEDIATELY rather than after the budget,
+ *   because no amount of waiting adds a container to a running pod.
+ *
+ * Failing is the conservative outcome: alchemy retries a failed converge, and the
+ * fingerprint is recorded only on success, so the next converge applies the full set. A
+ * recorded partial apply would never be retried at all.
+ *
+ * `deadline` lets {@link applyFanoutUntilCovered} spend ONE `waitForPod` budget across all
+ * of its reconcile passes rather than handing each pass a fresh one, so a set that keeps
+ * churning cannot stretch the converge without limit.
+ *
+ * `appliedSoFar` distinguishes the two ways the matching set can be empty. Empty from the
+ * start is "no pod matched the selector" — a selector or a namespace to fix. Empty AFTER
+ * this run applied to some pods is the replacement race {@link fanoutSetBecameEmptyClause}
+ * names, and it gets the same budget: a single-replica StatefulSet being replaced shows an
+ * empty set between the old pod disappearing and its same-named successor appearing, and
+ * the successor must be waited for and applied to rather than declared covered.
+ */
+async function selectFanoutPods(context, budgetDeadline, appliedSoFar = 0) {
+    const { config, resourceId } = context;
+    const deps = runtimeDeps(context);
+    const timeoutMs = config.waitForPod?.timeoutMs ?? DEFAULT_WAIT_FOR_POD_TIMEOUT_MS;
+    const container = resolveContainer(config);
+    const deadline = budgetDeadline ?? deps.now() + timeoutMs;
+    for (;;) {
+        const matching = await listMatchingPods(context);
+        const withoutContainer = matching.filter((pod) => !hasContainer(pod, container));
+        if (withoutContainer.length > 0) {
+            throw new ClickHouseSchemaError(`ClickHouseSchema '${resourceId}': execution.mode 'fanout' applies the statements to ` +
+                `EVERY pod matching ${selectorText(config)} in namespace ` +
+                `'${config.target.namespace}', and ${withoutContainer.length} of ${matching.length} ` +
+                `has no container '${container}' (` +
+                `${withoutContainer.map((pod) => `${pod.name}: ${pod.containers.join(', ')}`).join('; ')}` +
+                `). Set target.container, or narrow target.podSelector to the server pods.`, resourceId);
+        }
+        const notReady = matching.filter((pod) => !pod.ready);
+        if (matching.length > 0 && notReady.length === 0)
+            return matching;
+        if (deps.now() >= deadline) {
+            const state = matching.length === 0
+                ? appliedSoFar > 0
+                    ? fanoutSetBecameEmptyClause(appliedSoFar)
+                    : 'no pod matched the selector'
+                : `${notReady.length} of ${matching.length} pod(s) were still not Ready ` +
+                    `(${notReady.map(describePod).join('; ')})`;
+            throw new ClickHouseSchemaError(`ClickHouseSchema '${resourceId}': execution.mode 'fanout' applies the statements to ` +
+                `EVERY pod matching ${selectorText(config)} in namespace ` +
+                `'${config.target.namespace}' and never to a subset, but after ${timeoutMs}ms ` +
+                `${state}. Let the rollout finish, raise waitForPod.timeoutMs, or switch to ` +
+                `execution.mode 'onCluster'.`, resourceId);
+        }
+        await deps.sleep(Math.min(POD_POLL_INTERVAL_MS, Math.max(0, deadline - deps.now())));
+    }
+}
+/**
+ * `onCluster`: the ONE pod that initiates the distributed DDL.
+ *
+ * The opposite trade-off to {@link selectFanoutPods}, and deliberately so: the statements
+ * distribute themselves through Keeper's DDL queue, so a pod that is still starting is not
+ * a pod this converge has to wait for — the server it eventually becomes picks the DDL up
+ * from the queue. Only one usable Ready pod is needed, which is what makes `onCluster` the
+ * right mode on a large cluster where some replica is almost always rolling.
+ *
+ * EVERY Ready pod is considered, not just the first: a rollout can leave a Ready pod whose
+ * container set does not match — a sidecar-injected replica, a pod from an older template —
+ * and judging by the first one throws away perfectly good initiators behind it.
+ */
+async function selectInitiatorPod(context) {
+    const { config, resourceId } = context;
+    const deps = runtimeDeps(context);
+    const timeoutMs = config.waitForPod?.timeoutMs ?? DEFAULT_WAIT_FOR_POD_TIMEOUT_MS;
+    const container = resolveContainer(config);
+    const deadline = deps.now() + timeoutMs;
+    let lastSeen = 0;
+    for (;;) {
+        const matching = await listMatchingPods(context);
+        lastSeen = matching.length;
+        const ready = matching.filter((pod) => pod.ready);
+        const initiator = ready.find((pod) => hasContainer(pod, container));
+        if (initiator)
+            return [initiator];
+        if (ready.length > 0) {
+            throw new ClickHouseSchemaError(`ClickHouseSchema '${resourceId}': no Ready pod in namespace ` +
+                `'${config.target.namespace}' has a container '${container}' (` +
+                `${ready.map((pod) => `${pod.name}: ${pod.containers.join(', ')}`).join('; ')}). ` +
+                `Set target.container.`, resourceId);
+        }
+        if (deps.now() >= deadline) {
+            throw new ClickHouseSchemaError(`ClickHouseSchema '${resourceId}': no Ready pod matched ${selectorText(config)} in ` +
+                `namespace '${config.target.namespace}' within ${timeoutMs}ms (${lastSeen} pod(s) matched ` +
+                `the selector but none were Ready).`, resourceId);
+        }
+        await deps.sleep(Math.min(POD_POLL_INTERVAL_MS, Math.max(0, deadline - deps.now())));
+    }
+}
+/**
+ * Poll until the server pods the execution model needs are Ready, or the budget runs out.
+ *
+ * A ClickHouse server accepts connections only once it is Ready, and a CHI rollout has a
+ * window where pods exist but are still replaying logs — exec'ing then produces a
+ * connection-refused that looks like a SQL failure. Waiting for readiness first is what
+ * makes "ordered after the instance is ready" true in practice as well as in the
+ * dependency graph.
+ *
+ * WHICH pods have to be Ready is the execution model's whole difference: `fanout` reaches
+ * every server itself and so needs all of them ({@link selectFanoutPods}); `onCluster`
+ * hands the statements to Keeper and so needs exactly one ({@link selectInitiatorPod}).
+ */
+export async function selectExecutionPods(context) {
+    return context.config.execution.mode === 'fanout'
+        ? await selectFanoutPods(context)
+        : await selectInitiatorPod(context);
+}
+/**
+ * Run one statement on one pod, retrying only TRANSIENT transport failures.
+ *
+ * The two failure modes are kept strictly apart (see {@link ClickHouseExecutor}): a
+ * rejected exec is the transport, a non-zero exit code is the server. Re-issuing a
+ * statement the server actively rejected cannot help and, for a statement that is not
+ * perfectly idempotent, can compound the damage.
+ */
+async function runStatement(context, podName, statement, index) {
+    const { config, resourceId } = context;
+    const deps = runtimeDeps(context);
+    const maxAttempts = config.retry?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
+    const backoffMs = config.retry?.backoffMs ?? DEFAULT_BACKOFF_MS;
+    const command = {
+        namespace: config.target.namespace,
+        podName,
+        container: resolveContainer(config),
+        command: renderClickHouseCommand(config),
+        stdin: `${statement.trim()}\n`,
+        timeoutMs: config.statementTimeoutMs ?? DEFAULT_STATEMENT_TIMEOUT_MS,
+    };
+    let lastTransport;
+    for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+        let result;
+        try {
+            result = await context.executor.exec(command, context.abortSignal);
+        }
+        catch (error) {
+            lastTransport = error instanceof Error ? error : new Error(String(error));
+            if (attempt === maxAttempts)
+                break;
+            await deps.sleep(backoffMs * attempt);
+            continue;
+        }
+        if (result.exitCode === 0)
+            return;
+        // A server-side failure. The code and exception class are parsed from the RAW output
+        // — before redaction, which may well blank the very line that carries them — and the
+        // message is then redacted against the statement that was submitted.
+        const raw = `${result.stderr}\n${result.stdout}`.trim();
+        const code = parseClickHouseErrorCode(raw);
+        const exception = parseClickHouseExceptionName(raw);
+        throw new ClickHouseSchemaError(`ClickHouseSchema '${resourceId}': statement ${index} failed on pod ${podName}` +
+            `${code === undefined ? '' : ` with ClickHouse code ${code}`}` +
+            `${exception === undefined ? '' : ` (${exception})`} ` +
+            `(exit ${result.exitCode}).`, resourceId, index, code, redactClickHouseOutput(raw, statement), exception);
+    }
+    throw new ClickHouseSchemaError(`ClickHouseSchema '${resourceId}': exec transport failed for statement ${index} on pod ` +
+        `${podName} after ${maxAttempts} attempt(s): ` +
+        redactClickHouseOutput(lastTransport?.message ?? 'unknown error', statement), resourceId, index, undefined, undefined, undefined, lastTransport ? { cause: lastTransport } : undefined);
+}
+/**
+ * Run an ordered statement list against every selected pod, stopping at the first failure.
+ *
+ * Per POD, not per statement: each server receives the whole list in order, because the
+ * order is what makes the list meaningful (a table cannot be created before its database)
+ * and interleaving across pods would only make a partial failure harder to read.
+ */
+export async function runStatements(context, statements) {
+    const pods = await selectExecutionPods(context);
+    for (const pod of pods) {
+        await runStatementsOnPod(context, pod, statements);
+    }
+    return { podNames: pods.map((pod) => pod.name), pods };
+}
+/** The whole ordered list against ONE pod, in statement order. */
+async function runStatementsOnPod(context, pod, statements) {
+    for (const [index, statement] of statements.entries()) {
+        await runStatement(context, pod.name, statement, index);
+    }
+}
+/**
+ * The identity of one pod, for the set comparison: NAME AND UID.
+ *
+ * The name alone is not an identity — a StatefulSet replica that is deleted and recreated
+ * comes back as `chi-orders-0-0-0` with an empty disk and no schema — so a replacement is
+ * invisible to a set of names. `metadata.uid` is unique per pod object and never reused,
+ * which is exactly the distinction that was missing. The NUL separator keeps a name that
+ * happens to contain the delimiter from colliding with a UID.
+ */
+function podKey(pod) {
+    return `${pod.name}\u0000${pod.uid ?? ''}`;
+}
+/** What goes into state for one applied pod. */
+function appliedPod(pod) {
+    return { name: pod.name, ...(pod.uid !== undefined ? { uid: pod.uid } : {}) };
+}
+/** Set equality over two key lists, order-insensitively. */
+function sameKeySet(left, right) {
+    if (left.length !== right.length)
+        return false;
+    const sortedLeft = [...left].sort();
+    const sortedRight = [...right].sort();
+    return sortedLeft.every((key, index) => key === sortedRight[index]);
+}
+/**
+ * Whether the live pod set is the one the recorded state covers, by NAME AND UID.
+ *
+ * State written before UIDs were recorded has only names, and comparing a name-only record
+ * against UID-bearing live pods would re-apply on every converge forever. Such state falls
+ * back to the name comparison; the apply it eventually does rewrites state in the new
+ * shape, and the UID guarantee starts from there.
+ */
+function samePodIdentitySet(live, previous) {
+    if (previous.pods === undefined) {
+        return sameKeySet(live.map((pod) => pod.name), previous.podNames);
+    }
+    return sameKeySet(live.map(podKey), previous.pods.map(podKey));
+}
+/**
+ * THE EXIT PREDICATE of a `fanout` apply — the single condition under which a pod set may
+ * be recorded as successfully applied. All three parts are load-bearing:
+ *
+ * 1. `observed` is NON-EMPTY. An empty observation is never convergence: "no pod is
+ *    uncovered" is vacuously true of the empty set, so a run that applied to a pod which
+ *    then disappeared would otherwise commit `pods: []` as success. During a single-replica
+ *    StatefulSet replacement that empty window sits exactly between the old pod going away
+ *    and its SAME-NAMED successor arriving, and the successor — a new pod object with an
+ *    empty disk — would be left unapplied behind a fingerprint that says the work is done.
+ * 2. every pod in `observed` is in `applied`, by NAME AND UID ({@link podKey}). Coverage is
+ *    of pod OBJECTS; a replacement wearing the applied pod's name is not covered by it.
+ * 3. `observed` equals the observation BEFORE it. The two preceding parts describe a single
+ *    snapshot, and a snapshot cannot distinguish a settled set from one that is still
+ *    moving: a pod that vanished between the selection and the re-list leaves every
+ *    remaining pod covered while the set itself is mid-change. Requiring two consecutive
+ *    identical observations is what makes the set, and not merely the last glimpse of it,
+ *    the thing that is recorded.
+ *
+ * The cost of part 3 on a settled cluster is nothing: the selection's list and the re-list
+ * are the two observations, so one pass still converges.
+ */
+function fanoutCovered(observed, previous, applied) {
+    if (observed.length === 0)
+        return false;
+    if (!observed.every((pod) => applied.has(podKey(pod))))
+        return false;
+    return sameKeySet(observed.map(podKey), previous.map(podKey));
+}
+/** Which part of {@link fanoutCovered} an observation failed, in one readable clause. */
+function unconvergedReason(observed, previous, uncovered, appliedTotal) {
+    if (observed.length === 0)
+        return fanoutSetBecameEmptyClause(appliedTotal);
+    if (uncovered.length > 0) {
+        return (`${uncovered.length} pod(s) had still not been applied to ` +
+            `(${uncovered.map((pod) => pod.name).join(', ')})`);
+    }
+    return (`the matching set was still moving between two consecutive observations ` +
+        `(${previous.map((pod) => pod.name).join(', ')} then ` +
+        `${observed.map((pod) => pod.name).join(', ')})`);
+}
+/**
+ * `fanout`: apply, re-list, apply again — until the LIVE pod set is covered, or fail.
+ *
+ * THE SCALE RACE. Selection sees one matching set; a replica can be added, replaced or
+ * removed while the statements are still running, so the set that is live when the run
+ * finishes is not necessarily the set that was applied to. Recording the live set instead
+ * would claim coverage of a pod nothing ran on — and that claim is never revisited,
+ * because it makes the two sets agree. Recording only what was reached and WARNING about
+ * the difference does not fix it either: alchemy commits that state without another
+ * reconcile, so the newly observed pod stays unapplied until some future deployment
+ * happens to change the fingerprint or the set. A converge that observed an uncovered pod
+ * and returned success is exactly the half-applied schema `fanout` exists to rule out.
+ *
+ * So the loop keeps going instead:
+ *
+ * 1. select the complete Ready set (the all-or-nothing rules of {@link selectFanoutPods});
+ * 2. apply the whole ordered list to every pod not yet applied to IN THIS RUN — a pod
+ *    already covered by an earlier pass is not re-run, so a settled set costs one pass;
+ * 3. re-list. Pods that appeared are uncovered and get another pass; pods that DISAPPEARED
+ *    are dropped from the recorded set, because state must describe coverage of pods that
+ *    exist rather than of ones that are gone;
+ * 4. repeat until {@link fanoutCovered} — the ONE exit predicate — holds.
+ *
+ * Two bounds keep a genuinely churning cluster from looping forever: `maxReconcilePasses`
+ * and the overall `waitForPod` budget, which is spent ACROSS the passes rather than renewed
+ * by each one. Hitting either before the predicate holds FAILS the converge, naming what is
+ * wrong with the observation ({@link unconvergedReason}) — never returns a successful state
+ * — so alchemy does not commit a partial apply and the next converge starts over. Failing is
+ * the recoverable outcome; a recorded partial apply would never be retried at all.
+ */
+async function applyFanoutUntilCovered(context, statements) {
+    const { config, resourceId } = context;
+    const deps = runtimeDeps(context);
+    const maxPasses = config.maxReconcilePasses ?? DEFAULT_MAX_RECONCILE_PASSES;
+    const timeoutMs = config.waitForPod?.timeoutMs ?? DEFAULT_WAIT_FOR_POD_TIMEOUT_MS;
+    const budgetDeadline = deps.now() + timeoutMs;
+    const logger = getComponentLogger('alchemy-clickhouse-schema');
+    /** Pods this run has applied to, by {@link podKey}; pruned to what is still live. */
+    const applied = new Map();
+    /** How many pods this run has applied to IN TOTAL, including ones since departed. */
+    let appliedTotal = 0;
+    for (let pass = 1;; pass += 1) {
+        // The selection's own final list is the FIRST of the two observations the exit
+        // predicate compares; the re-list below is the second.
+        const selected = await selectFanoutPods(context, budgetDeadline, appliedTotal);
+        for (const pod of selected) {
+            if (applied.has(podKey(pod)))
+                continue;
+            await runStatementsOnPod(context, pod, statements);
+            applied.set(podKey(pod), pod);
+            appliedTotal += 1;
+        }
+        const live = await listMatchingPods(context);
+        const liveKeys = new Set(live.map(podKey));
+        for (const key of [...applied.keys()]) {
+            if (!liveKeys.has(key))
+                applied.delete(key);
+        }
+        if (fanoutCovered(live, selected, applied))
+            return [...applied.values()].sort(byName);
+        const uncovered = live.filter((pod) => !applied.has(podKey(pod)));
+        if (pass >= maxPasses) {
+            throw new ClickHouseSchemaError(`ClickHouseSchema '${resourceId}': execution.mode 'fanout' applies the statements to ` +
+                `EVERY pod matching ${selectorText(config)} in namespace ` +
+                `'${config.target.namespace}', but the pod set kept changing: after ${maxPasses} ` +
+                `reconcile pass(es), ${unconvergedReason(live, selected, uncovered, appliedTotal)}. ` +
+                `Nothing is recorded, so the next converge re-applies the whole list. Let the ` +
+                `rollout settle, raise maxReconcilePasses, or switch to execution.mode 'onCluster'.`, resourceId);
+        }
+        logger.info('ClickHouse server pod set changed while the schema was being applied; reconciling again', {
+            resourceId,
+            pass,
+            maxPasses,
+            appliedTo: [...applied.values()].map((pod) => pod.name),
+            uncovered: uncovered.map((pod) => pod.name),
+            reason: unconvergedReason(live, selected, uncovered, appliedTotal),
+        });
+    }
+}
+/**
+ * Converge the schema: no-op when nothing changed, otherwise re-run EVERY statement.
+ *
+ * There is no partial application. A fingerprint change re-runs the whole ordered list,
+ * which is only correct because every statement is required to be idempotent — and it is
+ * also what makes a failed converge recoverable: the fingerprint is recorded only after
+ * the last statement succeeds, so a run that dies at statement 7 re-runs 0..6 next time.
+ *
+ * Under `fanout` the POD SET is part of what "nothing changed" means, and the set is
+ * compared by NAME AND UID. A statement list applied to two replicas is not applied to the
+ * third one that a scale-out added, nor to the replacement a drain put back under the same
+ * name with an empty disk, and the fingerprint cannot see either — so an otherwise-unchanged
+ * converge still lists pods and re-applies when the set moved. That listing is one API call
+ * and no exec, so an unchanged, unchanged-topology converge stays free. When it does apply,
+ * it applies until the live set is COVERED; see {@link applyFanoutUntilCovered}.
+ *
+ * `onCluster` has no coverage to reconcile: one execution hands the DDL to Keeper's queue,
+ * which reaches the pods this converge never looked at, including ones that appear later.
+ */
+export async function applyClickHouseSchema(context, previous) {
+    const { config } = context;
+    const deps = runtimeDeps(context);
+    if (previous && !needsApply(config, previous, context.clusterId)) {
+        if (config.execution.mode !== 'fanout')
+            return previous;
+        const live = await selectExecutionPods(context);
+        if (samePodIdentitySet(live, previous))
+            return previous;
+    }
+    const pods = config.execution.mode === 'fanout'
+        ? await applyFanoutUntilCovered(context, config.statements)
+        : (await runStatements(context, config.statements)).pods;
+    return {
+        fingerprint: computeFingerprint(config),
+        appliedAt: new Date(deps.now()).toISOString(),
+        statementCount: config.statements.length,
+        database: resolveDatabase(config),
+        target: config.target,
+        podNames: pods.map((pod) => pod.name),
+        pods: pods.map(appliedPod),
+        ...(context.clusterId !== undefined ? { clusterId: context.clusterId } : {}),
+    };
+}
+/**
+ * Teardown. `retain` touches nothing at all — not even the cluster — so a destroyed
+ * stack leaves the data and the schema exactly as they were.
+ */
+export async function deleteClickHouseSchema(context) {
+    const statements = context.config.deleteStatements ?? [];
+    if (context.config.onDelete !== 'run' || statements.length === 0)
+        return;
+    await runStatements(context, statements);
+}
+//# sourceMappingURL=runner.js.map

package/dist/alchemy/clickhouse-schema/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../../src/alchemy/clickhouse-schema/runner.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,MAAM,6BAA6B,CAAC;AACjE,OAAO,EAKL,qBAAqB,EAErB,wBAAwB,EACxB,4BAA4B,EAC5B,sBAAsB,GACvB,MAAM,YAAY,CAAC;AAEpB,4FAA4F;AAC5F,MAAM,CAAC,MAAM,4BAA4B,GAAG,YAAY,CAAC;AAEzD,2EAA2E;AAC3E,MAAM,CAAC,MAAM,uBAAuB,GAAG,IAAI,CAAC;AAE5C,MAAM,CAAC,MAAM,uBAAuB,GAAG,SAAS,CAAC;AACjD,MAAM,CAAC,MAAM,2BAA2B,GAAG,SAAS,CAAC;AAErD;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,+BAA+B,GAAG,qBAAqB,CAAC;AAErE,MAAM,CAAC,MAAM,+BAA+B,GAAG,OAAO,CAAC;AACvD,MAAM,CAAC,MAAM,4BAA4B,GAAG,OAAO,CAAC;AACpD,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC;AACtC,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,CAAC;AAExC;;;;;GAKG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAG,CAAC,CAAC;AAC9C,MAAM,oBAAoB,GAAG,KAAK,CAAC;AAQnC,MAAM,CAAC,MAAM,kBAAkB,GAAgC;IAC7D,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE;IACrB,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,CACZ,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QACtB,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAC1B,CAAC,CAAC;CACL,CAAC;AAEF,wFAAwF;AACxF,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC;AAC/C,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,MAA8B;IAC5D,OAAO,MAAM,CAAC,MAAM,EAAE,QAAQ,IAAI,2BAA2B,CAAC;AAChE,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAA8B;IAC7D,OAAO,MAAM,CAAC,MAAM,CAAC,SAAS,IAAI,4BAA4B,CAAC;AACjE,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,uBAAuB,CAAC,MAA8B;IACpE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,IAAI,uBAAuB,CAAC;IAC5D,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,IAAI,uBAAuB,CAAC;IAC5D,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,WAAW,IAAI,+BAA+B,CAAC;IAClF,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CACxD,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,IAAI,IAAI,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAC5D,CAAC;IACF,MAAM,MAAM,GAAG;QACb,wBAAwB;QACxB,kBAAkB;QAClB,UAAU,IAAI,EAAE;QAChB,UAAU,UAAU,CAAC,IAAI,CAAC,EAAE;QAC5B,cAAc,UAAU,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE;QACnD,6EAA6E;QAC7E,kBAAkB,WAAW,MAAM;QACnC,GAAG,QAAQ;KACZ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACZ,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;AAC9B,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAA8B;IAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAC/B,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,QAAQ,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC;aAC5C,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAoB,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;aAC/D,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,EAAE;YACN,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,IAAI,uBAAuB;YACpD,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,IAAI,+BAA+B;YAC1E,QAAQ,EAAE,eAAe,CAAC,MAAM,CAAC;YACjC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,IAAI,uBAAuB;SACrD;QACD,SAAS,EAAE,MAAM,CAAC,SAAS;KAC5B,CAAC,CAAC;IACH,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC9D,CAAC;AAqBD,SAAS,WAAW,CAAC,OAAmC;IACtD,OAAO,OAAO,CAAC,IAAI,IAAI,kBAAkB,CAAC;AAC5C,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,UAAU,CACxB,MAA8B,EAC9B,QAA2C,EAC3C,SAAkB;IAElB,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3B,IAAI,QAAQ,CAAC,WAAW,KAAK,kBAAkB,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IACrE,IAAI,QAAQ,CAAC,SAAS,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IAClD,OAAO,CACL,IAAI,CAAC,SAAS,CAAC;QACb,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,SAAS;QACpC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,WAAW;QACxC,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,SAAS;KACrC,CAAC;QACF,IAAI,CAAC,SAAS,CAAC;YACb,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS;YAClC,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW;YACtC,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS;SACnC,CAAC,CACH,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,MAA8B;IAClD,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC;SAC7C,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC;SACxC,IAAI,CAAC,GAAG,CAAC,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,SAAS,YAAY,CAAC,GAAyB,EAAE,SAAiB;IAChE,OAAO,GAAG,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;AAC3E,CAAC;AAED,gGAAgG;AAChG,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC;AAEzD;;;;;;;;GAQG;AACH,SAAS,aAAa,CAAC,GAAyB;IAC9C,OAAO,GAAG,CAAC,WAAW,KAAK,IAAI,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;AAC3E,CAAC;AAED,gGAAgG;AAChG,SAAS,WAAW,CAAC,GAAyB;IAC5C,IAAI,GAAG,CAAC,KAAK;QAAE,OAAO,GAAG,GAAG,CAAC,IAAI,SAAS,CAAC;IAC3C,OAAO,GAAG,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,KAAK,IAAI,eAAe,aAAa,CAAC;AACnE,CAAC;AAED,SAAS,MAAM,CAAC,IAA0B,EAAE,KAA2B;IACrE,OAAO,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACtE,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,gBAAgB,CAC7B,OAAmC;IAEnC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAC1C,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,EAC/B,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EACjC,OAAO,CAAC,WAAW,CACpB,CAAC;IACF,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACtD,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,0BAA0B,CAAC,YAAoB;IACtD,OAAO,CACL,uCAAuC,YAAY,qCAAqC;QACxF,8BAA8B,CAC/B,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,KAAK,UAAU,gBAAgB,CAC7B,OAAmC,EACnC,cAAuB,EACvB,YAAY,GAAG,CAAC;IAEhB,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IACvC,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IAClC,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,EAAE,SAAS,IAAI,+BAA+B,CAAC;IAClF,MAAM,SAAS,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,cAAc,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IAE1D,SAAS,CAAC;QACR,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,CAAC;QAEjD,MAAM,gBAAgB,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC,CAAC;QACjF,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,qBAAqB,CAC7B,qBAAqB,UAAU,uDAAuD;gBACpF,sBAAsB,YAAY,CAAC,MAAM,CAAC,gBAAgB;gBAC1D,IAAI,MAAM,CAAC,MAAM,CAAC,SAAS,UAAU,gBAAgB,CAAC,MAAM,OAAO,QAAQ,CAAC,MAAM,GAAG;gBACrF,qBAAqB,SAAS,KAAK;gBACnC,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC1F,2EAA2E,EAC7E,UAAU,CACX,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACtD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,QAAQ,CAAC;QAElE,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,KAAK,GACT,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACnB,CAAC,CAAC,YAAY,GAAG,CAAC;oBAChB,CAAC,CAAC,0BAA0B,CAAC,YAAY,CAAC;oBAC1C,CAAC,CAAC,6BAA6B;gBACjC,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,OAAO,QAAQ,CAAC,MAAM,+BAA+B;oBACvE,IAAI,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;YAClD,MAAM,IAAI,qBAAqB,CAC7B,qBAAqB,UAAU,uDAAuD;gBACpF,sBAAsB,YAAY,CAAC,MAAM,CAAC,gBAAgB;gBAC1D,IAAI,MAAM,CAAC,MAAM,CAAC,SAAS,sCAAsC,SAAS,KAAK;gBAC/E,GAAG,KAAK,qEAAqE;gBAC7E,6BAA6B,EAC/B,UAAU,CACX,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,oBAAoB,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IACvF,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,KAAK,UAAU,kBAAkB,CAC/B,OAAmC;IAEnC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IACvC,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IAClC,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,EAAE,SAAS,IAAI,+BAA+B,CAAC;IAClF,MAAM,SAAS,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IACxC,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,SAAS,CAAC;QACR,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,CAAC;QACjD,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC3B,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAClD,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC,CAAC;QAEpE,IAAI,SAAS;YAAE,OAAO,CAAC,SAAS,CAAC,CAAC;QAClC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,IAAI,qBAAqB,CAC7B,qBAAqB,UAAU,+BAA+B;gBAC5D,IAAI,MAAM,CAAC,MAAM,CAAC,SAAS,sBAAsB,SAAS,KAAK;gBAC/D,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK;gBAClF,uBAAuB,EACzB,UAAU,CACX,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,IAAI,qBAAqB,CAC7B,qBAAqB,UAAU,2BAA2B,YAAY,CAAC,MAAM,CAAC,MAAM;gBAClF,cAAc,MAAM,CAAC,MAAM,CAAC,SAAS,YAAY,SAAS,OAAO,QAAQ,kBAAkB;gBAC3F,oCAAoC,EACtC,UAAU,CACX,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,oBAAoB,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IACvF,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,OAAmC;IAEnC,OAAO,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,KAAK,QAAQ;QAC/C,CAAC,CAAC,MAAM,gBAAgB,CAAC,OAAO,CAAC;QACjC,CAAC,CAAC,MAAM,kBAAkB,CAAC,OAAO,CAAC,CAAC;AACxC,CAAC;AAED;;;;;;;GAOG;AACH,KAAK,UAAU,YAAY,CACzB,OAAmC,EACnC,OAAe,EACf,SAAiB,EACjB,KAAa;IAEb,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IACvC,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IAClC,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,EAAE,WAAW,IAAI,oBAAoB,CAAC;IACtE,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,EAAE,SAAS,IAAI,kBAAkB,CAAC;IAChE,MAAM,OAAO,GAAG;QACd,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS;QAClC,OAAO;QACP,SAAS,EAAE,gBAAgB,CAAC,MAAM,CAAC;QACnC,OAAO,EAAE,uBAAuB,CAAC,MAAM,CAAC;QACxC,KAAK,EAAE,GAAG,SAAS,CAAC,IAAI,EAAE,IAAI;QAC9B,SAAS,EAAE,MAAM,CAAC,kBAAkB,IAAI,4BAA4B;KACrE,CAAC;IAEF,IAAI,aAAgC,CAAC;IACrC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,WAAW,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;QAC3D,IAAI,MAAuD,CAAC;QAC5D,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QACrE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,aAAa,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAC1E,IAAI,OAAO,KAAK,WAAW;gBAAE,MAAM;YACnC,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,OAAO,CAAC,CAAC;YACtC,SAAS;QACX,CAAC;QAED,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC;YAAE,OAAO;QAElC,qFAAqF;QACrF,qFAAqF;QACrF,qEAAqE;QACrE,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;QACxD,MAAM,IAAI,GAAG,wBAAwB,CAAC,GAAG,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,4BAA4B,CAAC,GAAG,CAAC,CAAC;QACpD,MAAM,IAAI,qBAAqB,CAC7B,qBAAqB,UAAU,gBAAgB,KAAK,kBAAkB,OAAO,EAAE;YAC7E,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,yBAAyB,IAAI,EAAE,EAAE;YAC9D,GAAG,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,SAAS,GAAG,GAAG;YACtD,SAAS,MAAM,CAAC,QAAQ,IAAI,EAC9B,UAAU,EACV,KAAK,EACL,IAAI,EACJ,sBAAsB,CAAC,GAAG,EAAE,SAAS,CAAC,EACtC,SAAS,CACV,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,qBAAqB,CAC7B,qBAAqB,UAAU,0CAA0C,KAAK,UAAU;QACtF,GAAG,OAAO,UAAU,WAAW,eAAe;QAC9C,sBAAsB,CAAC,aAAa,EAAE,OAAO,IAAI,eAAe,EAAE,SAAS,CAAC,EAC9E,UAAU,EACV,KAAK,EACL,SAAS,EACT,SAAS,EACT,SAAS,EACT,aAAa,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,SAAS,CACrD,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAmC,EACnC,UAA6B;IAK7B,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAChD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,kBAAkB,CAAC,OAAO,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC;AACzD,CAAC;AAED,kEAAkE;AAClE,KAAK,UAAU,kBAAkB,CAC/B,OAAmC,EACnC,GAAyB,EACzB,UAA6B;IAE7B,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;QACtD,MAAM,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;IAC1D,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,MAAM,CAAC,GAAsD;IACpE,OAAO,GAAG,GAAG,CAAC,IAAI,SAAS,GAAG,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC;AAC7C,CAAC;AAED,gDAAgD;AAChD,SAAS,UAAU,CAAC,GAAyB;IAC3C,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;AAChF,CAAC;AAED,4DAA4D;AAC5D,SAAS,UAAU,CAAC,IAAuB,EAAE,KAAwB;IACnE,IAAI,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC/C,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;IACpC,MAAM,WAAW,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;IACtC,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;AACtE,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,kBAAkB,CACzB,IAAqC,EACrC,QAA+B;IAE/B,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAChC,OAAO,UAAU,CACf,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAC3B,QAAQ,CAAC,QAAQ,CAClB,CAAC;IACJ,CAAC;IACD,OAAO,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;AACjE,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,SAAS,aAAa,CACpB,QAAyC,EACzC,QAAyC,EACzC,OAAkD;IAElD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC;IACrE,OAAO,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;AAChE,CAAC;AAED,yFAAyF;AACzF,SAAS,iBAAiB,CACxB,QAAyC,EACzC,QAAyC,EACzC,SAA0C,EAC1C,YAAoB;IAEpB,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,0BAA0B,CAAC,YAAY,CAAC,CAAC;IAC3E,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,OAAO,CACL,GAAG,SAAS,CAAC,MAAM,wCAAwC;YAC3D,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CACnD,CAAC;IACJ,CAAC;IACD,OAAO,CACL,yEAAyE;QACzE,IAAI,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ;QACtD,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CACjD,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,KAAK,UAAU,uBAAuB,CACpC,OAAmC,EACnC,UAA6B;IAE7B,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IACvC,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IAClC,MAAM,SAAS,GAAG,MAAM,CAAC,kBAAkB,IAAI,4BAA4B,CAAC;IAC5E,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,EAAE,SAAS,IAAI,+BAA+B,CAAC;IAClF,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IAC9C,MAAM,MAAM,GAAG,kBAAkB,CAAC,2BAA2B,CAAC,CAAC;IAC/D,qFAAqF;IACrF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAgC,CAAC;IACxD,qFAAqF;IACrF,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,IAAI,IAAI,GAAG,CAAC,GAAI,IAAI,IAAI,CAAC,EAAE,CAAC;QAC/B,+EAA+E;QAC/E,uDAAuD;QACvD,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,CAAC,CAAC;QAC/E,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAAE,SAAS;YACvC,MAAM,kBAAkB,CAAC,OAAO,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC;YACnD,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;YAC9B,YAAY,IAAI,CAAC,CAAC;QACpB,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,CAAC;QAC7C,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QAC3C,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;YACtC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC9C,CAAC;QAED,IAAI,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC;YAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEtF,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAClE,IAAI,IAAI,IAAI,SAAS,EAAE,CAAC;YACtB,MAAM,IAAI,qBAAqB,CAC7B,qBAAqB,UAAU,uDAAuD;gBACpF,sBAAsB,YAAY,CAAC,MAAM,CAAC,gBAAgB;gBAC1D,IAAI,MAAM,CAAC,MAAM,CAAC,SAAS,2CAA2C,SAAS,GAAG;gBAClF,uBAAuB,iBAAiB,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,YAAY,CAAC,IAAI;gBACrF,+EAA+E;gBAC/E,oFAAoF,EACtF,UAAU,CACX,CAAC;QACJ,CAAC;QAED,MAAM,CAAC,IAAI,CACT,yFAAyF,EACzF;YACE,UAAU;YACV,IAAI;YACJ,SAAS;YACT,SAAS,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC;YACvD,SAAS,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC;YAC3C,MAAM,EAAE,iBAAiB,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,YAAY,CAAC;SACnE,CACF,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,OAAmC,EACnC,QAA2C;IAE3C,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAC3B,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IAElC,IAAI,QAAQ,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QACjE,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,QAAQ,CAAC;QACxD,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAChD,IAAI,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC;YAAE,OAAO,QAAQ,CAAC;IAC1D,CAAC;IAED,MAAM,IAAI,GACR,MAAM,CAAC,SAAS,CAAC,IAAI,KAAK,QAAQ;QAChC,CAAC,CAAC,MAAM,uBAAuB,CAAC,OAAO,EAAE,MAAM,CAAC,UAAU,CAAC;QAC3D,CAAC,CAAC,CAAC,MAAM,aAAa,CAAC,OAAO,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;IAE7D,OAAO;QACL,WAAW,EAAE,kBAAkB,CAAC,MAAM,CAAC;QACvC,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,WAAW,EAAE;QAC7C,cAAc,EAAE,MAAM,CAAC,UAAU,CAAC,MAAM;QACxC,QAAQ,EAAE,eAAe,CAAC,MAAM,CAAC;QACjC,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC;QACrC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC;QAC1B,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC7E,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,OAAmC;IAC9E,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,gBAAgB,IAAI,EAAE,CAAC;IACzD,IAAI,OAAO,CAAC,MAAM,CAAC,QAAQ,KAAK,KAAK,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IACzE,MAAM,aAAa,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;AAC3C,CAAC"}