solid-objects 0.16.0 → 0.17.0

package/dist/runtime.js

@@ -1,3 +1,5 @@
+import { actorDiagnostics } from "./diagnostics.js";
+import { telemetryEvent, deliverTelemetry } from "./telemetry.js";
 import { BroadcastWorker } from "./broadcast-worker.js";
 import { buildSettings, broadcastsEnabled, validateComponent, } from "./configuration.js";
 import { currentActor, withActorContext, withActorProjection, withApplicationWritesForbidden, } from "./context.js";
@@ -60,6 +62,7 @@ export class SolidObjectsRuntime {
     realtime;
     processes;
     administration;
+    observers = new Set();
     registry = new Map();
     effects = new Map();
     commitActions = new Map();
@@ -580,7 +583,7 @@ export class SolidObjectsRuntime {
                 : (await this.repository.remindersForInstance(instance.id)).map(scheduledReminderOf),
         });
         const stateBefore = stableJson(actorState(actor, registered.definition.stateKeys));
-        const intentCount = actor.intentCount();
+        const intentsBefore = actor.intentSnapshot();
         const snapshot = { ...state };
         await withActorProjection({ actor, runtime: this }, async () => {
             for (const query of registered.definition.queries) {
@@ -593,9 +596,15 @@ export class SolidObjectsRuntime {
             }
         });
         if (stableJson(actorState(actor, registered.definition.stateKeys)) !== stateBefore ||
-            actor.intentCount() !== intentCount) {
+            actor.intentSnapshot() !== intentsBefore) {
             throw new QueryMutatedState("snapshot getters must not mutate actor state or stage work");
         }
+        this.emitInstrumentation("snapshot.read", {
+            actorType: reference.actorType,
+            actorId: reference.actorId,
+            instanceId: instance?.id ?? null,
+            revision: String(instance?.state_revision ?? 0),
+        });
         return {
             snapshot: readonlyCopy(snapshot),
             instanceId: instance?.id ?? "0",
@@ -955,6 +964,10 @@ export class SolidObjectsRuntime {
                     });
         }
         catch (error) {
+            this.emitInstrumentation("activation.failed", {
+                ...activationInstrumentation(turn),
+                errorName: error instanceof Error ? error.name : "Error",
+            });
             throw new ActorSetupFailed(error);
         }
         const definition = registered.definition;
@@ -968,23 +981,25 @@ export class SolidObjectsRuntime {
         let stateBefore;
         if (!activated) {
             try {
+                this.emitInstrumentation("activation.started", activationInstrumentation(turn));
                 await withActorContext({ actor, runtime: this }, () => actor.activate());
                 activated = true;
-                this.emitInstrumentation("activation.started", {
-                    actorType: turn.message.actor_type,
-                    actorId: turn.message.actor_id,
-                    instanceId: turn.instance.id,
-                    generation: String(turn.activationGeneration),
-                });
+                this.emitInstrumentation("activation.completed", activationInstrumentation(turn));
             }
             catch (error) {
                 renewalController.abort();
                 await renewal;
                 actor.discardIntents();
+                this.emitInstrumentation("activation.failed", {
+                    ...activationInstrumentation(turn),
+                    errorName: error instanceof Error ? error.name : "Error",
+                });
                 throw new ActorSetupFailed(renewalError ?? error);
             }
         }
-        const startedAt = Date.now();
+        const startedAt = performance.now();
+        if (Number(turn.message.attempt_count) > 1 && turn.message.error === null)
+            this.emitInstrumentation("recovery.reclaimed", messageInstrumentation(turn.message));
         this.emitInstrumentation("message.started", messageInstrumentation(turn.message));
         try {
             stateBefore = deepCopy(actorState(actor, definition.stateKeys));
@@ -1022,6 +1037,7 @@ export class SolidObjectsRuntime {
                         throw new UnknownCommitAction(`unknown commit action ${JSON.stringify(intent.name)}`);
                     const attributes = {
                         commitAction: intent.name,
+                        activationGeneration: String(turn.activationGeneration),
                         ...messageInstrumentation(turn.message),
                     };
                     this.emitInstrumentation("commit_action.started", attributes);
@@ -1064,10 +1080,13 @@ export class SolidObjectsRuntime {
                     nextRunAt: new Date(replacement.nextRunAtMilliseconds).toISOString(),
                 });
             }
+            if (recoveryMessage(turn.message))
+                this.emitInstrumentation("recovery.completed", messageInstrumentation(turn.message));
             this.warnAboutLargeState(turn.message, committed);
             this.emitInstrumentation("message.completed", {
                 ...messageInstrumentation(turn.message),
-                durationMilliseconds: Date.now() - startedAt,
+                revision: String(turn.message.sequence),
+                durationMilliseconds: performance.now() - startedAt,
             });
             return { actor, retainActivation: true, activated };
         }
@@ -1080,7 +1099,7 @@ export class SolidObjectsRuntime {
             if (error instanceof LostActivation) {
                 this.emitInstrumentation("activation.lost", {
                     ...messageInstrumentation(turn.message),
-                    durationMilliseconds: Date.now() - startedAt,
+                    durationMilliseconds: performance.now() - startedAt,
                 });
                 return { actor, retainActivation: false, activated };
             }
@@ -1093,7 +1112,7 @@ export class SolidObjectsRuntime {
                 this.emitInstrumentation("message.rejected", {
                     ...messageInstrumentation(turn.message),
                     code: error.code,
-                    durationMilliseconds: Date.now() - startedAt,
+                    durationMilliseconds: performance.now() - startedAt,
                 });
                 return { actor, retainActivation: activated, activated };
             }
@@ -1106,12 +1125,15 @@ export class SolidObjectsRuntime {
                 ...messageInstrumentation(turn.message),
                 retryable,
                 errorName: error instanceof Error ? error.name : "Error",
-                durationMilliseconds: Date.now() - startedAt,
+                durationMilliseconds: performance.now() - startedAt,
                 outcome,
             });
-            if (outcome === "dead") {
+            if (outcome === "retrying")
+                this.emitInstrumentation("message.retry", messageInstrumentation(turn.message));
+            if (outcome === "dead" && recoveryMessage(turn.message))
+                this.emitInstrumentation("recovery.failed", messageInstrumentation(turn.message));
+            if (outcome === "dead")
                 this.emitInstrumentation("dead_letter.created", messageInstrumentation(turn.message));
-            }
             return { actor, retainActivation: activated, activated };
         }
     }
@@ -1153,6 +1175,7 @@ export class SolidObjectsRuntime {
         await this.callerWorker?.stop();
         this.realtime.close();
         await this.closeWakeUp();
+        this.observers.clear();
         await this.settings.database.close();
         clearDefaultRuntime(this);
     }
@@ -1208,27 +1231,55 @@ export class SolidObjectsRuntime {
         await this.discardWakeUp();
         await this.repository.resetForTesting();
     }
+    diagnostics(reference, options = {}) {
+        return actorDiagnostics({
+            runtime: this,
+            actorType: reference.actorType,
+            actorId: reference.actorId,
+            options,
+        });
+    }
+    async observe(reference, options) {
+        await this.authorizeAdministration({
+            action: "observe",
+            resource: "actor_diagnostics",
+            resourceId: JSON.stringify([reference.actorType, reference.actorId]),
+            authorizationContext: options.authorizationContext,
+        });
+        if (this.observers.size >= 1000)
+            throw new RangeError("at most 1000 local observers may be registered");
+        const observer = (event) => {
+            if (event.actorType === reference.actorType && event.actorId === reference.actorId)
+                return options.onEvent(event);
+        };
+        this.observers.add(observer);
+        return () => {
+            this.observers.delete(observer);
+        };
+    }
     emitInstrumentation(name, attributes) {
-        const instrumentation = this.settings.instrumentation;
-        if (!instrumentation)
+        if (!this.settings.instrumentation && this.observers.size === 0)
             return;
         try {
-            instrumentation(Object.freeze({
-                name: `solid_objects.${name}`,
-                occurredAt: new Date().toISOString(),
-                attributes: readonlyCopy(attributes),
-            }));
-        }
-        catch (error) {
-            this.settings.logger.error({
-                event: "solid_objects.instrumentation.failed",
-                instrumentationEvent: `solid_objects.${name}`,
-                errorName: error instanceof Error ? error.name : "Error",
-            });
+            const event = telemetryEvent({ name, adapter: this.settings.database.family, attributes });
+            if (this.settings.instrumentation)
+                deliverTelemetry({
+                    observer: this.settings.instrumentation,
+                    event,
+                    logger: this.settings.logger,
+                });
+            for (const observer of this.observers)
+                deliverTelemetry({ observer, event, logger: this.settings.logger });
         }
+        catch { }
     }
     async executeEffect(effect) {
         try {
+            this.emitInstrumentation("outbox.age", {
+                ...effectInstrumentation(effect),
+                outboxKind: "effect",
+                ageMilliseconds: Math.max(0, Date.now() - Number(effect.available_at_ms)),
+            });
             const handler = this.effects.get(effect.name);
             if (!handler)
                 throw new UnknownEffect(`unknown effect ${JSON.stringify(effect.name)}`);
@@ -1266,20 +1317,30 @@ export class SolidObjectsRuntime {
         if (!actor.operations.has(dispatchOperation)) {
             throw new UnknownOperation(`unknown reminder operation ${JSON.stringify(dispatchOperation)}`);
         }
-        if (!(await this.repository.enqueueReminder(reminder, options)))
+        const message = await this.repository.enqueueReminder(reminder, options);
+        if (!message)
             return;
         this.wakeUp("actors");
         this.emitInstrumentation("reminder.enqueued", {
             reminderId: reminder.id,
+            messageId: message.id,
+            attempt: Number(message.attempt_count),
             actorType: reminder.actor_type,
             actorId: reminder.actor_id,
             operation: reminder.operation,
+            instanceId: reminder.instance_id,
+            latenessMilliseconds: Math.max(0, (options.nowMilliseconds ?? Date.now()) - Number(reminder.run_at_ms)),
             occurrence: Number(reminder.occurrence),
         });
     }
     async executeBroadcast(broadcast) {
         const deliver = this.settings.broadcast;
         try {
+            this.emitInstrumentation("outbox.age", {
+                ...broadcastInstrumentation(broadcast),
+                outboxKind: "broadcast",
+                ageMilliseconds: Math.max(0, Date.now() - Number(broadcast.available_at_ms)),
+            });
             const event = readonlyCopy({
                 actorType: broadcast.actor_type,
                 actorId: broadcast.actor_id,
@@ -1722,14 +1783,14 @@ export class SolidObjectsRuntime {
                 state: deepCopy(options.snapshot.state),
             });
             const stateBefore = stableJson(actorState(actor, options.registered.definition.stateKeys));
-            const intentCount = actor.intentCount();
+            const intentsBefore = actor.intentSnapshot();
             const handler = options.registered.definition.payloads[options.name];
             if (!handler) {
                 throw new UnknownPayloadBroadcast(`unknown payload broadcast ${options.name}`);
             }
             const rawPayload = await withActorProjection({ actor, runtime: this }, () => handler(actor, options.authorizationContext));
             if (stableJson(actorState(actor, options.registered.definition.stateKeys)) !== stateBefore ||
-                actor.intentCount() !== intentCount) {
+                actor.intentSnapshot() !== intentsBefore) {
                 throw new QueryMutatedState("payload broadcasts must not mutate actor state or stage work");
             }
             const payload = normalizeJson(rawPayload, { maxBytes: this.settings.maxPayloadBytes });
@@ -1926,8 +1987,14 @@ function messageFailure(message) {
 function actorDestroyedWhileWaiting() {
     return new ActorDestroyed("actor was destroyed while waiting for its result");
 }
+function recoveryMessage(message) {
+    return (message.delivery_mode === "internal" &&
+        message.idempotency_key?.startsWith("effect:") === true &&
+        message.idempotency_key.endsWith(":recovery"));
+}
 function messageInstrumentation(message) {
     return {
+        instanceId: message.instance_id,
         messageId: message.id,
         requestId: message.request_id,
         actorType: message.actor_type,
@@ -1938,6 +2005,13 @@ function messageInstrumentation(message) {
         attempt: Number(message.attempt_count),
     };
 }
+function activationInstrumentation(turn) {
+    return {
+        ...messageInstrumentation(turn.message),
+        generation: String(turn.activationGeneration),
+        ownerId: turn.processId,
+    };
+}
 function actorMessageContext(message) {
     return Object.freeze({
         id: message.id,
@@ -1957,6 +2031,7 @@ function restoreActorState(options) {
 }
 function effectInstrumentation(effect) {
     return {
+        instanceId: effect.instance_id,
         effectId: effect.id,
         effectName: effect.name,
         messageId: effect.message_id,
@@ -1967,6 +2042,7 @@ function effectInstrumentation(effect) {
 }
 function broadcastInstrumentation(broadcast) {
     return {
+        instanceId: broadcast.instance_id,
         broadcastId: broadcast.id,
         messageId: broadcast.message_id,
         actorType: broadcast.actor_type,