rivet-design 0.28.1 → 0.28.3

package/dist/services/profiles/DesignProfiles.js

@@ -41,8 +41,16 @@ const ProviderRateLimit_1 = require("../../utils/ProviderRateLimit");
 const design_profile_3 = require("../../types/design-profile");
 const errorMessage_1 = require("../../utils/errorMessage");
 const APPROVAL_TIMEOUT_MS = 45_000;
+const MS_PER_HOUR = 3_600_000;
 const SAVED_REFERENCE_IMAGE_LIMIT = 20;
 const VERIFICATION_TARGET_INSTRUCTIONS = 'Choose tools for the actual browser, simulator, or other target. Do not assume a URL, DOM, CSS, or Playwright. If required tools or the target are unavailable, report unable to verify.';
+const preparationOffReason = ({ mode, isEnabled, }) => {
+    if (mode === 'off')
+        return 'request_mode_off';
+    if (!isEnabled)
+        return 'profile_disabled';
+    return 'no_uploads';
+};
 const librarySources = (records) => records.sources().map(({ id, title, provider, isIncluded }) => ({
     id,
     title,
@@ -544,21 +552,25 @@ class DesignProfiles {
     async importReferences(input) {
         const request = reference_onboarding_1.ReferenceApprovalSchema.parse(input);
         const accountId = this.options.accountId();
+        if (!accountId)
+            throw new Error('Sign in to Rivet to connect design references');
+        const context = await this.context();
+        const wasEnabled = context.records.profile().isEnabled;
         const plan = await this.withReferenceRecords(async (records, currentAccountId) => {
             const pending = records.plans.plan(request.id);
             if (!pending.approvedAt)
                 await this.planning(records, currentAccountId).verifyAccount(pending.profile, AbortSignal.timeout(APPROVAL_TIMEOUT_MS));
             return records.approvePlan(request);
         });
-        if (accountId !== this.options.accountId())
-            throw new Error('Profile account changed during import approval');
-        const context = await this.context();
         if (context.accountId !== accountId ||
             accountId !== this.options.accountId())
             throw new Error('Profile account changed during import approval');
-        if (context.records.profile().isEnabled)
+        if (context.records.profile().isEnabled) {
+            if (!wasEnabled)
+                this.resume(context, plan.jobIds);
             for (const id of plan.jobIds)
-                this.enqueue(context, () => this.runImport(context, id));
+                this.enqueue(context, () => this.runImport(context, id, 'connect'));
+        }
         return (0, reference_onboarding_1.referencePlanSummary)(plan);
     }
     planning(records, accountId) {
@@ -668,12 +680,15 @@ class DesignProfiles {
             await this.flush(context, job.runtime);
             context.records.completeRebuild(job.id);
             context.issue = null;
+            const sources = context.records.sources();
             this.options.telemetry?.trackProfileOperation({
                 stage: 'maintenance',
                 operation: 'index_rebuild',
                 outcome: 'completed',
                 durationMs: Date.now() - started,
                 referenceCount: context.records.coverage().indexed,
+                providerCount: new Set(sources.map((source) => source.provider)).size,
+                sourceCount: sources.length,
             });
         }
         catch (error) {
@@ -682,6 +697,7 @@ class DesignProfiles {
                 operation: 'index_rebuild',
                 outcome: 'failed',
                 durationMs: Date.now() - started,
+                error,
             });
             throw error;
         }
@@ -763,7 +779,7 @@ class DesignProfiles {
             isComplete: true,
             total: new Set(references.map((reference) => reference.remoteId)).size,
         });
-        this.enqueue(context, () => this.runImport(context, job.id));
+        this.enqueue(context, () => this.runImport(context, job.id, 'upload'));
         return { source, jobId: job.id };
     }
     async pins(projectId) {
@@ -797,7 +813,7 @@ class DesignProfiles {
             },
             runtime: value.runtime,
         });
-        this.enqueue(context, () => this.runImport(context, result.job.id));
+        this.enqueue(context, () => this.runImport(context, result.job.id, 'upload'));
         return { source: result.source, jobId: result.job.id };
     }
     async connect(input) {
@@ -852,7 +868,7 @@ class DesignProfiles {
                 throw new Error('Profile account changed or was disabled during X authorization');
         }
         const result = context.imports.connect({ ...input, xAccount });
-        this.enqueue(context, () => this.runImport(context, result.job.id));
+        this.enqueue(context, () => this.runImport(context, result.job.id, 'connect'));
         return result;
     }
     async updateJob({ id, action, }) {
@@ -866,7 +882,7 @@ class DesignProfiles {
             state: action === 'pause' ? 'paused' : 'pending',
         });
         if (action !== 'pause')
-            this.enqueue(context, () => this.runImport(context, id));
+            this.enqueue(context, () => this.runImport(context, id, 'retry'));
         return this.status();
     }
     async refresh({ id, runtime }) {
@@ -880,7 +896,7 @@ class DesignProfiles {
         if (!(0, design_profile_1.isRemoteProfileProvider)(source.provider))
             throw new Error('Only connected provider sources support refresh');
         const job = context.records.startImport({ sourceId: id, runtime });
-        this.enqueue(context, () => this.runImport(context, job.id));
+        this.enqueue(context, () => this.runImport(context, job.id, 'refresh'));
         return { job };
     }
     async updateSource({ id, isIncluded, isRemoved, runtime, }) {
@@ -1152,13 +1168,26 @@ class DesignProfiles {
                 failedUploadCount: undefined,
             }
             : packet;
+        const selectedProviders = result.selected.flatMap((item) => context.records.reference(item.id)?.provider ?? []);
         this.options.telemetry?.trackProfileOperation({
             stage: 'preparation',
             operation: 'preparation',
             outcome: 'completed',
             durationMs: result.durationMs,
             referenceCount: result.selected.length,
+            candidateCount: result.candidateIds.length,
             status: result.status,
+            ...(selectedProviders.length
+                ? { providerMix: [...new Set(selectedProviders)].sort() }
+                : {}),
+            ...(result.status === 'off'
+                ? {
+                    offReason: preparationOffReason({
+                        mode: validated.mode,
+                        isEnabled: context.records.profile().isEnabled,
+                    }),
+                }
+                : {}),
         });
         return { ...result, profileId: context.records.profile().id, accountId };
     }
@@ -1204,7 +1233,7 @@ class DesignProfiles {
                         runtime,
                     });
                     reference = context.records.references(saved.source.id)[0];
-                    this.enqueue(context, () => this.runImport(context, saved.job.id));
+                    this.enqueue(context, () => this.runImport(context, saved.job.id, 'attachment'));
                 }
                 if (!reference)
                     throw new Error('Saved image reference is unavailable');
@@ -1292,13 +1321,14 @@ class DesignProfiles {
                     media,
                     agent,
                     isCurrentAccount: () => this.options.accountId() === accountId,
+                    telemetry: this.options.telemetry,
                 }),
                 schedule: new ProfileImportSchedule_1.ProfileImportSchedule({
                     records,
                     isActive: () => this.current === context &&
                         !context.abort.signal.aborted &&
                         this.options.accountId() === accountId,
-                    onReady: (id) => this.enqueue(context, () => this.runImport(context, id)),
+                    onReady: (id) => this.enqueue(context, () => this.runImport(context, id, 'cooldown_retry')),
                 }),
                 abort: new AbortController(),
                 work: Promise.resolve(),
@@ -1316,15 +1346,15 @@ class DesignProfiles {
             this.opening = undefined;
         }
     }
-    resume(context) {
+    resume(context, claimedJobIds = []) {
         this.enqueue(context, async () => {
             await context.memory.start();
             context.issue = null;
         });
         for (const job of context.records
             .jobs()
-            .filter((job) => job.state === 'pending'))
-            this.enqueue(context, () => this.runImport(context, job.id));
+            .filter((job) => job.state === 'pending' && !claimedJobIds.includes(job.id)))
+            this.enqueue(context, () => this.runImport(context, job.id, 'resume'));
         this.enqueue(context, () => this.runRebuild(context));
     }
     enqueue(context, operation) {
@@ -1343,36 +1373,64 @@ class DesignProfiles {
         while (context.records.pendingIndex(1).length)
             await context.index.flush(runtime, context.abort.signal);
     }
-    async runImport(context, id) {
+    async runImport(context, id, kind) {
         if (!context.schedule.isReady(id))
             return;
         const job = context.records.jobs().find((item) => item.id === id);
         if (!job || job.state !== 'pending')
             return;
         const started = Date.now();
+        const importContext = {
+            provider: context.records
+                .sources()
+                .find((source) => source.id === job.sourceId)?.provider,
+            importKind: kind,
+        };
+        const syncTimes = context.records
+            .jobs()
+            .filter((item) => item.sourceId === job.sourceId && item.state === 'complete')
+            .map((item) => Date.parse(item.updatedAt));
+        const lastSyncedAt = syncTimes.length ? Math.max(...syncTimes) : undefined;
+        let step = 'memory_start';
         try {
             await context.memory.start();
             const source = context.records.source(job.sourceId);
-            if ((0, design_profile_1.isRemoteProfileProvider)(source.provider)) {
+            const isSync = (0, design_profile_1.isRemoteProfileProvider)(source.provider);
+            if (isSync) {
+                step = 'enumerate';
                 await context.imports.enumerate(id, context.abort.signal);
+                step = 'download';
                 await context.imports.download(id, context.abort.signal);
             }
+            step = 'analysis';
             await context.analysis.run({ jobId: id, signal: context.abort.signal });
             if (context.records.jobs().find((item) => item.id === id)?.state !==
                 'pending')
                 return;
+            step = 'index';
             context.records.queueIndex(job.runtime);
             await this.runRebuild(context);
             if (context.records.jobs().find((item) => item.id === id)?.state !==
                 'pending')
                 return;
             context.records.updateJob({ id, state: 'complete' });
+            const completed = context.records.jobs().find((item) => item.id === id);
             this.options.telemetry?.trackProfileOperation({
                 stage: 'import',
                 operation: 'import',
                 outcome: 'completed',
                 durationMs: Date.now() - started,
                 referenceCount: context.records.references(job.sourceId).length,
+                ...importContext,
+                ...(isSync && completed
+                    ? {
+                        addedCount: completed.addedCount,
+                        removedCount: completed.removedCount,
+                        ...(lastSyncedAt === undefined
+                            ? {}
+                            : { staleHours: (started - lastSyncedAt) / MS_PER_HOUR }),
+                    }
+                    : {}),
             });
         }
         catch (error) {
@@ -1387,6 +1445,7 @@ class DesignProfiles {
                     operation: 'import',
                     outcome: 'deferred',
                     durationMs: Date.now() - started,
+                    ...importContext,
                 });
                 return;
             }
@@ -1395,6 +1454,8 @@ class DesignProfiles {
                 operation: 'import',
                 outcome: 'failed',
                 durationMs: Date.now() - started,
+                ...importContext,
+                failedStep: error instanceof ProfileRecords_1.UnfinishedImport ? error.step : step,
                 error,
             });
             if (!context.abort.signal.aborted &&

package/dist/services/profiles/ProfileAnalysis.d.ts

@@ -1,12 +1,14 @@
 import type { ProfileRecords } from '../../storage/profiles/ProfileRecords';
 import type { ProfileMedia } from './ProfileMedia';
 import type { ProfileAgent } from './ProfileAgent';
+import type { TelemetryService } from '../TelemetryService';
 interface ProfileAnalysisOptions {
     records: ProfileRecords;
     media: ProfileMedia;
     agent: Pick<ProfileAgent, 'run'>;
     isCurrentAccount: () => boolean;
     batchSize?: 1 | 2 | 4;
+    telemetry?: Pick<TelemetryService, 'trackProfileOperation'>;
 }
 export declare class ProfileAnalysis {
     private readonly options;
@@ -17,6 +19,7 @@ export declare class ProfileAnalysis {
     }): Promise<void>;
     private isPending;
     private analyze;
+    private track;
     private fail;
 }
 export {};

package/dist/services/profiles/ProfileAnalysis.js

@@ -4,6 +4,15 @@ exports.ProfileAnalysis = void 0;
 const modelContract_1 = require("./modelContract");
 const analysisBatch_1 = require("./analysisBatch");
 const errorMessage_1 = require("../../utils/errorMessage");
+const failureReason_1 = require("../../utils/failureReason");
+const analysisMediaKind = (references) => {
+    const kinds = new Set(references.flatMap((reference) => reference.media.length
+        ? reference.media.map((media) => media.representation)
+        : ['text']));
+    if (kinds.size > 1)
+        return 'mixed';
+    return [...kinds][0] ?? 'text';
+};
 class ProfileAnalysis {
     options;
     constructor(options) {
@@ -14,7 +23,12 @@ class ProfileAnalysis {
         const job = records.jobs().find((item) => item.id === jobId);
         if (!job || job.state !== 'pending')
             return;
-        const input = { jobId, runtime: job.runtime, signal };
+        const input = {
+            jobId,
+            runtime: job.runtime,
+            signal,
+            provider: records.source(job.sourceId).provider,
+        };
         let batch = [];
         for (const reference of records.references(job.sourceId)) {
             if (!this.isPending(input))
@@ -23,6 +37,7 @@ class ProfileAnalysis {
                 continue;
             if (reference.media.length < (reference.remoteImages?.length ?? 0))
                 continue;
+            let startedAt = Date.now();
             try {
                 if (!(0, analysisBatch_1.fitsAnalysisBatch)([reference]))
                     throw new Error('Reference exceeds the analysis budget; split it into smaller references');
@@ -35,6 +50,7 @@ class ProfileAnalysis {
                     batch = [];
                     if (!this.isPending(input))
                         return;
+                    startedAt = Date.now();
                 }
                 const images = await Promise.all(reference.media.map((image) => media.resolve(image)));
                 batch.push({ reference, images });
@@ -43,6 +59,13 @@ class ProfileAnalysis {
                 if (!this.isPending(input))
                     return;
                 this.fail({ jobId, reference, error });
+                this.track({
+                    job: input,
+                    references: [reference],
+                    outcome: 'failed',
+                    startedAt,
+                    error,
+                });
             }
         }
         if (batch.length && this.isPending(input))
@@ -60,6 +83,7 @@ class ProfileAnalysis {
             return;
         const { batch, runtime, signal, jobId } = input;
         const { agent, records } = this.options;
+        const startedAt = Date.now();
         let result;
         try {
             result = await agent.run({
@@ -73,8 +97,18 @@ class ProfileAnalysis {
                 return;
             for (const { reference } of batch)
                 this.fail({ jobId, reference, error });
+            this.track({
+                job: input,
+                references: batch.map((item) => item.reference),
+                outcome: 'failed',
+                startedAt,
+                error,
+            });
             return;
         }
+        const analyzed = [];
+        const failed = [];
+        let failure;
         for (const { reference, images } of batch) {
             if (!this.isPending(input))
                 return;
@@ -95,11 +129,41 @@ class ProfileAnalysis {
                     usage: (0, analysisBatch_1.analysisUsage)((0, modelContract_1.reportedProfileUsage)(result.usage), batch.length),
                 });
                 records.clearFailure(reference.id, 'analysis');
+                analyzed.push(reference);
             }
             catch (error) {
                 this.fail({ jobId, reference, error });
+                failed.push(reference);
+                failure ??= error;
             }
         }
+        if (analyzed.length)
+            this.track({
+                job: input,
+                references: analyzed,
+                outcome: 'completed',
+                startedAt,
+            });
+        if (failed.length)
+            this.track({
+                job: input,
+                references: failed,
+                outcome: 'failed',
+                startedAt,
+                error: failure,
+            });
+    }
+    track({ job, references, outcome, startedAt, error, }) {
+        this.options.telemetry?.trackProfileOperation({
+            stage: 'analysis',
+            operation: 'analysis',
+            outcome,
+            durationMs: Date.now() - startedAt,
+            referenceCount: references.length,
+            provider: job.provider,
+            mediaKind: analysisMediaKind(references),
+            error,
+        });
     }
     fail({ jobId, reference, error, }) {
         this.options.records.saveFailure({
@@ -108,6 +172,7 @@ class ProfileAnalysis {
             revision: reference.revision,
             stage: 'analysis',
             error: (0, errorMessage_1.errorMessage)(error, 'Reference analysis failed'),
+            reasonCode: (0, failureReason_1.failureReasonCode)(error),
         });
     }
 }

package/dist/services/profiles/ProfileImports.d.ts

@@ -36,6 +36,8 @@ export declare class ProfileImports {
             total: number | null;
             error: string | null;
             updatedAt: string;
+            addedCount: number;
+            removedCount: number;
         };
     };
     enumerate(jobId: string, signal: AbortSignal): Promise<void>;

package/dist/services/profiles/ProfileImports.js

@@ -12,6 +12,7 @@ const ProfileMedia_1 = require("./ProfileMedia");
 const boardPage_1 = require("./boardPage");
 const sourceContext_1 = require("../../agent-variants/sourceContext");
 const errorMessage_1 = require("../../utils/errorMessage");
+const failureReason_1 = require("../../utils/failureReason");
 const DOWNLOAD_TIMEOUT_MS = 20_000;
 const MAX_REDIRECTS = 5;
 const MEDIA_HOSTS = ['pinimg.com', 'are.na', 'cloudfront.net', 'amazonaws.com'];
@@ -202,6 +203,7 @@ class ProfileImports {
                     revision: current.revision,
                     stage: 'download',
                     error: (0, errorMessage_1.errorMessage)(error, 'Image download failed'),
+                    reasonCode: (0, failureReason_1.failureReasonCode)(error),
                 });
             }
         }

package/dist/services/templates/viteReactTs.js

@@ -200,7 +200,7 @@ textarea {
 
 .eyebrow {
   margin: 0 0 16px;
-  color: #ff3300;
+  color: #e54b2c;
   font-size: 13px;
   font-weight: 700;
   letter-spacing: 0;

package/dist/storage/profiles/ProfileRecords.d.ts

@@ -1,5 +1,7 @@
 import { ProfilePlans } from './ProfilePlans';
 import { type GuidanceDocument, type GuidanceInput, type GuidanceSelection, type GuidanceUpdate, type SavedGuidance } from '../../types/guidance';
+import { type ProfileImportStep } from '../../types/design-profile';
+import type { FailureReasonCode } from '../../utils/failureReason';
 import type { DesignLibraryReference } from '../../types/design-library';
 import type { ProfileCandidate, ProfileCollectionProgress, ProfileIndexWork, ProfileObservation, ProfileReference, ProfileReferenceInput, ProfileRuntime, ProfileSourceInput } from '../../types/design-profile';
 type ImportPage = {
@@ -12,6 +14,12 @@ type ImportPage = {
     title?: string;
     collection?: ProfileCollectionProgress;
 };
+type UnfinishedImportStep = Extract<ProfileImportStep, 'enumerate' | 'download' | 'analysis'>;
+export declare class UnfinishedImport extends Error {
+    readonly step: UnfinishedImportStep;
+    readonly reasonCode: FailureReasonCode;
+    constructor(step: UnfinishedImportStep, reasonCode: FailureReasonCode);
+}
 export declare class ProfileRecords {
     readonly directory: string;
     private readonly db;
@@ -245,6 +253,8 @@ export declare class ProfileRecords {
             total: number | null;
             error: string | null;
             updatedAt: string;
+            addedCount: number;
+            removedCount: number;
         };
     };
     startImport({ sourceId, runtime, }: {
@@ -262,6 +272,8 @@ export declare class ProfileRecords {
         total: number | null;
         error: string | null;
         updatedAt: string;
+        addedCount: number;
+        removedCount: number;
     };
     private writeImport;
     jobs(): {
@@ -276,6 +288,8 @@ export declare class ProfileRecords {
         total: number | null;
         error: string | null;
         updatedAt: string;
+        addedCount: number;
+        removedCount: number;
     }[];
     deferImport({ id, retryAfterMs }: {
         id: string;
@@ -290,12 +304,14 @@ export declare class ProfileRecords {
         state: 'pending' | 'paused' | 'failed' | 'complete';
         error?: string | null;
     }): void;
+    private unfinishedImport;
     failures(jobId?: string): {
         jobId: string;
         referenceId: string;
         revision: number;
         stage: "download" | "analysis";
         error: string;
+        reasonCode: "timeout" | "unknown" | "aborted" | "multiple_lockfiles" | "unsupported_package_manager" | "invalid_package_manager_field" | "invalid_manifest" | "invalid_workspace" | "path_outside_workspace" | "dangling_symlink" | "version_unreadable" | "version_mismatch" | "no_install_command" | "install_failed" | "lockfile_outdated" | "engine_incompatible" | "install_script_failed" | "installer_unstoppable" | "inputs_changed" | "ENOENT" | "EACCES" | "EPERM" | "EEXIST" | "ENOTDIR" | "EISDIR" | "ELOOP" | "ENOSPC" | "EMFILE" | "EADDRINUSE" | "ECONNREFUSED" | "ECONNRESET" | "ETIMEDOUT" | "EBUSY" | "ENOTSUP" | "EINVAL" | "ENOTEMPTY" | "EROFS" | "EAI_AGAIN" | null;
         attempts: number;
         updatedAt: string;
     }[];
@@ -305,6 +321,7 @@ export declare class ProfileRecords {
         revision: number;
         stage: 'download' | 'analysis';
         error: string;
+        reasonCode: FailureReasonCode;
     }): void;
     clearFailure(referenceId: string, stage: 'download' | 'analysis'): void;
     saveImportPage(input: ImportPage): void;

package/dist/storage/profiles/ProfileRecords.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ProfileRecords = void 0;
+exports.ProfileRecords = exports.UnfinishedImport = void 0;
 const guidanceVerifier_1 = require("./guidanceVerifier");
 const guidance_verifier_1 = require("../../types/guidance-verifier");
 const reference_connectors_1 = require("../../types/reference-connectors");
@@ -25,6 +25,17 @@ const MIN_RETRY_MS = 1000;
 const MAX_BACKOFF_EXPONENT = 4;
 const MAX_DATE_MS = 8_640_000_000_000_000;
 const identity = (...parts) => (0, crypto_1.createHash)('sha256').update(JSON.stringify(parts)).digest('hex');
+class UnfinishedImport extends Error {
+    step;
+    reasonCode;
+    constructor(step, reasonCode) {
+        super('Import has unfinished references');
+        this.step = step;
+        this.reasonCode = reasonCode;
+        this.name = 'UnfinishedImport';
+    }
+}
+exports.UnfinishedImport = UnfinishedImport;
 const guidanceDocument = (document) => guidance_1.GuidanceDocumentSchema.parse({
     id: document.id,
     title: document.title,
@@ -649,18 +660,33 @@ class ProfileRecords {
         const job = this.jobs().find((item) => item.id === id);
         if (!job || job.state === 'superseded')
             throw new Error('Import job is no longer active');
-        if (state === 'complete' &&
-            (job.state !== 'pending' ||
-                !job.isEnumerated ||
-                this.failures(id).length ||
-                this.references(job.sourceId).some((reference) => !reference.unsupportedReason && !this.observation(reference.id))))
-            throw new Error('Import has unfinished references');
+        if (state === 'complete') {
+            if (job.state !== 'pending')
+                throw new Error('Import has unfinished references');
+            const unfinished = this.unfinishedImport(job);
+            if (unfinished)
+                throw unfinished;
+        }
         this.db
             .update(schema_1.importJobs)
             .set({ state, error, updatedAt: new Date().toISOString() })
             .where((0, drizzle_orm_1.eq)(schema_1.importJobs.id, id))
             .run();
     }
+    unfinishedImport(job) {
+        if (!job.isEnumerated)
+            return new UnfinishedImport('enumerate', 'unknown');
+        const failures = this.failures(job.id);
+        for (const stage of ['download', 'analysis']) {
+            const failure = failures.find((item) => item.stage === stage);
+            if (failure)
+                return new UnfinishedImport(stage, failure.reasonCode ?? 'unknown');
+        }
+        const isAnalysisMissing = this.references(job.sourceId).some((reference) => !reference.unsupportedReason && !this.observation(reference.id));
+        return isAnalysisMissing
+            ? new UnfinishedImport('analysis', 'unknown')
+            : null;
+    }
     failures(jobId) {
         return this.db
             .select({ failure: schema_1.referenceFailures })
@@ -777,7 +803,15 @@ class ProfileRecords {
                 .where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(schema_1.importCursors.jobId, jobId), (0, drizzle_orm_1.eq)(schema_1.importCursors.cursor, cursor)))
                 .get())
             throw new Error('Provider returned a repeated page cursor');
+        let addedCount = 0;
         for (const item of validated) {
+            const isMember = this.db
+                .select()
+                .from(schema_1.memberships)
+                .where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(schema_1.memberships.sourceId, job.sourceId), (0, drizzle_orm_1.eq)(schema_1.memberships.referenceId, identity(item.provider, item.remoteId))))
+                .get();
+            if (!isMember)
+                addedCount += 1;
             const reference = this.writeReference(job.sourceId, item, job.id);
             if (collection)
                 this.db
@@ -819,6 +853,7 @@ class ProfileRecords {
                 .set({ resolvedRemoteId: remoteId, ...(title ? { title } : {}) })
                 .where((0, drizzle_orm_1.eq)(schema_1.sources.id, job.sourceId))
                 .run();
+        let removedCount = 0;
         if (isComplete) {
             const unseen = this.db
                 .select()
@@ -834,6 +869,7 @@ class ProfileRecords {
                 if (reference)
                     this.queueOrphan(reference);
             });
+            removedCount = unseen.length;
         }
         this.db
             .update(schema_1.importJobs)
@@ -843,6 +879,8 @@ class ProfileRecords {
             total: collection && isComplete ? enumerated : (total ?? job.total),
             state: 'pending',
             isEnumerated: isComplete,
+            addedCount: job.addedCount + addedCount,
+            removedCount: job.removedCount + removedCount,
             updatedAt: new Date().toISOString(),
         })
             .where((0, drizzle_orm_1.eq)(schema_1.importJobs.id, job.id))

package/dist/storage/profiles/migrations.d.ts

@@ -1,4 +1,4 @@
-export declare const PROFILE_SCHEMA = "\nCREATE TABLE profile (singleton INTEGER PRIMARY KEY CHECK(singleton = 1), id TEXT NOT NULL UNIQUE, is_enabled INTEGER NOT NULL, revision INTEGER NOT NULL);\nCREATE TABLE sources (id TEXT PRIMARY KEY, provider TEXT NOT NULL, remote_id TEXT NOT NULL,\n  title TEXT NOT NULL, url TEXT, is_included INTEGER NOT NULL, removed_at TEXT, resolved_remote_id TEXT,\n  UNIQUE(provider, remote_id));\nCREATE TABLE profile_references (id TEXT PRIMARY KEY, revision INTEGER NOT NULL,\n  content_hash TEXT NOT NULL, body TEXT NOT NULL);\nCREATE TABLE memberships (source_id TEXT NOT NULL REFERENCES sources(id),\n  reference_id TEXT NOT NULL REFERENCES profile_references(id), seen_job_id TEXT,\n  PRIMARY KEY(source_id, reference_id));\nCREATE INDEX memberships_reference ON memberships(reference_id);\nCREATE TABLE observations (reference_id TEXT PRIMARY KEY REFERENCES profile_references(id),\n  revision INTEGER NOT NULL, body TEXT NOT NULL);\nCREATE TABLE index_work (reference_id TEXT PRIMARY KEY REFERENCES profile_references(id),\n  id TEXT NOT NULL, revision INTEGER NOT NULL, operation TEXT NOT NULL,\n  engine_id TEXT, state TEXT NOT NULL);\nCREATE TABLE import_jobs (id TEXT PRIMARY KEY, source_id TEXT NOT NULL REFERENCES sources(id),\n  runtime TEXT NOT NULL, state TEXT NOT NULL, cursor TEXT, enumerated INTEGER NOT NULL,\n  total INTEGER, error TEXT, updated_at TEXT NOT NULL, is_enumerated INTEGER NOT NULL DEFAULT 0);\nCREATE INDEX import_jobs_source ON import_jobs(source_id);\nCREATE TABLE index_rebuild (singleton INTEGER PRIMARY KEY CHECK(singleton = 1), id TEXT NOT NULL, runtime TEXT NOT NULL);\n\nCREATE TABLE import_cursors (job_id TEXT NOT NULL REFERENCES import_jobs(id), cursor TEXT NOT NULL, PRIMARY KEY(job_id, cursor));\nCREATE TABLE reference_failures (job_id TEXT NOT NULL REFERENCES import_jobs(id), reference_id TEXT NOT NULL REFERENCES profile_references(id), revision INTEGER NOT NULL, stage TEXT NOT NULL, error TEXT NOT NULL, attempts INTEGER NOT NULL, updated_at TEXT NOT NULL, PRIMARY KEY(job_id, reference_id, stage));\n\nCREATE TABLE provider_cooldowns (provider TEXT PRIMARY KEY, retry_at TEXT NOT NULL, attempts INTEGER NOT NULL);\nCREATE TABLE import_collection_references (job_id TEXT NOT NULL REFERENCES import_jobs(id), collection_id TEXT NOT NULL, reference_id TEXT NOT NULL REFERENCES profile_references(id), PRIMARY KEY(job_id, collection_id, reference_id));\n\nCREATE TABLE reference_discoveries (id TEXT PRIMARY KEY, body TEXT NOT NULL);\nCREATE TABLE reference_plans (id TEXT PRIMARY KEY, body TEXT NOT NULL);\nCREATE TABLE planned_sources (source_id TEXT PRIMARY KEY REFERENCES sources(id), discovery_id TEXT NOT NULL REFERENCES reference_discoveries(id), collection_id TEXT NOT NULL);\nCREATE TABLE reference_onboarding (singleton INTEGER PRIMARY KEY CHECK(singleton = 1), state TEXT NOT NULL, offered_at TEXT, remind_at TEXT);\n\nCREATE TABLE connector_onboarding (provider TEXT PRIMARY KEY, state TEXT NOT NULL, offered_at TEXT, remind_at TEXT);\nCREATE TABLE guidance (id TEXT PRIMARY KEY, title TEXT NOT NULL, markdown TEXT NOT NULL, is_enabled INTEGER NOT NULL);\nALTER TABLE guidance ADD COLUMN attachments TEXT;\nALTER TABLE guidance ADD COLUMN \"references\" TEXT;\nALTER TABLE guidance ADD COLUMN verifier TEXT;\n";
-export declare const PROFILE_MIGRATIONS: readonly ["ALTER TABLE import_jobs ADD COLUMN is_enumerated INTEGER NOT NULL DEFAULT 0", "CREATE TABLE index_rebuild (singleton INTEGER PRIMARY KEY CHECK(singleton = 1), id TEXT NOT NULL, runtime TEXT NOT NULL);", "ALTER TABLE sources ADD COLUMN resolved_remote_id TEXT; \nCREATE TABLE import_cursors (job_id TEXT NOT NULL REFERENCES import_jobs(id), cursor TEXT NOT NULL, PRIMARY KEY(job_id, cursor));\nCREATE TABLE reference_failures (job_id TEXT NOT NULL REFERENCES import_jobs(id), reference_id TEXT NOT NULL REFERENCES profile_references(id), revision INTEGER NOT NULL, stage TEXT NOT NULL, error TEXT NOT NULL, attempts INTEGER NOT NULL, updated_at TEXT NOT NULL, PRIMARY KEY(job_id, reference_id, stage));\n", "CREATE TABLE provider_cooldowns (provider TEXT PRIMARY KEY, retry_at TEXT NOT NULL, attempts INTEGER NOT NULL);", "CREATE TABLE import_collection_references (job_id TEXT NOT NULL REFERENCES import_jobs(id), collection_id TEXT NOT NULL, reference_id TEXT NOT NULL REFERENCES profile_references(id), PRIMARY KEY(job_id, collection_id, reference_id));", "\nCREATE TABLE reference_discoveries (id TEXT PRIMARY KEY, body TEXT NOT NULL);\nCREATE TABLE reference_plans (id TEXT PRIMARY KEY, body TEXT NOT NULL);\nCREATE TABLE planned_sources (source_id TEXT PRIMARY KEY REFERENCES sources(id), discovery_id TEXT NOT NULL REFERENCES reference_discoveries(id), collection_id TEXT NOT NULL);\nCREATE TABLE reference_onboarding (singleton INTEGER PRIMARY KEY CHECK(singleton = 1), state TEXT NOT NULL, offered_at TEXT, remind_at TEXT);\n", "CREATE TABLE connector_onboarding (provider TEXT PRIMARY KEY, state TEXT NOT NULL, offered_at TEXT, remind_at TEXT);", "CREATE TABLE guidance (id TEXT PRIMARY KEY, title TEXT NOT NULL, markdown TEXT NOT NULL, is_enabled INTEGER NOT NULL);", "ALTER TABLE guidance ADD COLUMN attachments TEXT;", "ALTER TABLE guidance ADD COLUMN \"references\" TEXT;", "ALTER TABLE guidance ADD COLUMN verifier TEXT;"];
+export declare const PROFILE_SCHEMA = "\nCREATE TABLE profile (singleton INTEGER PRIMARY KEY CHECK(singleton = 1), id TEXT NOT NULL UNIQUE, is_enabled INTEGER NOT NULL, revision INTEGER NOT NULL);\nCREATE TABLE sources (id TEXT PRIMARY KEY, provider TEXT NOT NULL, remote_id TEXT NOT NULL,\n  title TEXT NOT NULL, url TEXT, is_included INTEGER NOT NULL, removed_at TEXT, resolved_remote_id TEXT,\n  UNIQUE(provider, remote_id));\nCREATE TABLE profile_references (id TEXT PRIMARY KEY, revision INTEGER NOT NULL,\n  content_hash TEXT NOT NULL, body TEXT NOT NULL);\nCREATE TABLE memberships (source_id TEXT NOT NULL REFERENCES sources(id),\n  reference_id TEXT NOT NULL REFERENCES profile_references(id), seen_job_id TEXT,\n  PRIMARY KEY(source_id, reference_id));\nCREATE INDEX memberships_reference ON memberships(reference_id);\nCREATE TABLE observations (reference_id TEXT PRIMARY KEY REFERENCES profile_references(id),\n  revision INTEGER NOT NULL, body TEXT NOT NULL);\nCREATE TABLE index_work (reference_id TEXT PRIMARY KEY REFERENCES profile_references(id),\n  id TEXT NOT NULL, revision INTEGER NOT NULL, operation TEXT NOT NULL,\n  engine_id TEXT, state TEXT NOT NULL);\nCREATE TABLE import_jobs (id TEXT PRIMARY KEY, source_id TEXT NOT NULL REFERENCES sources(id),\n  runtime TEXT NOT NULL, state TEXT NOT NULL, cursor TEXT, enumerated INTEGER NOT NULL,\n  total INTEGER, error TEXT, updated_at TEXT NOT NULL, is_enumerated INTEGER NOT NULL DEFAULT 0);\nCREATE INDEX import_jobs_source ON import_jobs(source_id);\nCREATE TABLE index_rebuild (singleton INTEGER PRIMARY KEY CHECK(singleton = 1), id TEXT NOT NULL, runtime TEXT NOT NULL);\n\nCREATE TABLE import_cursors (job_id TEXT NOT NULL REFERENCES import_jobs(id), cursor TEXT NOT NULL, PRIMARY KEY(job_id, cursor));\nCREATE TABLE reference_failures (job_id TEXT NOT NULL REFERENCES import_jobs(id), reference_id TEXT NOT NULL REFERENCES profile_references(id), revision INTEGER NOT NULL, stage TEXT NOT NULL, error TEXT NOT NULL, attempts INTEGER NOT NULL, updated_at TEXT NOT NULL, PRIMARY KEY(job_id, reference_id, stage));\n\nCREATE TABLE provider_cooldowns (provider TEXT PRIMARY KEY, retry_at TEXT NOT NULL, attempts INTEGER NOT NULL);\nCREATE TABLE import_collection_references (job_id TEXT NOT NULL REFERENCES import_jobs(id), collection_id TEXT NOT NULL, reference_id TEXT NOT NULL REFERENCES profile_references(id), PRIMARY KEY(job_id, collection_id, reference_id));\n\nCREATE TABLE reference_discoveries (id TEXT PRIMARY KEY, body TEXT NOT NULL);\nCREATE TABLE reference_plans (id TEXT PRIMARY KEY, body TEXT NOT NULL);\nCREATE TABLE planned_sources (source_id TEXT PRIMARY KEY REFERENCES sources(id), discovery_id TEXT NOT NULL REFERENCES reference_discoveries(id), collection_id TEXT NOT NULL);\nCREATE TABLE reference_onboarding (singleton INTEGER PRIMARY KEY CHECK(singleton = 1), state TEXT NOT NULL, offered_at TEXT, remind_at TEXT);\n\nCREATE TABLE connector_onboarding (provider TEXT PRIMARY KEY, state TEXT NOT NULL, offered_at TEXT, remind_at TEXT);\nCREATE TABLE guidance (id TEXT PRIMARY KEY, title TEXT NOT NULL, markdown TEXT NOT NULL, is_enabled INTEGER NOT NULL);\nALTER TABLE guidance ADD COLUMN attachments TEXT;\nALTER TABLE guidance ADD COLUMN \"references\" TEXT;\nALTER TABLE guidance ADD COLUMN verifier TEXT;\n\nALTER TABLE import_jobs ADD COLUMN added_count INTEGER NOT NULL DEFAULT 0;\nALTER TABLE import_jobs ADD COLUMN removed_count INTEGER NOT NULL DEFAULT 0;\nALTER TABLE reference_failures ADD COLUMN reason_code TEXT;\n\n";
+export declare const PROFILE_MIGRATIONS: readonly ["ALTER TABLE import_jobs ADD COLUMN is_enumerated INTEGER NOT NULL DEFAULT 0", "CREATE TABLE index_rebuild (singleton INTEGER PRIMARY KEY CHECK(singleton = 1), id TEXT NOT NULL, runtime TEXT NOT NULL);", "ALTER TABLE sources ADD COLUMN resolved_remote_id TEXT; \nCREATE TABLE import_cursors (job_id TEXT NOT NULL REFERENCES import_jobs(id), cursor TEXT NOT NULL, PRIMARY KEY(job_id, cursor));\nCREATE TABLE reference_failures (job_id TEXT NOT NULL REFERENCES import_jobs(id), reference_id TEXT NOT NULL REFERENCES profile_references(id), revision INTEGER NOT NULL, stage TEXT NOT NULL, error TEXT NOT NULL, attempts INTEGER NOT NULL, updated_at TEXT NOT NULL, PRIMARY KEY(job_id, reference_id, stage));\n", "CREATE TABLE provider_cooldowns (provider TEXT PRIMARY KEY, retry_at TEXT NOT NULL, attempts INTEGER NOT NULL);", "CREATE TABLE import_collection_references (job_id TEXT NOT NULL REFERENCES import_jobs(id), collection_id TEXT NOT NULL, reference_id TEXT NOT NULL REFERENCES profile_references(id), PRIMARY KEY(job_id, collection_id, reference_id));", "\nCREATE TABLE reference_discoveries (id TEXT PRIMARY KEY, body TEXT NOT NULL);\nCREATE TABLE reference_plans (id TEXT PRIMARY KEY, body TEXT NOT NULL);\nCREATE TABLE planned_sources (source_id TEXT PRIMARY KEY REFERENCES sources(id), discovery_id TEXT NOT NULL REFERENCES reference_discoveries(id), collection_id TEXT NOT NULL);\nCREATE TABLE reference_onboarding (singleton INTEGER PRIMARY KEY CHECK(singleton = 1), state TEXT NOT NULL, offered_at TEXT, remind_at TEXT);\n", "CREATE TABLE connector_onboarding (provider TEXT PRIMARY KEY, state TEXT NOT NULL, offered_at TEXT, remind_at TEXT);", "CREATE TABLE guidance (id TEXT PRIMARY KEY, title TEXT NOT NULL, markdown TEXT NOT NULL, is_enabled INTEGER NOT NULL);", "ALTER TABLE guidance ADD COLUMN attachments TEXT;", "ALTER TABLE guidance ADD COLUMN \"references\" TEXT;", "ALTER TABLE guidance ADD COLUMN verifier TEXT;", "\nALTER TABLE import_jobs ADD COLUMN added_count INTEGER NOT NULL DEFAULT 0;\nALTER TABLE import_jobs ADD COLUMN removed_count INTEGER NOT NULL DEFAULT 0;\nALTER TABLE reference_failures ADD COLUMN reason_code TEXT;\n"];
 export declare const PROFILE_SCHEMA_VERSION: number;
 //# sourceMappingURL=migrations.d.ts.map