prjct-cli 1.18.0 → 1.19.0

package/CHANGELOG.md

@@ -1,5 +1,55 @@
 # Changelog
 
+## [1.19.0] - 2026-02-09
+
+### Features
+
+- implement aggressive archival of stale storage data (PRJ-267) (#161)
+
+
+## [1.19.0] - 2026-02-09
+
+### Features
+
+- **Aggressive archival of stale storage data** (PRJ-267): Automatic archival during `prjct sync` to keep LLM context lean
+  - Shipped features >90 days archived to SQLite `archives` table with 1-line summary
+  - Pending ideas >180 days marked `dormant` and excluded from LLM context
+  - Completed queue tasks >7 days auto-removed and archived
+  - Paused tasks >30 days archived with persistence (previously discarded)
+  - Memory log capped at 500 active entries, overflow archived
+
+### Implementation Details
+
+New modules:
+- `core/storage/archive-storage.ts` — Archive infrastructure: SQLite `archives` table, batch archival via transactions, restore, prune, stats
+- `core/__tests__/storage/archive-storage.test.ts` — 13 tests covering all archival paths
+
+Modified:
+- `core/storage/database.ts` — Migration v2: `archives` table with entity_type, entity_id, entity_data, summary, reason columns
+- `core/storage/shipped-storage.ts` — `archiveOldShipped()` method with 90-day retention policy
+- `core/storage/ideas-storage.ts` — `markDormantIdeas()` method, `dormant` status excluded from markdown context
+- `core/storage/queue-storage.ts` — `removeStaleCompleted()` method with 7-day retention
+- `core/storage/state-storage.ts` — `archiveStalePausedTasks()` now persists to archive table before removal
+- `core/services/memory-service.ts` — `capEntries()` method with 500-entry cap
+- `core/services/sync-service.ts` — `archiveStaleData()` orchestrates all archival in parallel during sync
+- `core/schemas/ideas.ts` + `core/types/storage.ts` — Added `dormant` to IdeaStatus enum
+
+### Test Plan
+
+#### For QA
+1. Run `prjct sync` with >90d shipped features — verify archive and removal from context
+2. Run sync with >180d pending ideas — verify dormant status, excluded from `ideas.md`
+3. Run sync with >7d completed queue tasks — verify removal and archival
+4. Run sync with >30d paused tasks — verify archival to SQLite
+5. Create >500 memory entries, sync — verify cap at 500
+6. `bun test` — all 947+ tests pass
+7. Verify recent items are NOT archived
+
+#### For Users
+**What changed:** Storage data automatically cleaned up during sync. Old data archived, not deleted.
+**How to use:** No action needed — runs automatically on every sync.
+**Breaking changes:** Ideas can now have `dormant` status (new enum value).
+
 ## [1.18.0] - 2026-02-09
 
 ### Features

package/core/__tests__/storage/archive-storage.test.ts

@@ -0,0 +1,455 @@
+/**
+ * Archive Storage Tests (PRJ-267)
+ *
+ * Tests for the archive infrastructure and archival policies:
+ * - Archive table operations (insert, query, restore)
+ * - Shipped features archival (>90 days)
+ * - Ideas dormancy (>180 days pending)
+ * - Queue cleanup (>7 days completed)
+ * - Paused task archival (>30 days)
+ * - Memory log capping (500 entries)
+ */
+
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test'
+import fs from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+import pathManager from '../../infrastructure/path-manager'
+import { ARCHIVE_POLICIES, archiveStorage } from '../../storage/archive-storage'
+import { prjctDb } from '../../storage/database'
+import { ideasStorage } from '../../storage/ideas-storage'
+import { queueStorage } from '../../storage/queue-storage'
+import { shippedStorage } from '../../storage/shipped-storage'
+import { stateStorage } from '../../storage/state-storage'
+import { getTimestamp } from '../../utils/date-helper'
+
+// =============================================================================
+// Test Setup
+// =============================================================================
+
+let tmpRoot: string
+let testProjectId: string
+
+const originalGetGlobalProjectPath = pathManager.getGlobalProjectPath.bind(pathManager)
+const originalGetFilePath = pathManager.getFilePath.bind(pathManager)
+
+function daysAgoISO(days: number): string {
+  const d = new Date()
+  d.setDate(d.getDate() - days)
+  return d.toISOString()
+}
+
+describe('Archive Storage', () => {
+  beforeEach(async () => {
+    tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'prjct-archive-test-'))
+    testProjectId = 'test-archive-project'
+
+    pathManager.getGlobalProjectPath = (projectId: string) => path.join(tmpRoot, projectId)
+
+    pathManager.getFilePath = (projectId: string, layer: string, filename: string) =>
+      path.join(tmpRoot, projectId, layer, filename)
+
+    // Ensure all required dirs exist
+    const dirs = ['context', 'memory', 'core', 'progress', 'planning', 'sync']
+    await Promise.all(
+      dirs.map((d) => fs.mkdir(path.join(tmpRoot, testProjectId, d), { recursive: true }))
+    )
+
+    // Create empty pending.json for event bus
+    await fs.writeFile(path.join(tmpRoot, testProjectId, 'sync', 'pending.json'), '[]', 'utf-8')
+
+    // Initialize the database (triggers migrations including archives table)
+    prjctDb.getDb(testProjectId)
+  })
+
+  afterEach(async () => {
+    prjctDb.close()
+    pathManager.getGlobalProjectPath = originalGetGlobalProjectPath
+    pathManager.getFilePath = originalGetFilePath
+
+    if (tmpRoot) {
+      await fs.rm(tmpRoot, { recursive: true, force: true })
+    }
+  })
+
+  // ===========================================================================
+  // Archive Table Operations
+  // ===========================================================================
+
+  describe('archive table', () => {
+    it('should archive a single item', () => {
+      const id = archiveStorage.archive(testProjectId, {
+        entityType: 'shipped',
+        entityId: 'ship-1',
+        entityData: { name: 'Feature A', version: '1.0.0' },
+        summary: 'Feature A v1.0.0',
+        reason: 'age',
+      })
+
+      expect(id).toBeTruthy()
+
+      const records = archiveStorage.getArchived(testProjectId, 'shipped')
+      expect(records).toHaveLength(1)
+      expect(records[0].entity_id).toBe('ship-1')
+      expect(records[0].summary).toBe('Feature A v1.0.0')
+    })
+
+    it('should archive multiple items in a transaction', () => {
+      const count = archiveStorage.archiveMany(testProjectId, [
+        { entityType: 'shipped', entityId: 's1', entityData: { a: 1 }, reason: 'age' },
+        { entityType: 'shipped', entityId: 's2', entityData: { a: 2 }, reason: 'age' },
+        { entityType: 'idea', entityId: 'i1', entityData: { b: 1 }, reason: 'dormant' },
+      ])
+
+      expect(count).toBe(3)
+
+      const stats = archiveStorage.getStats(testProjectId)
+      expect(stats.shipped).toBe(2)
+      expect(stats.idea).toBe(1)
+      expect(stats.total).toBe(3)
+    })
+
+    it('should restore an archived item', () => {
+      archiveStorage.archive(testProjectId, {
+        entityType: 'shipped',
+        entityId: 'ship-1',
+        entityData: { name: 'restored' },
+        reason: 'age',
+      })
+
+      const records = archiveStorage.getArchived(testProjectId)
+      expect(records).toHaveLength(1)
+
+      const data = archiveStorage.restore(testProjectId, records[0].id)
+      expect(data).toEqual({ name: 'restored' })
+
+      // Should be removed from archive
+      const after = archiveStorage.getArchived(testProjectId)
+      expect(after).toHaveLength(0)
+    })
+
+    it('should prune old archives', () => {
+      // Insert an archive with old timestamp
+      const db = prjctDb.getDb(testProjectId)
+      const oldDate = daysAgoISO(400)
+      db.prepare(
+        'INSERT INTO archives (id, entity_type, entity_id, entity_data, archived_at, reason) VALUES (?, ?, ?, ?, ?, ?)'
+      ).run('old-1', 'shipped', 's1', '{}', oldDate, 'age')
+
+      archiveStorage.archive(testProjectId, {
+        entityType: 'shipped',
+        entityId: 's2',
+        entityData: {},
+        reason: 'age',
+      })
+
+      const pruned = archiveStorage.pruneOldArchives(testProjectId, 365)
+      expect(pruned).toBe(1)
+
+      const remaining = archiveStorage.getArchived(testProjectId)
+      expect(remaining).toHaveLength(1)
+      expect(remaining[0].entity_id).toBe('s2')
+    })
+  })
+
+  // ===========================================================================
+  // Shipped Features Archival
+  // ===========================================================================
+
+  describe('shipped archival', () => {
+    it('should archive shipped features older than 90 days', async () => {
+      // Write shipped data with old and recent items
+      await shippedStorage.write(testProjectId, {
+        shipped: [
+          { id: 'recent', name: 'Recent', version: '2.0.0', shippedAt: daysAgoISO(10) },
+          { id: 'old', name: 'Old', version: '1.0.0', shippedAt: daysAgoISO(100) },
+        ],
+        lastUpdated: getTimestamp(),
+      })
+
+      const archived = await shippedStorage.archiveOldShipped(testProjectId)
+      expect(archived).toBe(1)
+
+      // Verify active storage only has recent
+      const data = await shippedStorage.read(testProjectId)
+      expect(data.shipped).toHaveLength(1)
+      expect(data.shipped[0].id).toBe('recent')
+
+      // Verify archive table has old item
+      const records = archiveStorage.getArchived(testProjectId, 'shipped')
+      expect(records).toHaveLength(1)
+      expect(records[0].entity_id).toBe('old')
+      expect(records[0].summary).toBe('Old v1.0.0')
+    })
+
+    it('should not archive recent shipped features', async () => {
+      await shippedStorage.write(testProjectId, {
+        shipped: [
+          { id: 'r1', name: 'R1', version: '1.0.0', shippedAt: daysAgoISO(5) },
+          { id: 'r2', name: 'R2', version: '1.1.0', shippedAt: daysAgoISO(30) },
+        ],
+        lastUpdated: getTimestamp(),
+      })
+
+      const archived = await shippedStorage.archiveOldShipped(testProjectId)
+      expect(archived).toBe(0)
+
+      const data = await shippedStorage.read(testProjectId)
+      expect(data.shipped).toHaveLength(2)
+    })
+  })
+
+  // ===========================================================================
+  // Ideas Dormancy
+  // ===========================================================================
+
+  describe('ideas dormancy', () => {
+    it('should mark pending ideas older than 180 days as dormant', async () => {
+      await ideasStorage.write(testProjectId, {
+        ideas: [
+          {
+            id: 'new',
+            text: 'New idea',
+            status: 'pending',
+            priority: 'medium',
+            tags: [],
+            addedAt: daysAgoISO(10),
+          },
+          {
+            id: 'stale',
+            text: 'Stale idea',
+            status: 'pending',
+            priority: 'low',
+            tags: [],
+            addedAt: daysAgoISO(200),
+          },
+          {
+            id: 'converted',
+            text: 'Converted',
+            status: 'converted',
+            priority: 'high',
+            tags: [],
+            addedAt: daysAgoISO(300),
+          },
+        ],
+        lastUpdated: getTimestamp(),
+      })
+
+      const dormant = await ideasStorage.markDormantIdeas(testProjectId)
+      expect(dormant).toBe(1)
+
+      const data = await ideasStorage.read(testProjectId)
+      const stale = data.ideas.find((i) => i.id === 'stale')
+      expect(stale?.status).toBe('dormant')
+
+      // New idea should remain pending
+      const fresh = data.ideas.find((i) => i.id === 'new')
+      expect(fresh?.status).toBe('pending')
+
+      // Converted should remain converted
+      const conv = data.ideas.find((i) => i.id === 'converted')
+      expect(conv?.status).toBe('converted')
+
+      // Archive table should have the dormant idea
+      const records = archiveStorage.getArchived(testProjectId, 'idea')
+      expect(records).toHaveLength(1)
+    })
+
+    it('should exclude dormant ideas from markdown context', async () => {
+      await ideasStorage.write(testProjectId, {
+        ideas: [
+          {
+            id: 'active',
+            text: 'Active idea',
+            status: 'pending',
+            priority: 'medium',
+            tags: [],
+            addedAt: daysAgoISO(5),
+          },
+          {
+            id: 'dormant',
+            text: 'Dormant idea',
+            status: 'dormant',
+            priority: 'low',
+            tags: [],
+            addedAt: daysAgoISO(200),
+          },
+        ],
+        lastUpdated: getTimestamp(),
+      })
+
+      // Read the generated markdown context file
+      const contextPath = pathManager.getFilePath(testProjectId, 'planning', 'ideas.md')
+      const md = await fs.readFile(contextPath, 'utf-8')
+
+      expect(md).toContain('Active idea')
+      expect(md).not.toContain('Dormant idea')
+      expect(md).toContain('1 dormant idea(s) excluded from context')
+    })
+  })
+
+  // ===========================================================================
+  // Queue Cleanup
+  // ===========================================================================
+
+  describe('queue cleanup', () => {
+    it('should remove completed tasks older than 7 days', async () => {
+      await queueStorage.write(testProjectId, {
+        tasks: [
+          {
+            id: 'active',
+            description: 'Active',
+            type: 'feature',
+            priority: 'medium',
+            section: 'active',
+            createdAt: daysAgoISO(1),
+            completed: false,
+          },
+          {
+            id: 'recent-done',
+            description: 'Recent done',
+            type: 'feature',
+            priority: 'medium',
+            section: 'active',
+            createdAt: daysAgoISO(5),
+            completed: true,
+            completedAt: daysAgoISO(2),
+          },
+          {
+            id: 'old-done',
+            description: 'Old done',
+            type: 'feature',
+            priority: 'low',
+            section: 'active',
+            createdAt: daysAgoISO(30),
+            completed: true,
+            completedAt: daysAgoISO(10),
+          },
+        ],
+        lastUpdated: getTimestamp(),
+      })
+
+      const removed = await queueStorage.removeStaleCompleted(testProjectId)
+      expect(removed).toBe(1)
+
+      const data = await queueStorage.read(testProjectId)
+      expect(data.tasks).toHaveLength(2)
+      expect(data.tasks.map((t) => t.id).sort()).toEqual(['active', 'recent-done'])
+
+      // Archive should have the old completed task
+      const records = archiveStorage.getArchived(testProjectId, 'queue_task')
+      expect(records).toHaveLength(1)
+      expect(records[0].entity_id).toBe('old-done')
+    })
+  })
+
+  // ===========================================================================
+  // Paused Task Archival
+  // ===========================================================================
+
+  describe('paused task archival', () => {
+    it('should archive paused tasks older than 30 days', async () => {
+      await stateStorage.write(testProjectId, {
+        currentTask: null,
+        previousTask: null,
+        pausedTasks: [
+          {
+            id: 'recent',
+            description: 'Recent pause',
+            status: 'paused',
+            startedAt: daysAgoISO(35),
+            pausedAt: daysAgoISO(5),
+          },
+          {
+            id: 'stale',
+            description: 'Stale pause',
+            status: 'paused',
+            startedAt: daysAgoISO(60),
+            pausedAt: daysAgoISO(40),
+          },
+        ],
+        lastUpdated: getTimestamp(),
+      })
+
+      const archived = await stateStorage.archiveStalePausedTasks(testProjectId)
+      expect(archived).toHaveLength(1)
+      expect(archived[0].id).toBe('stale')
+
+      // Active state should only have recent
+      const state = await stateStorage.read(testProjectId)
+      expect(state.pausedTasks).toHaveLength(1)
+      expect(state.pausedTasks![0].id).toBe('recent')
+
+      // Archive table should have stale
+      const records = archiveStorage.getArchived(testProjectId, 'paused_task')
+      expect(records).toHaveLength(1)
+      expect(records[0].entity_id).toBe('stale')
+    })
+  })
+
+  // ===========================================================================
+  // Memory Log Capping
+  // ===========================================================================
+
+  describe('memory log capping', () => {
+    it('should cap memory entries at max limit', async () => {
+      const memoryPath = pathManager.getFilePath(testProjectId, 'memory', 'context.jsonl')
+
+      // Write more entries than the limit
+      const entries: string[] = []
+      const total = ARCHIVE_POLICIES.MEMORY_MAX_ENTRIES + 50
+      for (let i = 0; i < total; i++) {
+        entries.push(
+          JSON.stringify({
+            timestamp: new Date(Date.now() - (total - i) * 1000).toISOString(),
+            action: `action-${i}`,
+            data: { index: i },
+          })
+        )
+      }
+      await fs.writeFile(memoryPath, `${entries.join('\n')}\n`, 'utf-8')
+
+      // Import and use memoryService
+      const { memoryService } = await import('../../services/memory-service')
+      const capped = await memoryService.capEntries(testProjectId)
+      expect(capped).toBe(50)
+
+      // File should now have exactly max entries
+      const content = await fs.readFile(memoryPath, 'utf-8')
+      const remaining = content.trim().split('\n').filter(Boolean)
+      expect(remaining).toHaveLength(ARCHIVE_POLICIES.MEMORY_MAX_ENTRIES)
+
+      // Archive should have the overflow
+      const records = archiveStorage.getArchived(testProjectId, 'memory_entry')
+      expect(records).toHaveLength(50)
+    })
+
+    it('should not cap if under limit', async () => {
+      const memoryPath = pathManager.getFilePath(testProjectId, 'memory', 'context.jsonl')
+
+      const entries: string[] = []
+      for (let i = 0; i < 10; i++) {
+        entries.push(JSON.stringify({ timestamp: getTimestamp(), action: `a-${i}`, data: {} }))
+      }
+      await fs.writeFile(memoryPath, `${entries.join('\n')}\n`, 'utf-8')
+
+      const { memoryService } = await import('../../services/memory-service')
+      const capped = await memoryService.capEntries(testProjectId)
+      expect(capped).toBe(0)
+    })
+  })
+
+  // ===========================================================================
+  // Archive Policies Constants
+  // ===========================================================================
+
+  describe('archive policies', () => {
+    it('should have correct default policy values', () => {
+      expect(ARCHIVE_POLICIES.SHIPPED_RETENTION_DAYS).toBe(90)
+      expect(ARCHIVE_POLICIES.IDEA_DORMANT_DAYS).toBe(180)
+      expect(ARCHIVE_POLICIES.QUEUE_COMPLETED_DAYS).toBe(7)
+      expect(ARCHIVE_POLICIES.PAUSED_TASK_DAYS).toBe(30)
+      expect(ARCHIVE_POLICIES.MEMORY_MAX_ENTRIES).toBe(500)
+    })
+  })
+})

package/core/schemas/ideas.ts

@@ -14,7 +14,7 @@ import { z } from 'zod'
 // =============================================================================
 
 export const IdeaPrioritySchema = z.enum(['low', 'medium', 'high'])
-export const IdeaStatusSchema = z.enum(['pending', 'converted', 'completed', 'archived'])
+export const IdeaStatusSchema = z.enum(['pending', 'converted', 'completed', 'archived', 'dormant'])
 export const ImpactLevelSchema = z.enum(['high', 'medium', 'low'])
 
 export const ImpactEffortSchema = z.object({

package/core/services/memory-service.ts

@@ -6,6 +6,7 @@
 
 import configManager from '../infrastructure/config-manager'
 import pathManager from '../infrastructure/path-manager'
+import { ARCHIVE_POLICIES, archiveStorage } from '../storage/archive-storage'
 import type { MemoryServiceEntry } from '../types'
 import { getErrorMessage, isNotFoundError } from '../types/fs'
 import { getTimestamp } from '../utils/date-helper'
@@ -133,6 +134,47 @@ export class MemoryService {
       return []
     }
   }
+
+  /**
+   * Cap memory log at max entries (PRJ-267).
+   * Moves overflow entries to archive table, keeps most recent entries.
+   * Returns count of archived entries.
+   */
+  async capEntries(projectId: string): Promise<number> {
+    try {
+      const memoryPath = pathManager.getFilePath(projectId, 'memory', 'context.jsonl')
+      const entries = await jsonlHelper.readJsonLines<MemoryServiceEntry>(memoryPath)
+
+      if (entries.length <= ARCHIVE_POLICIES.MEMORY_MAX_ENTRIES) {
+        return 0
+      }
+
+      const overflow = entries.slice(0, entries.length - ARCHIVE_POLICIES.MEMORY_MAX_ENTRIES)
+      const kept = entries.slice(-ARCHIVE_POLICIES.MEMORY_MAX_ENTRIES)
+
+      // Archive overflow entries in batch
+      archiveStorage.archiveMany(
+        projectId,
+        overflow.map((entry, i) => ({
+          entityType: 'memory_entry' as const,
+          entityId: `memory-${entry.timestamp || i}`,
+          entityData: entry,
+          summary: entry.action,
+          reason: 'overflow',
+        }))
+      )
+
+      // Rewrite file with only kept entries
+      await jsonlHelper.writeJsonLines(memoryPath, kept)
+
+      return overflow.length
+    } catch (error) {
+      if (!isNotFoundError(error)) {
+        console.error(`Memory cap error: ${getErrorMessage(error)}`)
+      }
+      return 0
+    }
+  }
 }
 
 export const memoryService = new MemoryService()

package/core/services/sync-service.ts

@@ -37,8 +37,13 @@ import commandInstaller from '../infrastructure/command-installer'
 import configManager from '../infrastructure/config-manager'
 import pathManager from '../infrastructure/path-manager'
 import { analysisStorage } from '../storage/analysis-storage'
+import { archiveStorage } from '../storage/archive-storage'
+import { ideasStorage } from '../storage/ideas-storage'
 import { metricsStorage } from '../storage/metrics-storage'
 import { migrateJsonToSqlite } from '../storage/migrate-json'
+import { queueStorage } from '../storage/queue-storage'
+import { shippedStorage } from '../storage/shipped-storage'
+import { stateStorage } from '../storage/state-storage'
 import type {
   GitData,
   IncrementalInfo,
@@ -56,6 +61,7 @@ import * as dateHelper from '../utils/date-helper'
 import log from '../utils/logger'
 import { ContextFileGenerator } from './context-generator'
 import { localStateGenerator } from './local-state-generator'
+import { memoryService } from './memory-service'
 import { skillInstaller } from './skill-installer'
 import { StackDetector } from './stack-detector'
 import { syncVerifier } from './sync-verifier'
@@ -290,6 +296,9 @@ class SyncService {
       const duration = Date.now() - startTime
       const syncMetrics = await this.recordSyncMetrics(stats, contextFiles, agents, duration)
 
+      // 9b. Archive stale data (PRJ-267)
+      await this.archiveStaleData()
+
       // 10. Update global config and commands (CLI does EVERYTHING)
       // This ensures `prjct sync` from terminal updates global CLAUDE.md and commands
       await commandInstaller.installGlobalConfig()
@@ -1270,6 +1279,48 @@ You are the ${name} expert for this project. Apply best practices for the detect
     }
   }
 
+  // ==========================================================================
+  // ARCHIVAL (PRJ-267)
+  // ==========================================================================
+
+  /**
+   * Archive stale data across all storage types.
+   * Runs during sync to keep active storage lean.
+   */
+  private async archiveStaleData(): Promise<void> {
+    if (!this.projectId) return
+
+    try {
+      const [shipped, dormant, staleQueue, stalePaused, memoryCapped] = await Promise.all([
+        shippedStorage.archiveOldShipped(this.projectId).catch(() => 0),
+        ideasStorage.markDormantIdeas(this.projectId).catch(() => 0),
+        queueStorage.removeStaleCompleted(this.projectId).catch(() => 0),
+        stateStorage.archiveStalePausedTasks(this.projectId).catch(() => []),
+        memoryService.capEntries(this.projectId).catch(() => 0),
+      ])
+
+      const totalArchived =
+        shipped + dormant + staleQueue + (stalePaused as unknown[]).length + memoryCapped
+
+      if (totalArchived > 0) {
+        log.info('Archived stale data', {
+          shipped,
+          dormant,
+          staleQueue,
+          stalePaused: (stalePaused as unknown[]).length,
+          memoryCapped,
+          total: totalArchived,
+        })
+
+        // Record archive stats
+        const stats = archiveStorage.getStats(this.projectId)
+        log.debug('Archive stats', stats)
+      }
+    } catch (error) {
+      log.debug('Archival failed (non-critical)', { error: getErrorMessage(error) })
+    }
+  }
+
   // ==========================================================================
   // HELPERS
   // ==========================================================================