services-as-software 2.1.1 → 2.1.3

package/tests/metrics.test.ts

@@ -0,0 +1,478 @@
+/**
+ * Service Metrics Tests
+ *
+ * Tests for quality assurance and performance metrics tracking.
+ */
+
+import { describe, it, expect, vi } from 'vitest'
+import {
+  createMetricsTracker,
+  trackQuality,
+  trackPerformance,
+  aggregateMetrics,
+  createQualityGate,
+  evaluateQuality,
+  MetricType,
+  QualityDimension,
+} from '../src/metrics/index.js'
+
+describe('Service Metrics', () => {
+  describe('Metrics Tracker', () => {
+    it('should create a metrics tracker', () => {
+      const tracker = createMetricsTracker({
+        serviceId: 'translation-service',
+        retention: '30d',
+      })
+
+      expect(tracker.serviceId).toBe('translation-service')
+      expect(tracker.retention).toBe('30d')
+    })
+
+    it('should record metrics', async () => {
+      const tracker = createMetricsTracker({
+        serviceId: 'test-service',
+      })
+
+      await tracker.record({
+        name: 'requests',
+        value: 1,
+        type: 'counter',
+        timestamp: new Date(),
+      })
+
+      const metrics = await tracker.query({ name: 'requests' })
+      expect(metrics).toHaveLength(1)
+      expect(metrics[0].value).toBe(1)
+    })
+
+    it('should support metric dimensions/labels', async () => {
+      const tracker = createMetricsTracker({
+        serviceId: 'test-service',
+      })
+
+      await tracker.record({
+        name: 'response_time',
+        value: 150,
+        type: 'gauge',
+        dimensions: {
+          endpoint: '/translate',
+          region: 'us-east',
+        },
+      })
+
+      const metrics = await tracker.query({
+        name: 'response_time',
+        dimensions: { endpoint: '/translate' },
+      })
+
+      expect(metrics).toHaveLength(1)
+      expect(metrics[0].dimensions.endpoint).toBe('/translate')
+    })
+
+    it('should aggregate metrics over time', async () => {
+      const tracker = createMetricsTracker({
+        serviceId: 'test-service',
+      })
+
+      // Record multiple values
+      for (let i = 0; i < 10; i++) {
+        await tracker.record({
+          name: 'latency',
+          value: 100 + i * 10, // 100-190
+          type: 'histogram',
+        })
+      }
+
+      const aggregated = await tracker.aggregate({
+        name: 'latency',
+        aggregations: ['avg', 'p50', 'p95', 'p99', 'min', 'max'],
+      })
+
+      expect(aggregated.avg).toBe(145)
+      expect(aggregated.min).toBe(100)
+      expect(aggregated.max).toBe(190)
+    })
+  })
+
+  describe('Quality Metrics', () => {
+    it('should track quality score', async () => {
+      const result = await trackQuality({
+        executionId: 'exec-123',
+        dimensions: {
+          accuracy: 0.95,
+          completeness: 0.90,
+          timeliness: 1.0,
+          customerSatisfaction: 0.85,
+        },
+        weights: {
+          accuracy: 0.3,
+          completeness: 0.25,
+          timeliness: 0.25,
+          customerSatisfaction: 0.2,
+        },
+      })
+
+      expect(result.overallScore).toBeCloseTo(0.9225)
+      expect(result.dimensions.accuracy).toBe(0.95)
+    })
+
+    it('should track execution quality with AI assessment', async () => {
+      const result = await trackQuality({
+        executionId: 'exec-456',
+        output: 'Generated content here...',
+        aiAssessment: {
+          enabled: true,
+          criteria: ['relevance', 'clarity', 'accuracy'],
+        },
+      })
+
+      expect(result.aiScores).toBeDefined()
+      expect(result.aiScores.relevance).toBeDefined()
+    })
+
+    it('should compare against baseline quality', async () => {
+      const result = await trackQuality({
+        executionId: 'exec-789',
+        dimensions: { accuracy: 0.85 },
+        baseline: { accuracy: 0.80 },
+      })
+
+      expect(result.comparison.accuracy.delta).toBe(0.05)
+      expect(result.comparison.accuracy.improved).toBe(true)
+    })
+
+    it('should flag quality issues', async () => {
+      const result = await trackQuality({
+        executionId: 'exec-111',
+        dimensions: {
+          accuracy: 0.60, // below threshold
+          completeness: 0.95,
+        },
+        thresholds: {
+          accuracy: 0.80,
+          completeness: 0.90,
+        },
+      })
+
+      expect(result.issues).toContainEqual(
+        expect.objectContaining({
+          dimension: 'accuracy',
+          severity: 'high',
+        })
+      )
+    })
+  })
+
+  describe('Performance Metrics', () => {
+    it('should track execution performance', async () => {
+      const result = await trackPerformance({
+        executionId: 'exec-123',
+        metrics: {
+          duration: 5000, // ms
+          tokensUsed: 1500,
+          apiCalls: 3,
+          cost: 0.05,
+        },
+      })
+
+      expect(result.duration).toBe(5000)
+      expect(result.tokensUsed).toBe(1500)
+      expect(result.cost).toBe(0.05)
+    })
+
+    it('should track resource utilization', async () => {
+      const result = await trackPerformance({
+        executionId: 'exec-456',
+        resourceUsage: {
+          cpu: 45, // percentage
+          memory: 512, // MB
+          storage: 100, // MB
+        },
+      })
+
+      expect(result.resourceUsage.cpu).toBe(45)
+      expect(result.resourceUsage.memory).toBe(512)
+    })
+
+    it('should track throughput', async () => {
+      const result = await trackPerformance({
+        executionId: 'exec-789',
+        throughput: {
+          requestsPerSecond: 100,
+          itemsProcessed: 1000,
+          processingRate: 200, // items per minute
+        },
+      })
+
+      expect(result.throughput.requestsPerSecond).toBe(100)
+    })
+
+    it('should compare performance to targets', async () => {
+      const result = await trackPerformance({
+        executionId: 'exec-111',
+        metrics: { duration: 3000 },
+        targets: { duration: 2000 },
+      })
+
+      expect(result.comparison.duration.exceeded).toBe(true)
+      expect(result.comparison.duration.percentage).toBe(50) // 50% over target
+    })
+
+    it('should detect performance anomalies', async () => {
+      const result = await trackPerformance({
+        executionId: 'exec-222',
+        metrics: { duration: 10000 },
+        anomalyDetection: {
+          enabled: true,
+          baseline: { duration: { mean: 2000, stdDev: 500 } },
+          threshold: 3, // 3 standard deviations
+        },
+      })
+
+      expect(result.anomalies).toContainEqual(
+        expect.objectContaining({
+          metric: 'duration',
+          deviation: expect.any(Number),
+        })
+      )
+    })
+  })
+
+  describe('Metrics Aggregation', () => {
+    it('should aggregate metrics by time period', async () => {
+      const metrics = [
+        { timestamp: new Date('2024-01-01'), value: 100 },
+        { timestamp: new Date('2024-01-01'), value: 200 },
+        { timestamp: new Date('2024-01-02'), value: 150 },
+        { timestamp: new Date('2024-01-02'), value: 250 },
+      ]
+
+      const aggregated = await aggregateMetrics(metrics, {
+        groupBy: 'day',
+        aggregation: 'avg',
+      })
+
+      expect(aggregated).toHaveLength(2)
+      expect(aggregated[0].value).toBe(150) // avg of 100, 200
+      expect(aggregated[1].value).toBe(200) // avg of 150, 250
+    })
+
+    it('should aggregate metrics by dimension', async () => {
+      const metrics = [
+        { region: 'us-east', value: 100 },
+        { region: 'us-east', value: 200 },
+        { region: 'eu-west', value: 150 },
+      ]
+
+      const aggregated = await aggregateMetrics(metrics, {
+        groupBy: 'region',
+        aggregation: 'sum',
+      })
+
+      expect(aggregated.find(m => m.region === 'us-east')?.value).toBe(300)
+      expect(aggregated.find(m => m.region === 'eu-west')?.value).toBe(150)
+    })
+
+    it('should calculate percentiles', async () => {
+      const metrics = Array.from({ length: 100 }, (_, i) => ({
+        value: i + 1, // 1-100
+      }))
+
+      const aggregated = await aggregateMetrics(metrics, {
+        aggregation: 'percentiles',
+        percentiles: [50, 90, 95, 99],
+      })
+
+      expect(aggregated.p50).toBe(50)
+      expect(aggregated.p90).toBe(90)
+      expect(aggregated.p95).toBe(95)
+      expect(aggregated.p99).toBe(99)
+    })
+
+    it('should calculate rate of change', async () => {
+      const metrics = [
+        { timestamp: new Date('2024-01-01'), value: 100 },
+        { timestamp: new Date('2024-01-02'), value: 120 },
+        { timestamp: new Date('2024-01-03'), value: 150 },
+      ]
+
+      const aggregated = await aggregateMetrics(metrics, {
+        aggregation: 'rate',
+      })
+
+      expect(aggregated.changeRate).toBeCloseTo(0.25) // 50% increase over 2 days = 25% daily
+    })
+  })
+
+  describe('Quality Gates', () => {
+    it('should create a quality gate', () => {
+      const gate = createQualityGate({
+        name: 'Production Ready',
+        stage: 'delivery',
+        criteria: [
+          { metric: 'accuracy', threshold: 0.95, comparison: '>=' },
+          { metric: 'testCoverage', threshold: 80, comparison: '>=' },
+          { metric: 'errorRate', threshold: 0.01, comparison: '<=' },
+        ],
+      })
+
+      expect(gate.name).toBe('Production Ready')
+      expect(gate.criteria).toHaveLength(3)
+    })
+
+    it('should evaluate quality gate pass', async () => {
+      const gate = createQualityGate({
+        name: 'Basic QA',
+        criteria: [
+          { metric: 'accuracy', threshold: 0.9, comparison: '>=' },
+        ],
+      })
+
+      const result = await evaluateQuality(gate, {
+        accuracy: 0.95,
+      })
+
+      expect(result.passed).toBe(true)
+      expect(result.results[0].passed).toBe(true)
+    })
+
+    it('should evaluate quality gate failure', async () => {
+      const gate = createQualityGate({
+        name: 'Strict QA',
+        criteria: [
+          { metric: 'accuracy', threshold: 0.95, comparison: '>=' },
+          { metric: 'completeness', threshold: 0.90, comparison: '>=' },
+        ],
+      })
+
+      const result = await evaluateQuality(gate, {
+        accuracy: 0.92, // below threshold
+        completeness: 0.95,
+      })
+
+      expect(result.passed).toBe(false)
+      expect(result.failedCriteria).toContain('accuracy')
+    })
+
+    it('should support weighted criteria', async () => {
+      const gate = createQualityGate({
+        name: 'Weighted QA',
+        criteria: [
+          { metric: 'accuracy', threshold: 0.9, weight: 0.5 },
+          { metric: 'speed', threshold: 1000, comparison: '<=', weight: 0.3 },
+          { metric: 'cost', threshold: 0.10, comparison: '<=', weight: 0.2 },
+        ],
+        passingScore: 0.8, // 80% weighted score to pass
+      })
+
+      const result = await evaluateQuality(gate, {
+        accuracy: 0.95, // passes
+        speed: 800, // passes
+        cost: 0.15, // fails
+      })
+
+      // Score: 0.5 + 0.3 + 0 = 0.8, exactly at threshold
+      expect(result.score).toBeCloseTo(0.8)
+      expect(result.passed).toBe(true)
+    })
+
+    it('should handle required vs optional criteria', async () => {
+      const gate = createQualityGate({
+        name: 'Mixed QA',
+        criteria: [
+          { metric: 'security', threshold: 1.0, required: true },
+          { metric: 'performance', threshold: 0.9, required: false },
+        ],
+      })
+
+      // Fails required criteria
+      let result = await evaluateQuality(gate, {
+        security: 0.95,
+        performance: 0.95,
+      })
+      expect(result.passed).toBe(false)
+
+      // Passes required, fails optional
+      result = await evaluateQuality(gate, {
+        security: 1.0,
+        performance: 0.85,
+      })
+      expect(result.passed).toBe(true)
+      expect(result.warnings).toContain('performance')
+    })
+  })
+
+  describe('Quality Dimensions', () => {
+    it('should evaluate accuracy dimension', async () => {
+      const score = await evaluateQuality.dimension('accuracy', {
+        expected: 'The capital of France is Paris.',
+        actual: 'Paris is the capital of France.',
+        evaluator: 'semantic', // semantic similarity
+      })
+
+      expect(score.value).toBeGreaterThan(0.9)
+    })
+
+    it('should evaluate completeness dimension', async () => {
+      const score = await evaluateQuality.dimension('completeness', {
+        requirements: ['introduction', 'body', 'conclusion', 'references'],
+        deliverables: ['introduction', 'body', 'conclusion'],
+      })
+
+      expect(score.value).toBe(0.75) // 3/4 complete
+      expect(score.missing).toContain('references')
+    })
+
+    it('should evaluate timeliness dimension', async () => {
+      const score = await evaluateQuality.dimension('timeliness', {
+        deadline: new Date('2024-01-15'),
+        completedAt: new Date('2024-01-14'),
+      })
+
+      expect(score.value).toBe(1.0) // on time
+      expect(score.onTime).toBe(true)
+    })
+
+    it('should evaluate customer satisfaction', async () => {
+      const score = await evaluateQuality.dimension('customerSatisfaction', {
+        ratings: [5, 4, 5, 4, 5, 3, 4, 5],
+        feedback: ['Great work!', 'Very satisfied', 'Minor issues but overall good'],
+      })
+
+      expect(score.value).toBeCloseTo(0.875) // avg 4.375 out of 5
+      expect(score.nps).toBeDefined()
+    })
+  })
+
+  describe('Metrics Dashboard', () => {
+    it('should generate metrics dashboard data', async () => {
+      const tracker = createMetricsTracker({ serviceId: 'test-service' })
+
+      const dashboard = await tracker.getDashboard({
+        timeRange: '7d',
+        metrics: ['requests', 'latency', 'errors', 'quality'],
+      })
+
+      expect(dashboard.summary).toBeDefined()
+      expect(dashboard.charts).toBeDefined()
+      expect(dashboard.trends).toBeDefined()
+    })
+
+    it('should calculate health score', async () => {
+      const tracker = createMetricsTracker({ serviceId: 'test-service' })
+
+      const health = await tracker.getHealthScore({
+        weights: {
+          uptime: 0.3,
+          performance: 0.3,
+          quality: 0.2,
+          customerSatisfaction: 0.2,
+        },
+      })
+
+      expect(health.score).toBeGreaterThanOrEqual(0)
+      expect(health.score).toBeLessThanOrEqual(1)
+      expect(health.status).toMatch(/healthy|degraded|unhealthy/)
+    })
+  })
+})