pi-harness-runtime 0.10.13 → 0.10.15

package/RFC/0034_OBSERVABILITY.md

@@ -1,416 +0,0 @@
-# RFC 0034: Observability
-
-## Summary
-
-A comprehensive observability system for monitoring, tracing, metrics collection, and alerting in the harness runtime.
-
-## Motivation
-
-Currently, observability is minimal with scattered logging. We need:
-
-1. Structured logging with correlation IDs
-2. Distributed tracing across components
-3. Metrics collection and export
-4. Health endpoints
-5. Alerting system
-6. Dashboard-ready data exports
-
-## Architecture
-
-```
-+-----------------------------------------------------------------+
-|                      Observability System                         |
-+-----------------------------------------------------------------+
-|  +--------------+  +--------------+  +--------------+           |
-|  |   Logger     |  |   Tracer     |  |   Metrics    |             |
-|  |   (Structured|  |   (OpenTelemetry|  |   (Prometheus|           |
-|  |    Logging)  |  |    Compatible)|  |    Format)  |           |
-|  +--------------+  +--------------+  +--------------+           |
-+-----------------------------------------------------------------+
-|  +--------------+  +--------------+  +--------------+           |
-|  |   Health     |  |   Alerts     |  |   Exporters  |             |
-|  |   Monitor     |  |   Engine     |  |   (Prom/OTLP)|           |
-|  +--------------+  +--------------+  +--------------+           |
-+-----------------------------------------------------------------+
-```
-
-## Key Components
-
-### 1. Logger
-
-```typescript
-interface LoggerConfig {
-  level: LogLevel;                   // 'debug' | 'info' | 'warn' | 'error'
-  format: 'json' | 'pretty';        // Output format
-  output: 'stdout' | 'file' | 'both';
-  filePath?: string;
-  maxFileSize?: number;             // Rotation size in bytes
-  maxFiles?: number;               // Number of rotated files to keep
-  correlationIdHeader?: string;     // Header name for correlation ID
-}
-
-type LogLevel = 'debug' | 'info' | 'warn' | 'error' | 'fatal';
-
-interface LogEntry {
-  timestamp: string;
-  level: LogLevel;
-  message: string;
-  correlationId?: string;
-  traceId?: string;
-  spanId?: string;
-  jobId?: string;
-  taskId?: string;
-  component?: string;
-  durationMs?: number;
-  metadata?: Record<string, unknown>;
-}
-
-class Logger {
-  constructor(config: LoggerConfig);
-  
-  debug(message: string, meta?: object): void;
-  info(message: string, meta?: object): void;
-  warn(message: string, meta?: object): void;
-  error(message: string, error?: Error, meta?: object): void;
-  fatal(message: string, error?: Error, meta?: object): void;
-  
-  // Child logger with defaults
-  child(defaults: Partial<LogEntry>): Logger;
-  
-  // Correlation
-  withCorrelationId(id: string): Logger;
-  withJob(jobId: string): Logger;
-  withTask(taskId: string): Logger;
-}
-```
-
-### 2. Tracer (OpenTelemetry Compatible)
-
-```typescript
-interface TracerConfig {
-  serviceName: string;
-  exporter: 'console' | 'otlp' | 'jaeger';
-  endpoint?: string;
-  sampleRate: number;              // 0-1, percentage to sample
-}
-
-interface Span {
-  name: string;
-  traceId: string;
-  spanId: string;
-  parentSpanId?: string;
-  startTime: number;
-  endTime?: number;
-  status: 'ok' | 'error';
-  attributes: Record<string, string | number | boolean>;
-  events: SpanEvent[];
-  links: SpanLink[];
-}
-
-interface SpanEvent {
-  name: string;
-  timestamp: number;
-  attributes?: Record<string, string>;
-}
-
-interface SpanLink {
-  traceId: string;
-  spanId: string;
-}
-
-class Tracer {
-  constructor(config: TracerConfig);
-  
-  startSpan(name: string, options?: StartSpanOptions): SpanHandle;
-  recordException(span: SpanHandle, error: Error): void;
-  addEvent(span: SpanHandle, name: string, attributes?: object): void;
-  getCurrentSpan(): SpanHandle | null;
-  
-  // Async wrapper
-  trace<T>(name: string, fn: () => Promise<T>): Promise<T>;
-  trace<T>(name: string, options: SpanOptions, fn: () => Promise<T>): Promise<T>;
-}
-
-interface SpanHandle {
-  end(): void;
-  setStatus(status: 'ok' | 'error'): void;
-  setAttribute(key: string, value: string | number | boolean): void;
-  addEvent(name: string, attributes?: object): void;
-  recordException(error: Error): void;
-}
-```
-
-### 3. Metrics Collector
-
-```typescript
-interface MetricsConfig {
-  serviceName: string;
-  defaultLabels: Record<string, string>;
-  exportInterval: number;          // How often to flush metrics
-}
-
-class Metrics {
-  constructor(config: MetricsConfig);
-  
-  // Counters
-  counter(name: string, help: string, labels?: string[]): Counter;
-  increment(name: string, value?: number, labels?: Labels): void;
-  
-  // Gauges
-  gauge(name: string, help: string, labels?: string[]): Gauge;
-  set(name: string, value: number, labels?: Labels): void;
-  
-  // Histograms
-  histogram(name: string, help: string, buckets?: number[], labels?: string[]): Histogram;
-  observe(name: string, value: number, labels?: Labels): void;
-  
-  // Summaries
-  summary(name: string, help: string, labels?: string[]): Summary;
-  
-  // Timing helpers
-  timer(name: string, labels?: Labels): () => void;
-  asyncTimer<T>(name: string, fn: () => Promise<T>, labels?: Labels): Promise<T>;
-  
-  // Export
-  async export(): Promise<MetricsSnapshot>;
-  async exportPrometheus(): Promise<string>;
-}
-
-// Built-in metrics
-interface BuiltInMetrics {
-  // Job metrics
-  'harness_jobs_total': Counter;           // Total jobs created
-  'harness_jobs_active': Gauge;           // Currently active jobs
-  'harness_jobs_completed': Counter;      // Completed jobs
-  'harness_jobs_failed': Counter;         // Failed jobs
-  
-  // Task metrics
-  'harness_tasks_total': Counter;          // Total tasks created
-  'harness_tasks_duration_seconds': Histogram; // Task duration
-  'harness_tasks_retries_total': Counter; // Task retry count
-  
-  // Provider metrics
-  'harness_provider_requests_total': Counter;
-  'harness_provider_errors_total': Counter;
-  'harness_provider_latency_seconds': Histogram;
-  'harness_provider_tokens_total': Counter;
-  
-  // Session metrics
-  'harness_sessions_active': Gauge;
-  'harness_sessions_tokens_total': Counter;
-}
-```
-
-### 4. Health Monitor
-
-```typescript
-interface HealthCheck {
-  name: string;
-  check: () => Promise<HealthResult>;
-  critical: boolean;
-}
-
-interface HealthResult {
-  status: 'healthy' | 'degraded' | 'unhealthy';
-  message?: string;
-  details?: Record<string, unknown>;
-  timestamp: string;
-}
-
-class HealthMonitor {
-  constructor();
-  
-  registerHealthCheck(check: HealthCheck): void;
-  unregisterHealthCheck(name: string): void;
-  
-  async checkHealth(): Promise<HealthReport>;
-  async checkLiveness(): Promise<HealthResult>;  // K8s liveness probe
-  async checkReadiness(): Promise<HealthResult>;  // K8s readiness probe
-}
-
-interface HealthReport {
-  overall: 'healthy' | 'degraded' | 'unhealthy';
-  version: string;
-  uptime: number;
-  checks: Record<string, HealthResult>;
-  timestamp: string;
-}
-```
-
-### 5. Alerting Engine
-
-```typescript
-interface AlertRule {
-  name: string;
-  condition: AlertCondition;
-  severity: 'info' | 'warning' | 'critical';
-  cooldown: number;                // Minimum time between alerts
-  actions: AlertAction[];
-}
-
-type AlertCondition = 
-  | { type: 'threshold'; metric: string; op: '>' | '<' | '=='; value: number }
-  | { type: 'rate'; metric: string; window: number; threshold: number }
-  | { type: 'error_rate'; component: string; threshold: number };
-
-type AlertAction =
-  | { type: 'log'; level: LogLevel }
-  | { type: 'notify'; channel: string; message: string }
-  | { type: 'webhook'; url: string }
-  | { type: 'execute'; command: string };
-
-class AlertEngine {
-  constructor();
-  
-  registerRule(rule: AlertRule): void;
-  unregisterRule(name: string): void;
-  
-  evaluate(): Promise<Alert[]>;
-  triggerAlert(alert: Alert): Promise<void>;
-  getActiveAlerts(): Alert[];
-  getAlertHistory(): Alert[];
-}
-```
-
-## File Structure
-
-```
-packages/observability/
-+-- src/
-|   +-- index.ts                    # Public exports
-|   +-- logger.ts                  # Structured logger
-|   +-- tracer.ts                  # Distributed tracer
-|   +-- metrics.ts                 # Metrics collector
-|   +-- health.ts                  # Health monitor
-|   +-- alerts.ts                  # Alerting engine
-|   +-- exporters/
-|   |   +-- prometheus.ts          # Prometheus exporter
-|   |   +-- otlp.ts                # OTLP exporter
-|   +-- middleware.ts              # Express/HTTP middleware
-|   +-- types.ts                   # Types
-|   +-- errors.ts
-+-- test/
-+-- examples/
-+-- package.json
-+-- README.md
-```
-
-## Usage Examples
-
-### Structured Logging
-
-```typescript
-import { createLogger } from '@pi/observability';
-
-const logger = createLogger({
-  level: 'info',
-  format: 'json',
-  output: 'both',
-  filePath: './logs/harness.log'
-});
-
-const jobLogger = logger.withJob('job-123');
-jobLogger.info('Job started', { requirement: 'Build a web app' });
-
-// Output: {"timestamp":"...","level":"info","message":"Job started","jobId":"job-123","requirement":"Build a web app","traceId":"..."}
-```
-
-### Distributed Tracing
-
-```typescript
-import { createTracer } from '@pi/observability';
-
-const tracer = createTracer({
-  serviceName: 'harness-runtime',
-  exporter: 'otlp',
-  endpoint: 'http://localhost:4318'
-});
-
-const result = await tracer.trace('processJob', async (span) => {
-  span.setAttribute('jobId', 'job-123');
-  
-  await tracer.trace('planTasks', async () => {
-    // ...
-  });
-  
-  return result;
-});
-```
-
-### Metrics Collection
-
-```typescript
-import { createMetrics } from '@pi/observability';
-
-const metrics = createMetrics({ serviceName: 'harness-runtime' });
-
-// Record task duration
-const endTimer = metrics.asyncTimer('task_duration', { taskType: 'code' });
-await executeTask();
-endTimer();
-
-// Check metrics
-const snapshot = await metrics.exportPrometheus();
-// # HELP harness_task_duration_seconds Task duration
-// # TYPE harness_task_duration_seconds histogram
-```
-
-### Health Checks
-
-```typescript
-const health = new HealthMonitor();
-
-health.registerHealthCheck({
-  name: 'providers',
-  critical: true,
-  check: async () => {
-    const providers = await checkProviderHealth();
-    return providers.length > 0 
-      ? { status: 'healthy' }
-      : { status: 'unhealthy', message: 'No providers available' };
-  }
-});
-
-// K8s endpoints
-app.get('/health/liveness', async (req, res) => {
-  const result = await health.checkLiveness();
-  res.status(result.status === 'healthy' ? 200 : 503).json(result);
-});
-
-app.get('/health/readiness', async (req, res) => {
-  const result = await health.checkReadiness();
-  res.status(result.status === 'healthy' ? 200 : 503).json(result);
-});
-```
-
-## Integration
-
-```
-+-------------+     +--------------+     +-------------+
-|   Harness   |---->| Observability |---->|  Prometheus |
-|   Runtime   |     |   Package     |     |   Server    |
-+-------------+     +--------------+     +-------------+
-                           |
-                           ▼
-                    +--------------+
-                    |   Grafana    |
-                    |  Dashboard   |
-                    +--------------+
-```
-
-## Acceptance Criteria
-
-1. ✅ All log entries include correlation IDs for tracing
-2. ✅ Tracer supports OpenTelemetry protocol
-3. ✅ Metrics export in Prometheus format
-4. ✅ Health endpoints for Kubernetes probes
-5. ✅ Alert rules can trigger on metric thresholds
-6. ✅ Logs are structured JSON with required fields
-7. ✅ Performance overhead < 5% for logging/tracing
-
-## Dependencies
-
-- `packages/types` - for runtime-types
-- Optional: `@opentelemetry/sdk-node` - OTLP export
-- Optional: `prom-client` - Prometheus format
-- No required external dependencies

package/RFC/0035_ROADMAP_V1.md

@@ -1,295 +0,0 @@
-# RFC 0035: Roadmap V1
-
-## Summary
-
-A structured roadmap for implementing RFCs 0031-0040, including dependencies, priorities, and milestones.
-
-## Overview
-
-```
-Phase 1: Foundation          Phase 2: Core Systems         Phase 3: Advanced Features
------------------           --------------------           -------------------------
-• RFC-0031: Adapter SDK     • RFC-0032: Checkpoint Engine  • RFC-0036: Code Review Engine
-• RFC-0034: Observability   • RFC-0033: Session Manager     • RFC-0037: Code Generation Pipeline
-                           • RFC-0034: Observability        • RFC-0038: Framework Detector
-                                                           • RFC-0039: Test Data Generator
-                                                           • RFC-0040: Framework Plugin SDK
-```
-
-## Implementation Order & Dependencies
-
-```
-RFC-0031 (Adapter SDK) --+--► RFC-0036 (Code Review Engine)
-                          |
-RFC-0032 (Checkpoint) ◄--+
-                          |
-RFC-0033 (Session) ◄-----+--► RFC-0034 (Observability)
-                          |
-RFC-0034 (Observability) ◄+
-
-RFC-0038 (Framework Detector) --+--► RFC-0039 (Test Data Generator)
-                                 |
-RFC-0037 (Code Generation) ◄-----+
-                                 |
-RFC-0040 (Plugin SDK) ◄----------+
-```
-
-## Detailed Timeline
-
-### Phase 1: Foundation (Weeks 1-2)
-
-#### Week 1: RFC-0031 Provider Adapter SDK
-
-**Goal:** Standardized adapter development
-
-| Task | Description | Estimate |
-| ------ | ------------- | ---------- |
-| T1.1 | Design AdapterBuilder API | 2 days |
-| T1.2 | Implement AdapterRegistry | 2 days |
-| T1.3 | Create AdapterTester utilities | 2 days |
-| T1.4 | Migrate existing adapters | 1 day |
-
-**Deliverables:**
-
-- `packages/provider-adapter-sdk/src/`
-- Working Builder pattern for adapters
-- Test utilities for adapter validation
-
-#### Week 2: RFC-0034 Observability
-
-**Goal:** Full observability stack
-
-| Task | Description | Estimate |
-| ------ | ------------- | ---------- |
-| T2.1 | Implement Logger with correlation IDs | 2 days |
-| T2.2 | Implement Tracer (OpenTelemetry compatible) | 2 days |
-| T2.3 | Implement Metrics collector | 2 days |
-| T2.4 | Health monitor & alerting | 1 day |
-| T2.5 | Prometheus exporter | 1 day |
-
-**Deliverables:**
-
-- `packages/observability/src/`
-- Structured JSON logging
-- Distributed tracing support
-- Prometheus metrics endpoint
-
----
-
-### Phase 2: Core Systems (Weeks 3-5)
-
-#### Week 3: RFC-0032 Checkpoint Engine
-
-**Goal:** Enhanced checkpoint system
-
-| Task | Description | Estimate |
-| ------ | ------------- | ---------- |
-| T3.1 | Implement incremental diff storage | 2 days |
-| T3.2 | Add compression (gzip) | 1 day |
-| T3.3 | Implement recovery strategies | 2 days |
-| T3.4 | Add checksum verification | 1 day |
-| T3.5 | Auto-pruning logic | 1 day |
-
-**Deliverables:**
-
-- Enhanced `packages/checkpoint/`
-- 60%+ storage reduction
-- Sub-100ms recovery times
-
-#### Week 4: RFC-0033 Session Manager
-
-**Goal:** Comprehensive session management
-
-| Task | Description | Estimate |
-| ------ | ------------- | ---------- |
-| T4.1 | Implement Session store | 2 days |
-| T4.2 | Message history management | 2 days |
-| T4.3 | Session policy engine | 2 days |
-| T4.4 | Metrics collection | 1 day |
-| T4.5 | Integration with checkpoint | 1 day |
-
-**Deliverables:**
-
-- `packages/session/src/`
-- Session persistence across restarts
-- Token accounting per session
-
-#### Week 5: Integration & Testing
-
-**Goal:** Ensure all foundation components work together
-
-| Task | Description | Estimate |
-|------|-------------|----------|
-| T5.1 | Integration tests | 3 days |
-| T5.2 | Performance benchmarks | 1 day |
-| T5.3 | Documentation | 1 day |
-
----
-
-### Phase 3: Advanced Features (Weeks 6-10)
-
-#### Week 6-7: RFC-0036 Code Review Engine
-
-**Goal:** Automated code review pipeline
-
-| Task | Description | Estimate |
-| ------ | ------------- | ---------- |
-| T6.1 | Design review criteria DSL | 2 days |
-| T6.2 | Implement static analysis adapters | 2 days |
-| T6.3 | Build review aggregator | 2 days |
-| T6.4 | Integrate with AgentWorker | 2 days |
-
-**Deliverables:**
-
-- `packages/code-review/`
-- Configurable review rules
-- Linter integration
-- AI-powered review suggestions
-
-#### Week 7-8: RFC-0037 Code Generation Pipeline
-
-**Goal:** Automated code generation with validation
-
-| Task | Description | Estimate |
-| ------ | ------------- | ---------- |
-| T7.1 | Design generation template system | 2 days |
-| T7.2 | Implement template engine | 2 days |
-| T7.3 | Build validation pipeline | 2 days |
-| T7.4 | Add rollback mechanisms | 1 day |
-| T7.5 | Integrate with framework detector | 1 day |
-
-**Deliverables:**
-
-- `packages/code-generation/`
-- Template-based generation
-- Post-generation validation
-- Rollback on failure
-
-#### Week 8-9: RFC-0038 Framework Detector
-
-**Goal:** Detect project frameworks automatically
-
-| Task | Description | Estimate |
-| ------ | ------------- | ---------- |
-| T8.1 | Implement detection signals | 2 days |
-| T8.2 | Build confidence scoring | 1 day |
-| T8.3 | Add Frappe/ERPNext detection | 2 days |
-| T8.4 | Add Next.js/React detection | 2 days |
-| T8.5 | Add Django/Laravel detection | 1 day |
-
-**Deliverables:**
-
-- `packages/framework-detector/`
-- Multi-framework support
-- Signal-based detection
-- Confidence scoring
-
-#### Week 9-10: RFC-0039 Test Data Generator & RFC-0040 Plugin SDK
-
-##### RFC-0039: Test Data Generator
-
-| Task | Description | Estimate |
-| ------ | ------------- | ---------- |
-| T9.1 | Design data generation DSL | 1 day |
-| T9.2 | Implement faker adapters | 2 days |
-| T9.3 | Framework-specific generators | 2 days |
-| T9.4 | Integration with E2E engine | 1 day |
-
-##### RFC-0040: Framework Plugin SDK
-
-| Task | Description | Estimate |
-| ------ | ------------- | ---------- |
-| T10.1 | Design plugin interface | 1 day |
-| T10.2 | Implement plugin loader | 2 days |
-| T10.3 | Create plugin registry | 1 day |
-| T10.4 | Add sandboxing/security | 2 days |
-| T10.5 | Write plugin examples | 1 day |
-
----
-
-## Resource Allocation
-
-| Phase | Team Size | Focus |
-| ------- | ----------- | ------- |
-| Phase 1 | 1-2 | Foundation packages |
-| Phase 2 | 2 | Core systems + integration |
-| Phase 3 | 2-3 | Advanced features |
-
-## Risk Mitigation
-
-| Risk | Impact | Mitigation |
-| ------ | -------- | ------------ |
-| Framework detection accuracy | Medium | Iterative improvement with signal weighting |
-| Plugin security | High | Sandboxing + permission model |
-| Performance overhead | Low | Benchmarks in each PR |
-| Integration complexity | Medium | Weekly integration tests |
-
-## Success Metrics
-
-| Metric | Target |
-| -------- | -------- |
-| Checkpoint storage reduction | >60% |
-| Recovery time | <100ms |
-| Session restore success rate | >99% |
-| Framework detection accuracy | >90% |
-| Code review coverage | Configurable rules |
-| Plugin load time | <500ms |
-
-## Files to Create
-
-```
-packages/
-+-- provider-adapter-sdk/           # RFC-0031
-|   +-- src/
-|   +-- test/
-|   +-- package.json
-+-- observability/                   # RFC-0034
-|   +-- src/
-|   +-- test/
-|   +-- package.json
-+-- checkpoint/                      # RFC-0032 (enhanced)
-|   +-- src/
-|   +-- package.json
-+-- session/                         # RFC-0033
-|   +-- src/
-|   +-- test/
-|   +-- package.json
-+-- code-review/                     # RFC-0036
-|   +-- src/
-|   +-- test/
-|   +-- package.json
-+-- code-generation/                 # RFC-0037
-|   +-- src/
-|   +-- test/
-|   +-- package.json
-+-- framework-detector/              # RFC-0038
-|   +-- src/
-|   +-- test/
-|   +-- package.json
-+-- test-data-generator/             # RFC-0039
-|   +-- src/
-|   +-- test/
-|   +-- package.json
-+-- framework-plugin-sdk/            # RFC-0040
-    +-- src/
-    +-- test/
-    +-- package.json
-```
-
-## Implementation Checklist
-
-- [ ] Phase 1 Complete
-  - [ ] RFC-0031 Provider Adapter SDK
-  - [ ] RFC-0034 Observability
-- [ ] Phase 2 Complete
-  - [ ] RFC-0032 Checkpoint Engine
-  - [ ] RFC-0033 Session Manager
-  - [ ] Integration tests passing
-- [ ] Phase 3 Complete
-  - [ ] RFC-0036 Code Review Engine
-  - [ ] RFC-0037 Code Generation Pipeline
-  - [ ] RFC-0038 Framework Detector
-  - [ ] RFC-0039 Test Data Generator
-  - [ ] RFC-0040 Framework Plugin SDK
-- [ ] Documentation complete
-- [ ] Performance benchmarks meet targets