pi-harness-runtime 0.10.12 → 0.10.13

package/RFC/0034_OBSERVABILITY.md

@@ -0,0 +1,416 @@
+# RFC 0034: Observability
+
+## Summary
+
+A comprehensive observability system for monitoring, tracing, metrics collection, and alerting in the harness runtime.
+
+## Motivation
+
+Currently, observability is minimal with scattered logging. We need:
+
+1. Structured logging with correlation IDs
+2. Distributed tracing across components
+3. Metrics collection and export
+4. Health endpoints
+5. Alerting system
+6. Dashboard-ready data exports
+
+## Architecture
+
+```
++-----------------------------------------------------------------+
+|                      Observability System                         |
++-----------------------------------------------------------------+
+|  +--------------+  +--------------+  +--------------+           |
+|  |   Logger     |  |   Tracer     |  |   Metrics    |             |
+|  |   (Structured|  |   (OpenTelemetry|  |   (Prometheus|           |
+|  |    Logging)  |  |    Compatible)|  |    Format)  |           |
+|  +--------------+  +--------------+  +--------------+           |
++-----------------------------------------------------------------+
+|  +--------------+  +--------------+  +--------------+           |
+|  |   Health     |  |   Alerts     |  |   Exporters  |             |
+|  |   Monitor     |  |   Engine     |  |   (Prom/OTLP)|           |
+|  +--------------+  +--------------+  +--------------+           |
++-----------------------------------------------------------------+
+```
+
+## Key Components
+
+### 1. Logger
+
+```typescript
+interface LoggerConfig {
+  level: LogLevel;                   // 'debug' | 'info' | 'warn' | 'error'
+  format: 'json' | 'pretty';        // Output format
+  output: 'stdout' | 'file' | 'both';
+  filePath?: string;
+  maxFileSize?: number;             // Rotation size in bytes
+  maxFiles?: number;               // Number of rotated files to keep
+  correlationIdHeader?: string;     // Header name for correlation ID
+}
+
+type LogLevel = 'debug' | 'info' | 'warn' | 'error' | 'fatal';
+
+interface LogEntry {
+  timestamp: string;
+  level: LogLevel;
+  message: string;
+  correlationId?: string;
+  traceId?: string;
+  spanId?: string;
+  jobId?: string;
+  taskId?: string;
+  component?: string;
+  durationMs?: number;
+  metadata?: Record<string, unknown>;
+}
+
+class Logger {
+  constructor(config: LoggerConfig);
+  
+  debug(message: string, meta?: object): void;
+  info(message: string, meta?: object): void;
+  warn(message: string, meta?: object): void;
+  error(message: string, error?: Error, meta?: object): void;
+  fatal(message: string, error?: Error, meta?: object): void;
+  
+  // Child logger with defaults
+  child(defaults: Partial<LogEntry>): Logger;
+  
+  // Correlation
+  withCorrelationId(id: string): Logger;
+  withJob(jobId: string): Logger;
+  withTask(taskId: string): Logger;
+}
+```
+
+### 2. Tracer (OpenTelemetry Compatible)
+
+```typescript
+interface TracerConfig {
+  serviceName: string;
+  exporter: 'console' | 'otlp' | 'jaeger';
+  endpoint?: string;
+  sampleRate: number;              // 0-1, percentage to sample
+}
+
+interface Span {
+  name: string;
+  traceId: string;
+  spanId: string;
+  parentSpanId?: string;
+  startTime: number;
+  endTime?: number;
+  status: 'ok' | 'error';
+  attributes: Record<string, string | number | boolean>;
+  events: SpanEvent[];
+  links: SpanLink[];
+}
+
+interface SpanEvent {
+  name: string;
+  timestamp: number;
+  attributes?: Record<string, string>;
+}
+
+interface SpanLink {
+  traceId: string;
+  spanId: string;
+}
+
+class Tracer {
+  constructor(config: TracerConfig);
+  
+  startSpan(name: string, options?: StartSpanOptions): SpanHandle;
+  recordException(span: SpanHandle, error: Error): void;
+  addEvent(span: SpanHandle, name: string, attributes?: object): void;
+  getCurrentSpan(): SpanHandle | null;
+  
+  // Async wrapper
+  trace<T>(name: string, fn: () => Promise<T>): Promise<T>;
+  trace<T>(name: string, options: SpanOptions, fn: () => Promise<T>): Promise<T>;
+}
+
+interface SpanHandle {
+  end(): void;
+  setStatus(status: 'ok' | 'error'): void;
+  setAttribute(key: string, value: string | number | boolean): void;
+  addEvent(name: string, attributes?: object): void;
+  recordException(error: Error): void;
+}
+```
+
+### 3. Metrics Collector
+
+```typescript
+interface MetricsConfig {
+  serviceName: string;
+  defaultLabels: Record<string, string>;
+  exportInterval: number;          // How often to flush metrics
+}
+
+class Metrics {
+  constructor(config: MetricsConfig);
+  
+  // Counters
+  counter(name: string, help: string, labels?: string[]): Counter;
+  increment(name: string, value?: number, labels?: Labels): void;
+  
+  // Gauges
+  gauge(name: string, help: string, labels?: string[]): Gauge;
+  set(name: string, value: number, labels?: Labels): void;
+  
+  // Histograms
+  histogram(name: string, help: string, buckets?: number[], labels?: string[]): Histogram;
+  observe(name: string, value: number, labels?: Labels): void;
+  
+  // Summaries
+  summary(name: string, help: string, labels?: string[]): Summary;
+  
+  // Timing helpers
+  timer(name: string, labels?: Labels): () => void;
+  asyncTimer<T>(name: string, fn: () => Promise<T>, labels?: Labels): Promise<T>;
+  
+  // Export
+  async export(): Promise<MetricsSnapshot>;
+  async exportPrometheus(): Promise<string>;
+}
+
+// Built-in metrics
+interface BuiltInMetrics {
+  // Job metrics
+  'harness_jobs_total': Counter;           // Total jobs created
+  'harness_jobs_active': Gauge;           // Currently active jobs
+  'harness_jobs_completed': Counter;      // Completed jobs
+  'harness_jobs_failed': Counter;         // Failed jobs
+  
+  // Task metrics
+  'harness_tasks_total': Counter;          // Total tasks created
+  'harness_tasks_duration_seconds': Histogram; // Task duration
+  'harness_tasks_retries_total': Counter; // Task retry count
+  
+  // Provider metrics
+  'harness_provider_requests_total': Counter;
+  'harness_provider_errors_total': Counter;
+  'harness_provider_latency_seconds': Histogram;
+  'harness_provider_tokens_total': Counter;
+  
+  // Session metrics
+  'harness_sessions_active': Gauge;
+  'harness_sessions_tokens_total': Counter;
+}
+```
+
+### 4. Health Monitor
+
+```typescript
+interface HealthCheck {
+  name: string;
+  check: () => Promise<HealthResult>;
+  critical: boolean;
+}
+
+interface HealthResult {
+  status: 'healthy' | 'degraded' | 'unhealthy';
+  message?: string;
+  details?: Record<string, unknown>;
+  timestamp: string;
+}
+
+class HealthMonitor {
+  constructor();
+  
+  registerHealthCheck(check: HealthCheck): void;
+  unregisterHealthCheck(name: string): void;
+  
+  async checkHealth(): Promise<HealthReport>;
+  async checkLiveness(): Promise<HealthResult>;  // K8s liveness probe
+  async checkReadiness(): Promise<HealthResult>;  // K8s readiness probe
+}
+
+interface HealthReport {
+  overall: 'healthy' | 'degraded' | 'unhealthy';
+  version: string;
+  uptime: number;
+  checks: Record<string, HealthResult>;
+  timestamp: string;
+}
+```
+
+### 5. Alerting Engine
+
+```typescript
+interface AlertRule {
+  name: string;
+  condition: AlertCondition;
+  severity: 'info' | 'warning' | 'critical';
+  cooldown: number;                // Minimum time between alerts
+  actions: AlertAction[];
+}
+
+type AlertCondition = 
+  | { type: 'threshold'; metric: string; op: '>' | '<' | '=='; value: number }
+  | { type: 'rate'; metric: string; window: number; threshold: number }
+  | { type: 'error_rate'; component: string; threshold: number };
+
+type AlertAction =
+  | { type: 'log'; level: LogLevel }
+  | { type: 'notify'; channel: string; message: string }
+  | { type: 'webhook'; url: string }
+  | { type: 'execute'; command: string };
+
+class AlertEngine {
+  constructor();
+  
+  registerRule(rule: AlertRule): void;
+  unregisterRule(name: string): void;
+  
+  evaluate(): Promise<Alert[]>;
+  triggerAlert(alert: Alert): Promise<void>;
+  getActiveAlerts(): Alert[];
+  getAlertHistory(): Alert[];
+}
+```
+
+## File Structure
+
+```
+packages/observability/
++-- src/
+|   +-- index.ts                    # Public exports
+|   +-- logger.ts                  # Structured logger
+|   +-- tracer.ts                  # Distributed tracer
+|   +-- metrics.ts                 # Metrics collector
+|   +-- health.ts                  # Health monitor
+|   +-- alerts.ts                  # Alerting engine
+|   +-- exporters/
+|   |   +-- prometheus.ts          # Prometheus exporter
+|   |   +-- otlp.ts                # OTLP exporter
+|   +-- middleware.ts              # Express/HTTP middleware
+|   +-- types.ts                   # Types
+|   +-- errors.ts
++-- test/
++-- examples/
++-- package.json
++-- README.md
+```
+
+## Usage Examples
+
+### Structured Logging
+
+```typescript
+import { createLogger } from '@pi/observability';
+
+const logger = createLogger({
+  level: 'info',
+  format: 'json',
+  output: 'both',
+  filePath: './logs/harness.log'
+});
+
+const jobLogger = logger.withJob('job-123');
+jobLogger.info('Job started', { requirement: 'Build a web app' });
+
+// Output: {"timestamp":"...","level":"info","message":"Job started","jobId":"job-123","requirement":"Build a web app","traceId":"..."}
+```
+
+### Distributed Tracing
+
+```typescript
+import { createTracer } from '@pi/observability';
+
+const tracer = createTracer({
+  serviceName: 'harness-runtime',
+  exporter: 'otlp',
+  endpoint: 'http://localhost:4318'
+});
+
+const result = await tracer.trace('processJob', async (span) => {
+  span.setAttribute('jobId', 'job-123');
+  
+  await tracer.trace('planTasks', async () => {
+    // ...
+  });
+  
+  return result;
+});
+```
+
+### Metrics Collection
+
+```typescript
+import { createMetrics } from '@pi/observability';
+
+const metrics = createMetrics({ serviceName: 'harness-runtime' });
+
+// Record task duration
+const endTimer = metrics.asyncTimer('task_duration', { taskType: 'code' });
+await executeTask();
+endTimer();
+
+// Check metrics
+const snapshot = await metrics.exportPrometheus();
+// # HELP harness_task_duration_seconds Task duration
+// # TYPE harness_task_duration_seconds histogram
+```
+
+### Health Checks
+
+```typescript
+const health = new HealthMonitor();
+
+health.registerHealthCheck({
+  name: 'providers',
+  critical: true,
+  check: async () => {
+    const providers = await checkProviderHealth();
+    return providers.length > 0 
+      ? { status: 'healthy' }
+      : { status: 'unhealthy', message: 'No providers available' };
+  }
+});
+
+// K8s endpoints
+app.get('/health/liveness', async (req, res) => {
+  const result = await health.checkLiveness();
+  res.status(result.status === 'healthy' ? 200 : 503).json(result);
+});
+
+app.get('/health/readiness', async (req, res) => {
+  const result = await health.checkReadiness();
+  res.status(result.status === 'healthy' ? 200 : 503).json(result);
+});
+```
+
+## Integration
+
+```
++-------------+     +--------------+     +-------------+
+|   Harness   |---->| Observability |---->|  Prometheus |
+|   Runtime   |     |   Package     |     |   Server    |
++-------------+     +--------------+     +-------------+
+                           |
+                           ▼
+                    +--------------+
+                    |   Grafana    |
+                    |  Dashboard   |
+                    +--------------+
+```
+
+## Acceptance Criteria
+
+1. ✅ All log entries include correlation IDs for tracing
+2. ✅ Tracer supports OpenTelemetry protocol
+3. ✅ Metrics export in Prometheus format
+4. ✅ Health endpoints for Kubernetes probes
+5. ✅ Alert rules can trigger on metric thresholds
+6. ✅ Logs are structured JSON with required fields
+7. ✅ Performance overhead < 5% for logging/tracing
+
+## Dependencies
+
+- `packages/types` - for runtime-types
+- Optional: `@opentelemetry/sdk-node` - OTLP export
+- Optional: `prom-client` - Prometheus format
+- No required external dependencies

package/RFC/0035_ROADMAP_V1.md

@@ -0,0 +1,295 @@
+# RFC 0035: Roadmap V1
+
+## Summary
+
+A structured roadmap for implementing RFCs 0031-0040, including dependencies, priorities, and milestones.
+
+## Overview
+
+```
+Phase 1: Foundation          Phase 2: Core Systems         Phase 3: Advanced Features
+-----------------           --------------------           -------------------------
+• RFC-0031: Adapter SDK     • RFC-0032: Checkpoint Engine  • RFC-0036: Code Review Engine
+• RFC-0034: Observability   • RFC-0033: Session Manager     • RFC-0037: Code Generation Pipeline
+                           • RFC-0034: Observability        • RFC-0038: Framework Detector
+                                                           • RFC-0039: Test Data Generator
+                                                           • RFC-0040: Framework Plugin SDK
+```
+
+## Implementation Order & Dependencies
+
+```
+RFC-0031 (Adapter SDK) --+--► RFC-0036 (Code Review Engine)
+                          |
+RFC-0032 (Checkpoint) ◄--+
+                          |
+RFC-0033 (Session) ◄-----+--► RFC-0034 (Observability)
+                          |
+RFC-0034 (Observability) ◄+
+
+RFC-0038 (Framework Detector) --+--► RFC-0039 (Test Data Generator)
+                                 |
+RFC-0037 (Code Generation) ◄-----+
+                                 |
+RFC-0040 (Plugin SDK) ◄----------+
+```
+
+## Detailed Timeline
+
+### Phase 1: Foundation (Weeks 1-2)
+
+#### Week 1: RFC-0031 Provider Adapter SDK
+
+**Goal:** Standardized adapter development
+
+| Task | Description | Estimate |
+| ------ | ------------- | ---------- |
+| T1.1 | Design AdapterBuilder API | 2 days |
+| T1.2 | Implement AdapterRegistry | 2 days |
+| T1.3 | Create AdapterTester utilities | 2 days |
+| T1.4 | Migrate existing adapters | 1 day |
+
+**Deliverables:**
+
+- `packages/provider-adapter-sdk/src/`
+- Working Builder pattern for adapters
+- Test utilities for adapter validation
+
+#### Week 2: RFC-0034 Observability
+
+**Goal:** Full observability stack
+
+| Task | Description | Estimate |
+| ------ | ------------- | ---------- |
+| T2.1 | Implement Logger with correlation IDs | 2 days |
+| T2.2 | Implement Tracer (OpenTelemetry compatible) | 2 days |
+| T2.3 | Implement Metrics collector | 2 days |
+| T2.4 | Health monitor & alerting | 1 day |
+| T2.5 | Prometheus exporter | 1 day |
+
+**Deliverables:**
+
+- `packages/observability/src/`
+- Structured JSON logging
+- Distributed tracing support
+- Prometheus metrics endpoint
+
+---
+
+### Phase 2: Core Systems (Weeks 3-5)
+
+#### Week 3: RFC-0032 Checkpoint Engine
+
+**Goal:** Enhanced checkpoint system
+
+| Task | Description | Estimate |
+| ------ | ------------- | ---------- |
+| T3.1 | Implement incremental diff storage | 2 days |
+| T3.2 | Add compression (gzip) | 1 day |
+| T3.3 | Implement recovery strategies | 2 days |
+| T3.4 | Add checksum verification | 1 day |
+| T3.5 | Auto-pruning logic | 1 day |
+
+**Deliverables:**
+
+- Enhanced `packages/checkpoint/`
+- 60%+ storage reduction
+- Sub-100ms recovery times
+
+#### Week 4: RFC-0033 Session Manager
+
+**Goal:** Comprehensive session management
+
+| Task | Description | Estimate |
+| ------ | ------------- | ---------- |
+| T4.1 | Implement Session store | 2 days |
+| T4.2 | Message history management | 2 days |
+| T4.3 | Session policy engine | 2 days |
+| T4.4 | Metrics collection | 1 day |
+| T4.5 | Integration with checkpoint | 1 day |
+
+**Deliverables:**
+
+- `packages/session/src/`
+- Session persistence across restarts
+- Token accounting per session
+
+#### Week 5: Integration & Testing
+
+**Goal:** Ensure all foundation components work together
+
+| Task | Description | Estimate |
+|------|-------------|----------|
+| T5.1 | Integration tests | 3 days |
+| T5.2 | Performance benchmarks | 1 day |
+| T5.3 | Documentation | 1 day |
+
+---
+
+### Phase 3: Advanced Features (Weeks 6-10)
+
+#### Week 6-7: RFC-0036 Code Review Engine
+
+**Goal:** Automated code review pipeline
+
+| Task | Description | Estimate |
+| ------ | ------------- | ---------- |
+| T6.1 | Design review criteria DSL | 2 days |
+| T6.2 | Implement static analysis adapters | 2 days |
+| T6.3 | Build review aggregator | 2 days |
+| T6.4 | Integrate with AgentWorker | 2 days |
+
+**Deliverables:**
+
+- `packages/code-review/`
+- Configurable review rules
+- Linter integration
+- AI-powered review suggestions
+
+#### Week 7-8: RFC-0037 Code Generation Pipeline
+
+**Goal:** Automated code generation with validation
+
+| Task | Description | Estimate |
+| ------ | ------------- | ---------- |
+| T7.1 | Design generation template system | 2 days |
+| T7.2 | Implement template engine | 2 days |
+| T7.3 | Build validation pipeline | 2 days |
+| T7.4 | Add rollback mechanisms | 1 day |
+| T7.5 | Integrate with framework detector | 1 day |
+
+**Deliverables:**
+
+- `packages/code-generation/`
+- Template-based generation
+- Post-generation validation
+- Rollback on failure
+
+#### Week 8-9: RFC-0038 Framework Detector
+
+**Goal:** Detect project frameworks automatically
+
+| Task | Description | Estimate |
+| ------ | ------------- | ---------- |
+| T8.1 | Implement detection signals | 2 days |
+| T8.2 | Build confidence scoring | 1 day |
+| T8.3 | Add Frappe/ERPNext detection | 2 days |
+| T8.4 | Add Next.js/React detection | 2 days |
+| T8.5 | Add Django/Laravel detection | 1 day |
+
+**Deliverables:**
+
+- `packages/framework-detector/`
+- Multi-framework support
+- Signal-based detection
+- Confidence scoring
+
+#### Week 9-10: RFC-0039 Test Data Generator & RFC-0040 Plugin SDK
+
+##### RFC-0039: Test Data Generator
+
+| Task | Description | Estimate |
+| ------ | ------------- | ---------- |
+| T9.1 | Design data generation DSL | 1 day |
+| T9.2 | Implement faker adapters | 2 days |
+| T9.3 | Framework-specific generators | 2 days |
+| T9.4 | Integration with E2E engine | 1 day |
+
+##### RFC-0040: Framework Plugin SDK
+
+| Task | Description | Estimate |
+| ------ | ------------- | ---------- |
+| T10.1 | Design plugin interface | 1 day |
+| T10.2 | Implement plugin loader | 2 days |
+| T10.3 | Create plugin registry | 1 day |
+| T10.4 | Add sandboxing/security | 2 days |
+| T10.5 | Write plugin examples | 1 day |
+
+---
+
+## Resource Allocation
+
+| Phase | Team Size | Focus |
+| ------- | ----------- | ------- |
+| Phase 1 | 1-2 | Foundation packages |
+| Phase 2 | 2 | Core systems + integration |
+| Phase 3 | 2-3 | Advanced features |
+
+## Risk Mitigation
+
+| Risk | Impact | Mitigation |
+| ------ | -------- | ------------ |
+| Framework detection accuracy | Medium | Iterative improvement with signal weighting |
+| Plugin security | High | Sandboxing + permission model |
+| Performance overhead | Low | Benchmarks in each PR |
+| Integration complexity | Medium | Weekly integration tests |
+
+## Success Metrics
+
+| Metric | Target |
+| -------- | -------- |
+| Checkpoint storage reduction | >60% |
+| Recovery time | <100ms |
+| Session restore success rate | >99% |
+| Framework detection accuracy | >90% |
+| Code review coverage | Configurable rules |
+| Plugin load time | <500ms |
+
+## Files to Create
+
+```
+packages/
++-- provider-adapter-sdk/           # RFC-0031
+|   +-- src/
+|   +-- test/
+|   +-- package.json
++-- observability/                   # RFC-0034
+|   +-- src/
+|   +-- test/
+|   +-- package.json
++-- checkpoint/                      # RFC-0032 (enhanced)
+|   +-- src/
+|   +-- package.json
++-- session/                         # RFC-0033
+|   +-- src/
+|   +-- test/
+|   +-- package.json
++-- code-review/                     # RFC-0036
+|   +-- src/
+|   +-- test/
+|   +-- package.json
++-- code-generation/                 # RFC-0037
+|   +-- src/
+|   +-- test/
+|   +-- package.json
++-- framework-detector/              # RFC-0038
+|   +-- src/
+|   +-- test/
+|   +-- package.json
++-- test-data-generator/             # RFC-0039
+|   +-- src/
+|   +-- test/
+|   +-- package.json
++-- framework-plugin-sdk/            # RFC-0040
+    +-- src/
+    +-- test/
+    +-- package.json
+```
+
+## Implementation Checklist
+
+- [ ] Phase 1 Complete
+  - [ ] RFC-0031 Provider Adapter SDK
+  - [ ] RFC-0034 Observability
+- [ ] Phase 2 Complete
+  - [ ] RFC-0032 Checkpoint Engine
+  - [ ] RFC-0033 Session Manager
+  - [ ] Integration tests passing
+- [ ] Phase 3 Complete
+  - [ ] RFC-0036 Code Review Engine
+  - [ ] RFC-0037 Code Generation Pipeline
+  - [ ] RFC-0038 Framework Detector
+  - [ ] RFC-0039 Test Data Generator
+  - [ ] RFC-0040 Framework Plugin SDK
+- [ ] Documentation complete
+- [ ] Performance benchmarks meet targets