te.js 1.3.1 → 2.0.0

package/auto-docs/openapi/endpoint-processor.js

@@ -0,0 +1,277 @@
+/**
+ * Per-endpoint processing for OpenAPI spec: extract info, resolve deps, LLM enhance, build path ops and tag descriptions.
+ */
+
+import { analyzeHandler } from '../analysis/handler-analyzer.js';
+import {
+  resolveDependencySources,
+  formatDependencyContext,
+  resolveTargetFilePath,
+} from '../analysis/source-resolver.js';
+import {
+  HANDLER_SOURCE_MAX_LENGTH_BY_LEVEL,
+  DEPENDENCY_CONTEXT_MAX_CHARS,
+  METHOD_AGNOSTIC_OPERATION_KEY,
+} from '../constants.js';
+import { stripLlmUsage } from '../utils/strip-usage.js';
+import {
+  isMethodKeyed,
+  toOpenAPIPath,
+  getPathParameters,
+  isMethodAgnostic,
+  mergeMetadata,
+  mergeMethodAgnosticMeta,
+  buildOperation,
+} from './spec-builders.js';
+
+/**
+ * Extract path, handler, metadata, methods, groupId, tag from a registry target.
+ * @param {object} target - Endpoint-like with getPath(), getHandler(), getMetadata(), getGroup?()
+ * @returns {{ path: string, handler: function|null, explicitMeta: object, methods: string[], groupId: string|null, tag: string }}
+ */
+export function extractTargetInfo(target) {
+  const path = target.getPath();
+  const handler = target.getHandler();
+  const explicitMeta = target.getMetadata() || {};
+  const analyzed = analyzeHandler(handler);
+  const methods = Array.isArray(explicitMeta.methods) && explicitMeta.methods.length > 0
+    ? explicitMeta.methods
+    : analyzed.methods;
+  const groupId = target.getGroup?.() ?? null;
+  const tag = groupId != null && groupId !== '' ? groupId : 'default';
+  return { path, handler, explicitMeta, methods, groupId, tag };
+}
+
+/**
+ * Slice handler source to max length for the given level.
+ * @param {function|null} handler
+ * @param {number} effectiveLevel
+ * @returns {string}
+ */
+export function resolveHandlerSource(handler, effectiveLevel) {
+  const raw = typeof handler === 'function' ? handler.toString() : '';
+  const max = HANDLER_SOURCE_MAX_LENGTH_BY_LEVEL[effectiveLevel] ?? 2800;
+  return raw.slice(0, max);
+}
+
+/**
+ * Resolve and cache dependency context for a group. Mutates cache.
+ * @param {string|null} groupId
+ * @param {string} tag
+ * @param {number} effectiveLevel
+ * @param {string} dirTargets
+ * @param {Map<string,string>} cache - dependencyContextByGroup
+ * @returns {Promise<string>}
+ */
+export async function resolveDependencyContext(groupId, tag, effectiveLevel, dirTargets, cache) {
+  if (effectiveLevel !== 2 || !groupId) return '';
+  if (cache.has(tag)) return cache.get(tag) || '';
+  try {
+    const sources = await resolveDependencySources(groupId, dirTargets);
+    const targetPath = resolveTargetFilePath(groupId, dirTargets);
+    const context = formatDependencyContext(sources, targetPath, DEPENDENCY_CONTEXT_MAX_CHARS);
+    cache.set(tag, context);
+    return context;
+  } catch {
+    cache.set(tag, '');
+    return '';
+  }
+}
+
+/**
+ * Call LLM to enhance endpoint metadata. Returns { meta, metaByMethod }.
+ * @param {object} endpointInfo - { path, methods, metadata, handlerSource, dependencySources? }
+ * @param {object} llm
+ * @param {object} explicitMeta
+ * @param {boolean} preferEnhanced
+ * @param {string[]} methods
+ * @param {string} path
+ * @param {function} log
+ * @returns {Promise<{ meta: object, metaByMethod: Map<string,object>|null }>}
+ */
+export async function enhanceWithLlm(endpointInfo, llm, explicitMeta, preferEnhanced, methods, path, log) {
+  let meta = {
+    summary: explicitMeta.summary || path || 'Endpoint',
+    description: explicitMeta.description,
+    request: explicitMeta.request,
+    response: explicitMeta.response,
+  };
+  let metaByMethod = null;
+  try {
+    if (typeof llm.enhanceEndpointDocsPerMethod === 'function') {
+      const rawPerMethod = await llm.enhanceEndpointDocsPerMethod(endpointInfo);
+      const cleaned = stripLlmUsage(rawPerMethod);
+      if (cleaned && isMethodKeyed(cleaned)) {
+        metaByMethod = new Map();
+        for (const m of methods) {
+          const k = m.toLowerCase();
+          metaByMethod.set(k, mergeMetadata(explicitMeta, cleaned[k] || {}, { preferEnhanced }));
+        }
+        meta = mergeMetadata(explicitMeta, {}, { preferEnhanced });
+      } else {
+        meta = mergeMetadata(explicitMeta, cleaned || {}, { preferEnhanced });
+      }
+      const tokenStr = rawPerMethod?._usage?.total_tokens != null ? ` — ${rawPerMethod._usage.total_tokens} tokens` : '';
+      log(`  ${path} [${methods.join(', ').toUpperCase()}]${tokenStr}`);
+    } else {
+      const enhanced = await llm.enhanceEndpointDocs(endpointInfo);
+      meta = mergeMetadata(explicitMeta, stripLlmUsage(enhanced) || enhanced, { preferEnhanced });
+      const tokenStr = enhanced?._usage?.total_tokens != null ? ` — ${enhanced._usage.total_tokens} tokens` : '';
+      log(`  ${path} [${methods.join(', ').toUpperCase()}]${tokenStr}`);
+    }
+  } catch (err) {
+    log(`  ${path} [${(methods || []).join(', ').toUpperCase()}] — LLM failed`);
+  }
+  return { meta, metaByMethod };
+}
+
+/**
+ * Process one registry target: analyze, optionally enhance with LLM, build meta and path params.
+ * @param {object} target - Endpoint-like with getPath(), getHandler(), getMetadata(), getGroup?()
+ * @param {object} options - { llm?, effectiveLevel, dirTargets, dependencyContextByGroup, useLlm, preferEnhanced, log }
+ * @returns {Promise<{ openAPIPath: string, tag: string, methodAgnostic: boolean, meta: object, metaByMethod: Map|null, methods: string[], pathParams: array, groupEntry: object }>}
+ */
+export async function processEndpoint(target, options) {
+  const { llm, effectiveLevel, dirTargets, dependencyContextByGroup, useLlm, preferEnhanced, log } = options;
+
+  const { path, handler, explicitMeta, methods, groupId, tag } = extractTargetInfo(target);
+  let meta = {
+    summary: explicitMeta.summary || path || 'Endpoint',
+    description: explicitMeta.description,
+    request: explicitMeta.request,
+    response: explicitMeta.response,
+  };
+  const handlerSource = resolveHandlerSource(handler, effectiveLevel);
+  let metaByMethod = null;
+
+  if (useLlm) {
+    const dependencySources = await resolveDependencyContext(groupId, tag, effectiveLevel, dirTargets, dependencyContextByGroup);
+    const endpointInfo = {
+      path,
+      methods,
+      metadata: explicitMeta,
+      handlerSource,
+      ...(dependencySources && { dependencySources }),
+    };
+    const enhanced = await enhanceWithLlm(endpointInfo, llm, explicitMeta, preferEnhanced, methods, path, log);
+    meta = enhanced.meta;
+    metaByMethod = enhanced.metaByMethod;
+  }
+
+  const openAPIPath = toOpenAPIPath(path);
+  const pathParams = getPathParameters(path);
+  const handlerIsMethodAgnostic = isMethodAgnostic(methods);
+  if (handlerIsMethodAgnostic && metaByMethod != null) {
+    meta = mergeMethodAgnosticMeta(metaByMethod, methods, meta);
+    metaByMethod = null;
+  }
+  const methodAgnostic = metaByMethod == null && handlerIsMethodAgnostic;
+
+  const groupEntry = {
+    path,
+    methods,
+    summary: meta.summary,
+    description: meta.description,
+    handlerSource,
+    ...(dependencyContextByGroup.has(tag) && { dependencySources: dependencyContextByGroup.get(tag) }),
+  };
+
+  return {
+    openAPIPath,
+    tag,
+    methodAgnostic,
+    meta,
+    metaByMethod,
+    methods,
+    pathParams,
+    groupEntry,
+  };
+}
+
+/**
+ * Add one endpoint's operations to the paths object (mutates paths).
+ * @param {object} paths - OpenAPI paths object
+ * @param {object} result - From processEndpoint
+ */
+export function addEndpointToPaths(paths, result) {
+  const { openAPIPath, tag, methodAgnostic, meta, metaByMethod, methods, pathParams } = result;
+  if (!paths[openAPIPath]) paths[openAPIPath] = {};
+  if (methodAgnostic) {
+    const op = buildOperation(METHOD_AGNOSTIC_OPERATION_KEY, meta, pathParams, { methodAgnostic: true });
+    op.tags = [tag];
+    paths[openAPIPath][METHOD_AGNOSTIC_OPERATION_KEY] = op;
+  } else {
+    for (const method of methods) {
+      const key = method.toLowerCase();
+      const methodMeta = metaByMethod?.get(key) ?? meta;
+      const op = buildOperation(method, methodMeta, pathParams);
+      op.tags = [tag];
+      paths[openAPIPath][key] = op;
+    }
+  }
+}
+
+/**
+ * Build tag name and description for each group (LLM or fallback).
+ * @param {Map<string,object[]>} groupEndpoints
+ * @param {Map<string,string>} dependencyContextByGroup
+ * @param {object|null} llm
+ * @param {{ effectiveLevel: number, log: function }} options
+ * @returns {Promise<Map<string,{ name: string, description: string }>>}
+ */
+export async function buildTagDescriptions(groupEndpoints, dependencyContextByGroup, llm, options) {
+  const { effectiveLevel, log } = options;
+  const tagDescriptions = new Map();
+  if (llm && typeof llm.summarizeTargetGroup === 'function') {
+    for (const [groupId, endpoints] of groupEndpoints) {
+      try {
+        const dependencySources = effectiveLevel === 2 ? dependencyContextByGroup.get(groupId) || '' : '';
+        const infos = endpoints.map((e) => ({
+          path: e.path,
+          methods: e.methods,
+          summary: e.summary,
+          description: e.description,
+          handlerSource: e.handlerSource,
+          ...(e.dependencySources && { dependencySources: e.dependencySources }),
+        }));
+        const result = await llm.summarizeTargetGroup(groupId, infos, dependencySources);
+        const { name, description, _usage: summaryUsage } = result;
+        tagDescriptions.set(groupId, { name: name || groupId, description });
+        const tokenStr = summaryUsage?.total_tokens != null && summaryUsage.total_tokens > 0
+          ? ` — ${summaryUsage.total_tokens} tokens`
+          : '';
+        log(`  [group ${groupId}] summary${tokenStr}`);
+      } catch (err) {
+        tagDescriptions.set(groupId, {
+          name: groupId.split('/').pop() || groupId,
+          description: '',
+        });
+        log(`  [group ${groupId}] summary — failed`);
+      }
+    }
+  } else {
+    for (const groupId of groupEndpoints.keys()) {
+      tagDescriptions.set(groupId, {
+        name: groupId.split('/').pop() || groupId,
+        description: '',
+      });
+    }
+  }
+  return tagDescriptions;
+}
+
+/**
+ * Replace operation tags (groupId) with display names from tagDescriptions. Mutates paths.
+ * @param {object} paths - OpenAPI paths object
+ * @param {Map<string,{ name: string, description?: string }>} tagDescriptions
+ */
+export function applyTagDisplayNames(paths, tagDescriptions) {
+  for (const pathItem of Object.values(paths)) {
+    for (const op of Object.values(pathItem)) {
+      if (op?.tags?.[0]) {
+        const groupId = op.tags[0];
+        op.tags[0] = tagDescriptions.get(groupId)?.name ?? groupId;
+      }
+    }
+  }
+}

package/auto-docs/openapi/generator.js

@@ -0,0 +1,107 @@
+/**
+ * OpenAPI 3.0 spec generator for te.js auto-documentation.
+ * Orchestrates spec-builders and endpoint-processor; builds final spec object.
+ */
+
+import { OPENAPI_VERSION } from '../constants.js';
+import { createVerboseLogger } from '../utils/logger.js';
+import {
+  processEndpoint,
+  addEndpointToPaths,
+  buildTagDescriptions,
+  applyTagDisplayNames,
+} from './endpoint-processor.js';
+import {
+  toOpenAPIPath,
+  getPathParameters,
+  getQueryParameters,
+  buildSchemaFromMetadata,
+  buildRequestBody,
+  buildResponses,
+  buildOperation,
+  mergeMetadata,
+} from './spec-builders.js';
+
+/**
+ * Build OpenAPI 3.0 spec from registry and options.
+ * @param {object} registry - Target registry with .targets
+ * @param {object} [options] - { llm?, info?, servers?, level?, dirTargets?, verbose?, logger? }
+ * @returns {Promise<object>} OpenAPI 3.0 spec
+ */
+async function generateOpenAPISpec(registry, options = {}) {
+  const {
+    llm,
+    info = {},
+    servers,
+    level = 1,
+    dirTargets = process.env.DIR_TARGETS || 'targets',
+    verbose = false,
+    logger = null,
+  } = options;
+  const targets = registry?.targets ?? [];
+  const paths = {};
+  const groupEndpoints = new Map();
+  const dependencyContextByGroup = new Map();
+
+  const useLlm = !!llm && typeof llm.enhanceEndpointDocs === 'function';
+  const effectiveLevel = level === 3 ? 2 : Math.max(1, Math.min(2, level));
+  const log = createVerboseLogger(logger, verbose);
+  const preferEnhanced = effectiveLevel === 2;
+
+  for (const target of targets) {
+    const result = await processEndpoint(target, {
+      llm,
+      effectiveLevel,
+      dirTargets,
+      dependencyContextByGroup,
+      useLlm,
+      preferEnhanced,
+      log,
+    });
+    if (!groupEndpoints.has(result.tag)) groupEndpoints.set(result.tag, []);
+    groupEndpoints.get(result.tag).push(result.groupEntry);
+    addEndpointToPaths(paths, result);
+  }
+
+  const tagDescriptions = await buildTagDescriptions(
+    groupEndpoints,
+    dependencyContextByGroup,
+    useLlm ? llm : null,
+    { effectiveLevel, log },
+  );
+  applyTagDisplayNames(paths, tagDescriptions);
+
+  const tags = Array.from(tagDescriptions.entries()).map(([, { name, description }]) => ({
+    name,
+    ...(description && { description }),
+  }));
+
+  const spec = {
+    openapi: OPENAPI_VERSION,
+    info: {
+      title: info.title ?? 'API',
+      version: info.version ?? '1.0.0',
+      ...(info.description && { description: info.description }),
+    },
+    tags: tags.length > 0 ? tags : undefined,
+    paths,
+  };
+  if (Array.isArray(servers) && servers.length > 0) {
+    spec.servers = servers;
+  }
+  return spec;
+}
+
+export {
+  OPENAPI_VERSION,
+  toOpenAPIPath,
+  getPathParameters,
+  getQueryParameters,
+  buildSchemaFromMetadata,
+  buildRequestBody,
+  buildResponses,
+  buildOperation,
+  mergeMetadata,
+  generateOpenAPISpec,
+};
+export default generateOpenAPISpec;

package/auto-docs/openapi/level3.js

@@ -0,0 +1,131 @@
+/**
+ * Level 3 pipeline: reorder OpenAPI tag groups by importance and generate overview page.
+ * Single entry point is runLevel3(spec, options, llm). Runs after the spec is generated (level 2 quality).
+ */
+
+import path from 'node:path';
+import { writeFile } from 'node:fs/promises';
+import { createVerboseLogger } from '../utils/logger.js';
+
+/**
+ * Resolve overview file path from option or derive from outputPath.
+ * @param {string} [overviewPathOption] - Explicit path for API_OVERVIEW.md
+ * @param {string} [outputPath] - OpenAPI spec output path (dir used for default overview)
+ * @returns {string} Overview path or ''
+ */
+export function resolveOverviewPath(overviewPathOption, outputPath) {
+  if (overviewPathOption && typeof overviewPathOption === 'string') {
+    return overviewPathOption;
+  }
+  if (outputPath && typeof outputPath === 'string') {
+    return path.join(path.dirname(outputPath), 'API_OVERVIEW.md');
+  }
+  return '';
+}
+
+/**
+ * Generate overview markdown from the spec (no I/O). Pure except LLM call.
+ * @param {object} spec - OpenAPI 3 spec (after reorder)
+ * @param {object} options - { info?: { title?, version?, description? } }
+ * @param {object} llm - LLM provider with generateOverviewPage(spec, options)
+ * @returns {Promise<{ markdown: string }>}
+ */
+export async function generateOverview(spec, options, llm) {
+  if (typeof llm?.generateOverviewPage !== 'function') {
+    return { markdown: '' };
+  }
+  const info = options?.info ?? {};
+  const { markdown } = await llm.generateOverviewPage(spec, {
+    title: info.title,
+    version: info.version,
+    description: info.description,
+  });
+  return { markdown: markdown || '' };
+}
+
+/**
+ * Write level-3 artifacts: overview file and optionally the spec file.
+ * @param {object} spec - OpenAPI 3 spec (may have info.description set)
+ * @param {string} overviewPath - Path for API_OVERVIEW.md
+ * @param {string} [outputPath] - Path for openapi.json
+ * @param {string} [markdown] - Overview markdown content
+ * @returns {Promise<void>}
+ */
+export async function writeLevel3Artifacts(spec, overviewPath, outputPath, markdown) {
+  if (overviewPath && markdown) {
+    await writeFile(overviewPath, markdown, 'utf8');
+  }
+  if (outputPath && typeof outputPath === 'string') {
+    await writeFile(outputPath, JSON.stringify(spec, null, 2), 'utf8');
+  }
+}
+
+/**
+ * Run the full level-3 pipeline: reorder tags, generate overview, write artifacts.
+ * @param {object} spec - OpenAPI 3 spec (mutated: tags reordered, info.description set)
+ * @param {object} options - { outputPath?, overviewPath?, info?, verbose?, logger? }
+ * @param {object} llm - LLM provider with reorderTagsByImportance and generateOverviewPage
+ * @returns {Promise<void>}
+ */
+export async function runLevel3(spec, options, llm) {
+  const { outputPath, overviewPath: overviewPathOption, info = {}, verbose = false, logger = null } = options;
+  const log = createVerboseLogger(logger, verbose);
+
+  if (!spec?.tags?.length || !llm) return;
+
+  log('Level 3: reordering tag groups by importance...');
+  await reorderSpecTags(spec, llm);
+
+  const overviewPath = resolveOverviewPath(overviewPathOption, outputPath);
+  if (!overviewPath) {
+    await writeLevel3Artifacts(spec, '', outputPath);
+    if (outputPath) log('Spec updated with reordered tags.');
+    return;
+  }
+
+  log('Level 3: generating overview...');
+  const { markdown } = await generateOverview(spec, { info }, llm);
+  if (markdown) {
+    spec.info = spec.info || {};
+    spec.info.description = markdown;
+    log('Overview embedded in spec (info.description) for Scalar.');
+  }
+  log('Overview file written to ' + overviewPath + '.');
+
+  await writeLevel3Artifacts(spec, overviewPath, outputPath, markdown);
+  if (outputPath) log('Spec written with reordered tags and overview.');
+}
+
+/**
+ * Reorder spec.tags by importance using the LLM. Mutates and returns the same spec object.
+ * @param {object} spec - OpenAPI 3 spec with tags array
+ * @param {object} llm - LLM provider with reorderTagsByImportance(spec)
+ * @returns {Promise<object>} The same spec with tags reordered
+ */
+export async function reorderSpecTags(spec, llm) {
+  if (!spec?.tags?.length || typeof llm?.reorderTagsByImportance !== 'function') {
+    return spec;
+  }
+  const result = await llm.reorderTagsByImportance(spec);
+  const ordered = result._orderedTags;
+  if (Array.isArray(ordered) && ordered.length > 0) {
+    spec.tags = ordered;
+  }
+  return spec;
+}
+
+/**
+ * Generate overview markdown and write it to overviewPath. (Convenience: generateOverview + write.)
+ * @param {object} spec - OpenAPI 3 spec (after reorder)
+ * @param {object} options - { overviewPath: string, info?: { title?, version?, description? } }
+ * @param {object} llm - LLM provider with generateOverviewPage(spec, options)
+ * @returns {Promise<{ markdown: string }>}
+ */
+export async function generateAndWriteOverview(spec, options, llm) {
+  const overviewPath = options?.overviewPath;
+  const { markdown } = await generateOverview(spec, options, llm);
+  if (overviewPath && markdown) {
+    await writeFile(overviewPath, markdown, 'utf8');
+  }
+  return { markdown };
+}