patram 0.2.0 → 0.3.0

package/lib/discover-fields.js

@@ -0,0 +1,427 @@
+/* eslint-disable max-lines, max-lines-per-function */
+/**
+ * @import { ClaimOrigin, PatramClaim } from './parse-claims.types.ts';
+ * @import {
+ *   DiscoveredFieldMultiplicity,
+ *   DiscoveredFieldTypeName,
+ *   FieldDiscoveryClassUsage,
+ *   FieldDiscoveryEvidenceReference,
+ *   FieldDiscoveryMultiplicitySuggestion,
+ *   FieldDiscoveryResult,
+ *   FieldDiscoverySuggestion,
+ *   FieldDiscoveryTypeSuggestion,
+ * } from './discover-fields.types.ts';
+ */
+
+import { readFile } from 'node:fs/promises';
+import process from 'node:process';
+import { resolve } from 'node:path';
+
+import { DEFAULT_INCLUDE_PATTERNS } from './source-file-defaults.js';
+import { listSourceFiles } from './list-source-files.js';
+import { parseSourceFile } from './parse-claims.js';
+
+/**
+ * Field discovery from source claims.
+ *
+ * Scans the repository source files directly, infers likely metadata fields,
+ * and reports advisory suggestions without requiring repo config to load.
+ *
+ * Kind: discovery
+ * Status: active
+ * Tracked in: ../docs/plans/v1/field-model-redesign.md
+ * Decided by: ../docs/decisions/field-discovery-workflow.md
+ * @patram
+ * @see {@link ./render-field-discovery.js}
+ */
+
+const TYPE_NAME_ORDER = /** @type {const} */ ([
+  'integer',
+  'date_time',
+  'date',
+  'glob',
+  'path',
+  'enum',
+  'string',
+]);
+
+const INTEGER_PATTERN = /^-?\d+$/du;
+const DATE_PATTERN = /^\d{4}-\d{2}-\d{2}$/du;
+const DATE_TIME_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}/du;
+const ENUM_PATTERN = /^[a-z0-9_][a-z0-9_-]*$/du;
+const PATH_PATTERN = /^[a-z0-9_.-]+\.[a-z0-9]+$/du;
+
+/**
+ * @typedef {FieldDiscoveryClassUsage & { confidence: number }} InferredFieldClassUsage
+ */
+
+/**
+ * @typedef {(value: string) => number} FieldTypeScorer
+ */
+
+/**
+ * Discover likely field schema from source files.
+ *
+ * @param {string} [project_directory]
+ * @returns {Promise<FieldDiscoveryResult>}
+ */
+export async function discoverFields(project_directory = process.cwd()) {
+  const source_file_paths = await listSourceFiles(
+    DEFAULT_INCLUDE_PATTERNS,
+    project_directory,
+  );
+  const parse_results = await Promise.all(
+    source_file_paths.map(async (source_file_path) => {
+      const source_text = await readFile(
+        resolve(project_directory, source_file_path),
+        'utf8',
+      );
+
+      return {
+        claims: parseSourceFile({
+          path: source_file_path,
+          source: source_text,
+        }).claims,
+        path: source_file_path,
+      };
+    }),
+  );
+  /** @type {FieldObservation[]} */
+  const field_observations = parse_results.flatMap((parse_result) => {
+    /** @type {Set<string>} */
+    const document_classes = new Set();
+
+    for (const claim of parse_result.claims) {
+      if (
+        claim.type === 'directive' &&
+        claim.name === 'kind' &&
+        typeof claim.value === 'string' &&
+        claim.value.length > 0
+      ) {
+        document_classes.add(claim.value);
+      }
+    }
+
+    return parse_result.claims.flatMap((claim) => {
+      if (
+        claim.type !== 'directive' ||
+        !claim.name ||
+        claim.name.startsWith('$') ||
+        typeof claim.value !== 'string' ||
+        claim.value.length === 0
+      ) {
+        return [];
+      }
+
+      return [
+        {
+          class_names: new Set(document_classes),
+          document_id: claim.document_id,
+          name: claim.name,
+          origin: claim.origin,
+          value: claim.value,
+        },
+      ];
+    });
+  });
+  /** @type {Map<string, FieldBucket>} */
+  const field_buckets = field_observations.reduce(
+    (buckets, field_observation) => {
+      const bucket = buckets.get(field_observation.name) ?? {
+        name: field_observation.name,
+        observations: [],
+      };
+
+      bucket.observations.push(field_observation);
+      buckets.set(field_observation.name, bucket);
+      return buckets;
+    },
+    new Map(),
+  );
+  const fields = [...field_buckets.values()]
+    .map(buildFieldSuggestion)
+    .sort((left_suggestion, right_suggestion) =>
+      left_suggestion.confidence !== right_suggestion.confidence
+        ? right_suggestion.confidence - left_suggestion.confidence
+        : left_suggestion.name.localeCompare(right_suggestion.name, 'en'),
+    );
+
+  return {
+    fields,
+    summary: {
+      claim_count: parse_results.reduce(
+        (sum, parse_result) => sum + parse_result.claims.length,
+        0,
+      ),
+      count: fields.length,
+      source_file_count: source_file_paths.length,
+    },
+  };
+}
+
+/**
+ * @param {FieldBucket} field_bucket
+ * @returns {FieldDiscoverySuggestion}
+ */
+function buildFieldSuggestion(field_bucket) {
+  const type_result = inferFieldType(field_bucket.observations);
+  const multiplicity_result = inferFieldMultiplicity(field_bucket.observations);
+  const class_usage_result = inferFieldClassUsage(field_bucket.observations);
+  const evidence_references = buildEvidenceReferences(
+    field_bucket.observations,
+  );
+  const conflicting_evidence = buildEvidenceReferences(
+    field_bucket.observations.filter(
+      (field_observation) =>
+        scoreFieldValue(field_observation.value, type_result.name) === 0,
+    ),
+  );
+
+  return {
+    confidence:
+      Math.round(
+        ((type_result.confidence +
+          multiplicity_result.confidence +
+          class_usage_result.confidence) /
+          3) *
+          100,
+      ) / 100,
+    conflicting_evidence,
+    evidence_references,
+    likely_class_usage: {
+      classes: class_usage_result.classes,
+    },
+    likely_multiplicity: multiplicity_result,
+    likely_type: type_result,
+    name: field_bucket.name,
+  };
+}
+
+/**
+ * @param {FieldObservation[]} observations
+ * @returns {FieldDiscoveryEvidenceReference[]}
+ */
+function buildEvidenceReferences(observations) {
+  return observations
+    .map((observation) => ({
+      column: observation.origin.column,
+      line: observation.origin.line,
+      path: observation.origin.path,
+      value: observation.value,
+    }))
+    .sort(compareEvidenceReferences);
+}
+
+/**
+ * @param {FieldObservation[]} observations
+ * @returns {FieldDiscoveryMultiplicitySuggestion}
+ */
+function inferFieldMultiplicity(observations) {
+  /** @type {Map<string, Set<string>>} */
+  const values_by_document = observations.reduce((values, observation) => {
+    const current_values = values.get(observation.document_id);
+
+    if (current_values) {
+      current_values.add(observation.value);
+    } else {
+      values.set(observation.document_id, new Set([observation.value]));
+    }
+
+    return values;
+  }, new Map());
+  const repeated_identical_documents = [...values_by_document.values()].reduce(
+    (count, values) => {
+      if (values.size > 1) {
+        return Infinity;
+      }
+
+      return values.size === 1 ? count + 1 : count;
+    },
+    0,
+  );
+
+  if (repeated_identical_documents === Infinity) {
+    return {
+      confidence: 1,
+      name: 'multiple',
+    };
+  }
+
+  return {
+    confidence:
+      Math.round(
+        (values_by_document.size > 1 && repeated_identical_documents > 0
+          ? 0.9
+          : 0.8) * 100,
+      ) / 100,
+    name: 'single',
+  };
+}
+
+/**
+ * @param {FieldObservation[]} observations
+ * @returns {InferredFieldClassUsage}
+ */
+function inferFieldClassUsage(observations) {
+  /** @type {Map<string, number>} */
+  const class_counts = new Map();
+  let documented_observation_count = 0;
+
+  for (const observation of observations) {
+    if (observation.class_names.size === 0) {
+      continue;
+    }
+
+    documented_observation_count += 1;
+
+    for (const class_name of observation.class_names) {
+      class_counts.set(class_name, (class_counts.get(class_name) ?? 0) + 1);
+    }
+  }
+
+  if (class_counts.size === 0) {
+    return {
+      confidence: 0.2,
+      classes: ['document'],
+    };
+  }
+
+  return {
+    confidence:
+      Math.round(
+        (documented_observation_count / Math.max(observations.length, 1)) * 100,
+      ) / 100,
+    classes: [...class_counts.keys()].sort((left_class, right_class) =>
+      left_class.localeCompare(right_class, 'en'),
+    ),
+  };
+}
+
+/**
+ * @param {FieldObservation[]} observations
+ * @returns {FieldDiscoveryTypeSuggestion}
+ */
+function inferFieldType(observations) {
+  /** @type {FieldDiscoveryTypeSuggestion[]} */
+  const type_candidates = TYPE_NAME_ORDER.map((type_name) => ({
+    confidence: scoreFieldType(observations, type_name),
+    name: type_name,
+  }));
+
+  type_candidates.sort((left_candidate, right_candidate) => {
+    if (left_candidate.confidence !== right_candidate.confidence) {
+      return right_candidate.confidence - left_candidate.confidence;
+    }
+
+    return (
+      TYPE_NAME_ORDER.indexOf(left_candidate.name) -
+      TYPE_NAME_ORDER.indexOf(right_candidate.name)
+    );
+  });
+
+  return type_candidates[0];
+}
+
+/**
+ * @param {FieldDiscoveryEvidenceReference} left_reference
+ * @param {FieldDiscoveryEvidenceReference} right_reference
+ * @returns {number}
+ */
+function compareEvidenceReferences(left_reference, right_reference) {
+  const path_compare = left_reference.path.localeCompare(
+    right_reference.path,
+    'en',
+  );
+
+  if (path_compare !== 0) {
+    return path_compare;
+  }
+
+  if (left_reference.line !== right_reference.line) {
+    return left_reference.line - right_reference.line;
+  }
+
+  if (left_reference.column !== right_reference.column) {
+    return left_reference.column - right_reference.column;
+  }
+
+  return left_reference.value.localeCompare(right_reference.value, 'en');
+}
+
+/**
+ * @param {FieldObservation[]} observations
+ * @param {DiscoveredFieldTypeName} field_type_name
+ * @returns {number}
+ */
+function scoreFieldType(observations, field_type_name) {
+  if (observations.length === 0) {
+    return 0;
+  }
+
+  const total_score = observations.reduce(
+    (sum, observation) =>
+      sum + scoreFieldValue(observation.value, field_type_name),
+    0,
+  );
+
+  return Math.round((total_score / observations.length) * 100) / 100;
+}
+
+/**
+ * @param {string} value
+ * @param {DiscoveredFieldTypeName} field_type_name
+ * @returns {number}
+ */
+function scoreFieldValue(value, field_type_name) {
+  const scorer = FIELD_TYPE_SCORERS[field_type_name];
+  return scorer ? scorer(value) : 0;
+}
+
+/** @type {Record<DiscoveredFieldTypeName, FieldTypeScorer>} */
+const FIELD_TYPE_SCORERS = {
+  date: (value) => (DATE_PATTERN.test(value) ? 1 : 0),
+  date_time: (value) => (DATE_TIME_PATTERN.test(value) ? 1 : 0),
+  enum: (value) =>
+    ENUM_PATTERN.test(value) && value.includes(' ') === false ? 1 : 0,
+  glob: (value) =>
+    value.includes('*') ||
+    value.includes('?') ||
+    value.includes('[') ||
+    value.includes(']')
+      ? 1
+      : 0,
+  integer: (value) => (INTEGER_PATTERN.test(value) ? 1 : 0),
+  path: (value) =>
+    !(
+      value.includes('/') ||
+      PATH_PATTERN.test(value) ||
+      value.startsWith('docs/') ||
+      value.startsWith('lib/') ||
+      value.startsWith('test/')
+    )
+      ? 0
+      : value.includes('*') ||
+          value.includes('?') ||
+          value.includes('[') ||
+          value.includes(']')
+        ? 0.8
+        : 1,
+  string: () => 0.5,
+};
+
+/**
+ * @typedef {{
+ *   class_names: Set<string>,
+ *   document_id: string,
+ *   name: string,
+ *   origin: ClaimOrigin,
+ *   value: string,
+ * }} FieldObservation
+ */
+
+/**
+ * @typedef {{
+ *   name: string,
+ *   observations: FieldObservation[],
+ * }} FieldBucket
+ */

package/lib/discover-fields.types.ts

@@ -0,0 +1,52 @@
+export type DiscoveredFieldTypeName =
+  | 'date'
+  | 'date_time'
+  | 'enum'
+  | 'glob'
+  | 'integer'
+  | 'path'
+  | 'string';
+
+export type DiscoveredFieldMultiplicity = 'multiple' | 'single';
+
+export interface FieldDiscoveryEvidenceReference {
+  column: number;
+  line: number;
+  path: string;
+  value: string;
+}
+
+export interface FieldDiscoveryClassUsage {
+  classes: string[];
+}
+
+export interface FieldDiscoveryTypeSuggestion {
+  confidence: number;
+  name: DiscoveredFieldTypeName;
+}
+
+export interface FieldDiscoveryMultiplicitySuggestion {
+  confidence: number;
+  name: DiscoveredFieldMultiplicity;
+}
+
+export interface FieldDiscoverySuggestion {
+  confidence: number;
+  conflicting_evidence: FieldDiscoveryEvidenceReference[];
+  evidence_references: FieldDiscoveryEvidenceReference[];
+  likely_class_usage: FieldDiscoveryClassUsage;
+  likely_multiplicity: FieldDiscoveryMultiplicitySuggestion;
+  likely_type: FieldDiscoveryTypeSuggestion;
+  name: string;
+}
+
+export interface FieldDiscoverySummary {
+  claim_count: number;
+  count: number;
+  source_file_count: number;
+}
+
+export interface FieldDiscoveryResult {
+  fields: FieldDiscoverySuggestion[];
+  summary: FieldDiscoverySummary;
+}

package/lib/format-node-header.js

@@ -3,17 +3,19 @@
  * @returns {string}
  */
 export function formatNodeHeader(output_item) {
-  if (isDocumentNode(output_item)) {
-    return `document ${output_item.path}`;
+  if (output_item.path) {
+    return `${output_item.node_kind} ${output_item.path}`;
   }
 
-  return `${output_item.node_kind} ${output_item.id}`;
+  return `${output_item.node_kind} ${getOutputNodeKey(output_item.id)}`;
 }
 
 /**
- * @param {import('./output-view.types.ts').OutputNodeItem} output_item
- * @returns {boolean}
+ * @param {string} node_id
+ * @returns {string}
  */
-export function isDocumentNode(output_item) {
-  return output_item.id === `doc:${output_item.path}`;
+function getOutputNodeKey(node_id) {
+  return node_id.includes(':')
+    ? node_id.split(':').slice(1).join(':')
+    : node_id;
 }

package/lib/format-output-metadata.js

@@ -1,5 +1,4 @@
 import { formatDerivedSummaryRow } from './format-derived-summary-row.js';
-import { isDocumentNode } from './format-node-header.js';
 
 /**
  * @param {import('./output-view.types.ts').OutputNodeItem} output_item
@@ -8,18 +7,8 @@ import { isDocumentNode } from './format-node-header.js';
 export function formatOutputNodeMetadataRows(output_item) {
   /** @type {string[]} */
   const metadata_rows = [];
-  /** @type {string[]} */
-  const stored_metadata_fields = [];
-
-  if (isDocumentNode(output_item)) {
-    stored_metadata_fields.push(`kind: ${output_item.node_kind}`);
-  } else {
-    stored_metadata_fields.push(`path: ${output_item.path}`);
-  }
-
-  if (output_item.status) {
-    stored_metadata_fields.push(`status: ${output_item.status}`);
-  }
+  const stored_metadata_fields =
+    output_item.visible_fields.map(formatMetadataField);
 
   if (stored_metadata_fields.length > 0) {
     metadata_rows.push(stored_metadata_fields.join('  '));
@@ -39,16 +28,7 @@ export function formatOutputNodeMetadataRows(output_item) {
 export function formatResolvedLinkMetadataRows(target) {
   /** @type {string[]} */
   const metadata_rows = [];
-  /** @type {string[]} */
-  const stored_metadata_fields = [];
-
-  if (target.kind) {
-    stored_metadata_fields.push(`kind: ${target.kind}`);
-  }
-
-  if (target.status) {
-    stored_metadata_fields.push(`status: ${target.status}`);
-  }
+  const stored_metadata_fields = target.visible_fields.map(formatMetadataField);
 
   if (stored_metadata_fields.length > 0) {
     metadata_rows.push(stored_metadata_fields.join('  '));
@@ -60,3 +40,15 @@ export function formatResolvedLinkMetadataRows(target) {
 
   return metadata_rows;
 }
+
+/**
+ * @param {import('./output-view.types.ts').OutputMetadataField} output_field
+ * @returns {string}
+ */
+function formatMetadataField(output_field) {
+  const value = Array.isArray(output_field.value)
+    ? output_field.value.join(', ')
+    : output_field.value;
+
+  return `${output_field.name}: ${value}`;
+}

package/lib/layout-stored-queries.js

@@ -1,4 +1,3 @@
-/* eslint-disable max-lines */
 /**
  * @import { OutputStoredQueryItem } from './output-view.types.ts';
  */
@@ -98,22 +97,7 @@ function createStoredQueryPhrases(where_clause) {
 }
 
 /**
- * @param {{
- *   is_negated: boolean,
- *   term:
- *     | { field_name: 'id' | 'kind' | 'path' | 'status' | 'title', kind: 'field', operator: '=' | '^=' | '~', value: string }
- *     | { field_name: 'id' | 'kind' | 'path' | 'status' | 'title', kind: 'field_set', operator: 'in' | 'not in', values: string[] }
- *     | { kind: 'relation', relation_name: string }
- *     | { kind: 'relation_target', relation_name: string, target_id: string }
- *     | {
- *         aggregate_name: 'any' | 'count' | 'none',
- *         clauses: unknown[],
- *         comparison?: '!=' | '<' | '<=' | '=' | '>' | '>=',
- *         kind: 'aggregate',
- *         traversal: { direction: 'in' | 'out', relation_name: string },
- *         value?: number,
- *       },
- * }} clause
+ * @param {import('./parse-where-clause.types.ts').ParsedClause} clause
  * @param {boolean} should_prefix_and
  * @returns {StoredQuerySegment[]}
  */
@@ -137,20 +121,7 @@ function createClausePhrase(clause, should_prefix_and) {
 }
 
 /**
- * @param {
- *   | { field_name: 'id' | 'kind' | 'path' | 'status' | 'title', kind: 'field', operator: '=' | '^=' | '~', value: string }
- *   | { field_name: 'id' | 'kind' | 'path' | 'status' | 'title', kind: 'field_set', operator: 'in' | 'not in', values: string[] }
- *   | { kind: 'relation', relation_name: string }
- *   | { kind: 'relation_target', relation_name: string, target_id: string }
- *   | {
- *       aggregate_name: 'any' | 'count' | 'none',
- *       clauses: { is_negated: boolean, term: unknown }[],
- *       comparison?: '!=' | '<' | '<=' | '=' | '>' | '>=',
- *       kind: 'aggregate',
- *       traversal: { direction: 'in' | 'out', relation_name: string },
- *       value?: number,
- *     }
- * } term
+ * @param {import('./parse-where-clause.types.ts').ParsedTerm} term
  * @returns {StoredQuerySegment[]}
  */
 function createTermSegments(term) {
@@ -185,7 +156,7 @@ function createTermSegments(term) {
 }
 
 /**
- * @param {{ field_name: 'id' | 'kind' | 'path' | 'status' | 'title', kind: 'field_set', operator: 'in' | 'not in', values: string[] }} term
+ * @param {import('./parse-where-clause.types.ts').ParsedFieldSetTerm} term
  * @returns {StoredQuerySegment[]}
  */
 function createFieldSetSegments(term) {
@@ -201,14 +172,7 @@ function createFieldSetSegments(term) {
 }
 
 /**
- * @param {{
- *   aggregate_name: 'any' | 'count' | 'none',
- *   clauses: { is_negated: boolean, term: unknown }[],
- *   comparison?: '!=' | '<' | '<=' | '=' | '>' | '>=',
- *   kind: 'aggregate',
- *   traversal: { direction: 'in' | 'out', relation_name: string },
- *   value?: number,
- * }} term
+ * @param {import('./parse-where-clause.types.ts').ParsedAggregateTerm} term
  * @returns {StoredQuerySegment[]}
  */
 function createAggregateSegments(term) {
@@ -245,30 +209,12 @@ function createTraversalSegments(traversal) {
 }
 
 /**
- * @param {{ is_negated: boolean, term: unknown }[]} clauses
+ * @param {import('./parse-where-clause.types.ts').ParsedClause[]} clauses
  * @returns {StoredQuerySegment[]}
  */
 function createNestedClauseSegments(clauses) {
   return clauses.flatMap((clause, clause_index) => {
-    const clause_phrase = createClausePhrase(
-      /** @type {{
-       *   is_negated: boolean,
-       *   term:
-       *     | { field_name: 'id' | 'kind' | 'path' | 'status' | 'title', kind: 'field', operator: '=' | '^=' | '~', value: string }
-       *     | { field_name: 'id' | 'kind' | 'path' | 'status' | 'title', kind: 'field_set', operator: 'in' | 'not in', values: string[] }
-       *     | { kind: 'relation', relation_name: string }
-       *     | { kind: 'relation_target', relation_name: string, target_id: string }
-       *     | {
-       *         aggregate_name: 'any' | 'count' | 'none',
-       *         clauses: { is_negated: boolean, term: unknown }[],
-       *         comparison?: '!=' | '<' | '<=' | '=' | '>' | '>=',
-       *         kind: 'aggregate',
-       *         traversal: { direction: 'in' | 'out', relation_name: string },
-       *         value?: number,
-       *       },
-       * }} */ (clause),
-      clause_index > 0,
-    );
+    const clause_phrase = createClausePhrase(clause, clause_index > 0);
 
     if (clause_index === 0) {
       return clause_phrase;