patram 0.2.0 → 0.4.0

package/lib/directive-type-rules.js

@@ -0,0 +1,133 @@
+/**
+ * @import { MetadataFieldConfig } from './load-patram-config.types.ts';
+ */
+
+import { isPathLikeTarget } from './claim-helpers.js';
+
+/**
+ * @param {MetadataFieldConfig} type_definition
+ * @param {string} directive_value
+ * @returns {boolean}
+ */
+export function isDirectiveValueValid(type_definition, directive_value) {
+  if (directive_value.length === 0) {
+    return false;
+  }
+
+  switch (type_definition.type) {
+    case 'string':
+      return true;
+    case 'integer':
+      return /^-?\d+$/du.test(directive_value);
+    case 'path':
+      return isPathLikeTarget(directive_value);
+    case 'glob':
+      return true;
+    case 'date':
+      return isValidDateValue(directive_value);
+    case 'date_time':
+      return isValidDateTimeValue(directive_value);
+    default:
+      throw new Error(`Unsupported directive type "${type_definition.type}".`);
+  }
+}
+
+/**
+ * @param {string} directive_name
+ * @param {Exclude<MetadataFieldConfig['type'], 'enum'>} type_name
+ * @returns {string}
+ */
+export function getInvalidTypeMessage(directive_name, type_name) {
+  switch (type_name) {
+    case 'string':
+      return `Directive "${directive_name}" must be a non-empty string.`;
+    case 'integer':
+      return `Directive "${directive_name}" must be a base-10 integer.`;
+    case 'path':
+      return `Directive "${directive_name}" must be a path-like string.`;
+    case 'glob':
+      return `Directive "${directive_name}" must be a non-empty glob string.`;
+    case 'date':
+      return `Directive "${directive_name}" must use YYYY-MM-DD.`;
+    case 'date_time':
+      return `Directive "${directive_name}" must use YYYY-MM-DD HH:MM.`;
+    default:
+      throw new Error(`Unsupported directive type "${type_name}".`);
+  }
+}
+
+/**
+ * @param {string[]} values
+ * @returns {string}
+ */
+export function formatQuotedList(values) {
+  return values.map((value) => `"${value}"`).join(', ');
+}
+
+/**
+ * @param {string} directive_value
+ * @returns {boolean}
+ */
+function isValidDateValue(directive_value) {
+  const date_match = /^(?<year>\d{4})-(?<month>\d{2})-(?<day>\d{2})$/du.exec(
+    directive_value,
+  );
+
+  if (!date_match?.groups) {
+    return false;
+  }
+
+  return isRealCalendarDate(
+    Number(date_match.groups.year),
+    Number(date_match.groups.month),
+    Number(date_match.groups.day),
+  );
+}
+
+/**
+ * @param {string} directive_value
+ * @returns {boolean}
+ */
+function isValidDateTimeValue(directive_value) {
+  const date_time_match =
+    /^(?<year>\d{4})-(?<month>\d{2})-(?<day>\d{2}) (?<hour>\d{2}):(?<minute>\d{2})$/du.exec(
+      directive_value,
+    );
+
+  if (!date_time_match?.groups) {
+    return false;
+  }
+
+  const hour = Number(date_time_match.groups.hour);
+  const minute = Number(date_time_match.groups.minute);
+
+  if (hour > 23 || minute > 59) {
+    return false;
+  }
+
+  return isRealCalendarDate(
+    Number(date_time_match.groups.year),
+    Number(date_time_match.groups.month),
+    Number(date_time_match.groups.day),
+  );
+}
+
+/**
+ * @param {number} year
+ * @param {number} month
+ * @param {number} day
+ * @returns {boolean}
+ */
+function isRealCalendarDate(year, month, day) {
+  if (month < 1 || month > 12 || day < 1 || day > 31) {
+    return false;
+  }
+
+  const candidate_date = new Date(Date.UTC(year, month - 1, day));
+
+  return (
+    candidate_date.getUTCFullYear() === year &&
+    candidate_date.getUTCMonth() === month - 1 &&
+    candidate_date.getUTCDate() === day
+  );
+}

package/lib/discover-fields.js

@@ -0,0 +1,435 @@
+/* eslint-disable max-lines, max-lines-per-function */
+/**
+ * @import { ClaimOrigin, PatramClaim } from './parse-claims.types.ts';
+ * @import {
+ *   DiscoveredFieldMultiplicity,
+ *   DiscoveredFieldTypeName,
+ *   FieldDiscoveryClassUsage,
+ *   FieldDiscoveryEvidenceReference,
+ *   FieldDiscoveryMultiplicitySuggestion,
+ *   FieldDiscoveryResult,
+ *   FieldDiscoverySuggestion,
+ *   FieldDiscoveryTypeSuggestion,
+ * } from './discover-fields.types.ts';
+ */
+
+import { readFile } from 'node:fs/promises';
+import process from 'node:process';
+import { resolve } from 'node:path';
+
+import { DEFAULT_INCLUDE_PATTERNS } from './source-file-defaults.js';
+import { listSourceFiles } from './list-source-files.js';
+import { parseSourceFile } from './parse-claims.js';
+
+/**
+ * Field discovery from source claims.
+ *
+ * Scans the repository source files directly, infers likely metadata fields,
+ * and reports advisory suggestions without requiring repo config to load.
+ *
+ * Kind: discovery
+ * Status: active
+ * Tracked in: ../docs/plans/v1/field-model-redesign.md
+ * Decided by: ../docs/decisions/field-discovery-workflow.md
+ * @patram
+ * @see {@link ./render-field-discovery.js}
+ */
+
+const TYPE_NAME_ORDER = /** @type {const} */ ([
+  'integer',
+  'date_time',
+  'date',
+  'glob',
+  'path',
+  'enum',
+  'string',
+]);
+
+const INTEGER_PATTERN = /^-?\d+$/du;
+const DATE_PATTERN = /^\d{4}-\d{2}-\d{2}$/du;
+const DATE_TIME_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}/du;
+const ENUM_PATTERN = /^[a-z0-9_][a-z0-9_-]*$/du;
+const PATH_PATTERN = /^[a-z0-9_.-]+\.[a-z0-9]+$/du;
+
+/**
+ * @typedef {FieldDiscoveryClassUsage & { confidence: number }} InferredFieldClassUsage
+ */
+
+/**
+ * @typedef {(value: string) => number} FieldTypeScorer
+ */
+
+/**
+ * Discover likely field schema from source files.
+ *
+ * @param {string} [project_directory]
+ * @param {{ defined_field_names?: ReadonlySet<string> }} [options]
+ * @returns {Promise<FieldDiscoveryResult>}
+ */
+export async function discoverFields(
+  project_directory = process.cwd(),
+  options,
+) {
+  const defined_field_names = options?.defined_field_names ?? new Set();
+  const source_file_paths = await listSourceFiles(
+    DEFAULT_INCLUDE_PATTERNS,
+    project_directory,
+  );
+  const parse_results = await Promise.all(
+    source_file_paths.map(async (source_file_path) => {
+      const source_text = await readFile(
+        resolve(project_directory, source_file_path),
+        'utf8',
+      );
+
+      return {
+        claims: parseSourceFile({
+          path: source_file_path,
+          source: source_text,
+        }).claims,
+        path: source_file_path,
+      };
+    }),
+  );
+  /** @type {FieldObservation[]} */
+  const field_observations = parse_results.flatMap((parse_result) => {
+    /** @type {Set<string>} */
+    const document_classes = new Set();
+
+    for (const claim of parse_result.claims) {
+      if (
+        claim.type === 'directive' &&
+        claim.name === 'kind' &&
+        typeof claim.value === 'string' &&
+        claim.value.length > 0
+      ) {
+        document_classes.add(claim.value);
+      }
+    }
+
+    return parse_result.claims.flatMap((claim) => {
+      if (
+        claim.type !== 'directive' ||
+        !claim.name ||
+        claim.name.startsWith('$') ||
+        typeof claim.value !== 'string' ||
+        claim.value.length === 0
+      ) {
+        return [];
+      }
+
+      return [
+        {
+          class_names: new Set(document_classes),
+          document_id: claim.document_id,
+          name: claim.name,
+          origin: claim.origin,
+          value: claim.value,
+        },
+      ];
+    });
+  });
+  /** @type {Map<string, FieldBucket>} */
+  const field_buckets = field_observations.reduce(
+    (buckets, field_observation) => {
+      const bucket = buckets.get(field_observation.name) ?? {
+        name: field_observation.name,
+        observations: [],
+      };
+
+      bucket.observations.push(field_observation);
+      buckets.set(field_observation.name, bucket);
+      return buckets;
+    },
+    new Map(),
+  );
+  const fields = [...field_buckets.values()]
+    .map(buildFieldSuggestion)
+    .filter(
+      (field_suggestion) => !defined_field_names.has(field_suggestion.name),
+    )
+    .sort((left_suggestion, right_suggestion) =>
+      left_suggestion.confidence !== right_suggestion.confidence
+        ? right_suggestion.confidence - left_suggestion.confidence
+        : left_suggestion.name.localeCompare(right_suggestion.name, 'en'),
+    );
+
+  return {
+    fields,
+    summary: {
+      claim_count: parse_results.reduce(
+        (sum, parse_result) => sum + parse_result.claims.length,
+        0,
+      ),
+      count: fields.length,
+      source_file_count: source_file_paths.length,
+    },
+  };
+}
+
+/**
+ * @param {FieldBucket} field_bucket
+ * @returns {FieldDiscoverySuggestion}
+ */
+function buildFieldSuggestion(field_bucket) {
+  const type_result = inferFieldType(field_bucket.observations);
+  const multiplicity_result = inferFieldMultiplicity(field_bucket.observations);
+  const class_usage_result = inferFieldClassUsage(field_bucket.observations);
+  const evidence_references = buildEvidenceReferences(
+    field_bucket.observations,
+  );
+  const conflicting_evidence = buildEvidenceReferences(
+    field_bucket.observations.filter(
+      (field_observation) =>
+        scoreFieldValue(field_observation.value, type_result.name) === 0,
+    ),
+  );
+
+  return {
+    confidence:
+      Math.round(
+        ((type_result.confidence +
+          multiplicity_result.confidence +
+          class_usage_result.confidence) /
+          3) *
+          100,
+      ) / 100,
+    conflicting_evidence,
+    evidence_references,
+    likely_class_usage: {
+      classes: class_usage_result.classes,
+    },
+    likely_multiplicity: multiplicity_result,
+    likely_type: type_result,
+    name: field_bucket.name,
+  };
+}
+
+/**
+ * @param {FieldObservation[]} observations
+ * @returns {FieldDiscoveryEvidenceReference[]}
+ */
+function buildEvidenceReferences(observations) {
+  return observations
+    .map((observation) => ({
+      column: observation.origin.column,
+      line: observation.origin.line,
+      path: observation.origin.path,
+      value: observation.value,
+    }))
+    .sort(compareEvidenceReferences);
+}
+
+/**
+ * @param {FieldObservation[]} observations
+ * @returns {FieldDiscoveryMultiplicitySuggestion}
+ */
+function inferFieldMultiplicity(observations) {
+  /** @type {Map<string, Set<string>>} */
+  const values_by_document = observations.reduce((values, observation) => {
+    const current_values = values.get(observation.document_id);
+
+    if (current_values) {
+      current_values.add(observation.value);
+    } else {
+      values.set(observation.document_id, new Set([observation.value]));
+    }
+
+    return values;
+  }, new Map());
+  const repeated_identical_documents = [...values_by_document.values()].reduce(
+    (count, values) => {
+      if (values.size > 1) {
+        return Infinity;
+      }
+
+      return values.size === 1 ? count + 1 : count;
+    },
+    0,
+  );
+
+  if (repeated_identical_documents === Infinity) {
+    return {
+      confidence: 1,
+      name: 'multiple',
+    };
+  }
+
+  return {
+    confidence:
+      Math.round(
+        (values_by_document.size > 1 && repeated_identical_documents > 0
+          ? 0.9
+          : 0.8) * 100,
+      ) / 100,
+    name: 'single',
+  };
+}
+
+/**
+ * @param {FieldObservation[]} observations
+ * @returns {InferredFieldClassUsage}
+ */
+function inferFieldClassUsage(observations) {
+  /** @type {Map<string, number>} */
+  const class_counts = new Map();
+  let documented_observation_count = 0;
+
+  for (const observation of observations) {
+    if (observation.class_names.size === 0) {
+      continue;
+    }
+
+    documented_observation_count += 1;
+
+    for (const class_name of observation.class_names) {
+      class_counts.set(class_name, (class_counts.get(class_name) ?? 0) + 1);
+    }
+  }
+
+  if (class_counts.size === 0) {
+    return {
+      confidence: 0.2,
+      classes: ['document'],
+    };
+  }
+
+  return {
+    confidence:
+      Math.round(
+        (documented_observation_count / Math.max(observations.length, 1)) * 100,
+      ) / 100,
+    classes: [...class_counts.keys()].sort((left_class, right_class) =>
+      left_class.localeCompare(right_class, 'en'),
+    ),
+  };
+}
+
+/**
+ * @param {FieldObservation[]} observations
+ * @returns {FieldDiscoveryTypeSuggestion}
+ */
+function inferFieldType(observations) {
+  /** @type {FieldDiscoveryTypeSuggestion[]} */
+  const type_candidates = TYPE_NAME_ORDER.map((type_name) => ({
+    confidence: scoreFieldType(observations, type_name),
+    name: type_name,
+  }));
+
+  type_candidates.sort((left_candidate, right_candidate) => {
+    if (left_candidate.confidence !== right_candidate.confidence) {
+      return right_candidate.confidence - left_candidate.confidence;
+    }
+
+    return (
+      TYPE_NAME_ORDER.indexOf(left_candidate.name) -
+      TYPE_NAME_ORDER.indexOf(right_candidate.name)
+    );
+  });
+
+  return type_candidates[0];
+}
+
+/**
+ * @param {FieldDiscoveryEvidenceReference} left_reference
+ * @param {FieldDiscoveryEvidenceReference} right_reference
+ * @returns {number}
+ */
+function compareEvidenceReferences(left_reference, right_reference) {
+  const path_compare = left_reference.path.localeCompare(
+    right_reference.path,
+    'en',
+  );
+
+  if (path_compare !== 0) {
+    return path_compare;
+  }
+
+  if (left_reference.line !== right_reference.line) {
+    return left_reference.line - right_reference.line;
+  }
+
+  if (left_reference.column !== right_reference.column) {
+    return left_reference.column - right_reference.column;
+  }
+
+  return left_reference.value.localeCompare(right_reference.value, 'en');
+}
+
+/**
+ * @param {FieldObservation[]} observations
+ * @param {DiscoveredFieldTypeName} field_type_name
+ * @returns {number}
+ */
+function scoreFieldType(observations, field_type_name) {
+  if (observations.length === 0) {
+    return 0;
+  }
+
+  const total_score = observations.reduce(
+    (sum, observation) =>
+      sum + scoreFieldValue(observation.value, field_type_name),
+    0,
+  );
+
+  return Math.round((total_score / observations.length) * 100) / 100;
+}
+
+/**
+ * @param {string} value
+ * @param {DiscoveredFieldTypeName} field_type_name
+ * @returns {number}
+ */
+function scoreFieldValue(value, field_type_name) {
+  const scorer = FIELD_TYPE_SCORERS[field_type_name];
+  return scorer ? scorer(value) : 0;
+}
+
+/** @type {Record<DiscoveredFieldTypeName, FieldTypeScorer>} */
+const FIELD_TYPE_SCORERS = {
+  date: (value) => (DATE_PATTERN.test(value) ? 1 : 0),
+  date_time: (value) => (DATE_TIME_PATTERN.test(value) ? 1 : 0),
+  enum: (value) =>
+    ENUM_PATTERN.test(value) && value.includes(' ') === false ? 1 : 0,
+  glob: (value) =>
+    value.includes('*') ||
+    value.includes('?') ||
+    value.includes('[') ||
+    value.includes(']')
+      ? 1
+      : 0,
+  integer: (value) => (INTEGER_PATTERN.test(value) ? 1 : 0),
+  path: (value) =>
+    !(
+      value.includes('/') ||
+      PATH_PATTERN.test(value) ||
+      value.startsWith('docs/') ||
+      value.startsWith('lib/') ||
+      value.startsWith('test/')
+    )
+      ? 0
+      : value.includes('*') ||
+          value.includes('?') ||
+          value.includes('[') ||
+          value.includes(']')
+        ? 0.8
+        : 1,
+  string: () => 0.5,
+};
+
+/**
+ * @typedef {{
+ *   class_names: Set<string>,
+ *   document_id: string,
+ *   name: string,
+ *   origin: ClaimOrigin,
+ *   value: string,
+ * }} FieldObservation
+ */
+
+/**
+ * @typedef {{
+ *   name: string,
+ *   observations: FieldObservation[],
+ * }} FieldBucket
+ */

package/lib/discover-fields.types.ts

@@ -0,0 +1,52 @@
+export type DiscoveredFieldTypeName =
+  | 'date'
+  | 'date_time'
+  | 'enum'
+  | 'glob'
+  | 'integer'
+  | 'path'
+  | 'string';
+
+export type DiscoveredFieldMultiplicity = 'multiple' | 'single';
+
+export interface FieldDiscoveryEvidenceReference {
+  column: number;
+  line: number;
+  path: string;
+  value: string;
+}
+
+export interface FieldDiscoveryClassUsage {
+  classes: string[];
+}
+
+export interface FieldDiscoveryTypeSuggestion {
+  confidence: number;
+  name: DiscoveredFieldTypeName;
+}
+
+export interface FieldDiscoveryMultiplicitySuggestion {
+  confidence: number;
+  name: DiscoveredFieldMultiplicity;
+}
+
+export interface FieldDiscoverySuggestion {
+  confidence: number;
+  conflicting_evidence: FieldDiscoveryEvidenceReference[];
+  evidence_references: FieldDiscoveryEvidenceReference[];
+  likely_class_usage: FieldDiscoveryClassUsage;
+  likely_multiplicity: FieldDiscoveryMultiplicitySuggestion;
+  likely_type: FieldDiscoveryTypeSuggestion;
+  name: string;
+}
+
+export interface FieldDiscoverySummary {
+  claim_count: number;
+  count: number;
+  source_file_count: number;
+}
+
+export interface FieldDiscoveryResult {
+  fields: FieldDiscoverySuggestion[];
+  summary: FieldDiscoverySummary;
+}