patram 0.9.0 → 0.11.0

package/lib/scan/discover-fields.js

@@ -20,6 +20,11 @@ import { resolve } from 'node:path';
 import { DEFAULT_INCLUDE_PATTERNS } from '../config/source-file-defaults.js';
 import { listSourceFiles } from './list-source-files.js';
 import { parseSourceFile } from '../parse/parse-claims.js';
+import {
+  matchHiddenDirectiveFields,
+  matchVisibleDirectiveFields,
+} from '../parse/markdown/parse-markdown-directives.js';
+import { isPathLikeTarget } from '../parse/claim-helpers.js';
 
 /**
  * Field discovery from source claims.
@@ -71,10 +76,9 @@ export async function discoverFields(
   options,
 ) {
   const defined_field_names = options?.defined_field_names ?? new Set();
-  const source_file_paths = await listSourceFiles(
-    DEFAULT_INCLUDE_PATTERNS,
-    project_directory,
-  );
+  const source_file_paths = (
+    await listSourceFiles(DEFAULT_INCLUDE_PATTERNS, project_directory)
+  ).filter((source_file_path) => source_file_path.includes('/'));
   const parse_results = await Promise.all(
     source_file_paths.map(async (source_file_path) => {
       const source_text = await readFile(
@@ -88,6 +92,7 @@ export async function discoverFields(
           source: source_text,
         }).claims,
         path: source_file_path,
+        source_text,
       };
     }),
   );
@@ -107,13 +112,20 @@ export async function discoverFields(
       }
     }
 
+    const allowed_markdown_lines = collectAllowedMarkdownDirectiveLines(
+      parse_result.path,
+      parse_result.source_text,
+      parse_result.claims,
+    );
+
     return parse_result.claims.flatMap((claim) => {
       if (
         claim.type !== 'directive' ||
         !claim.name ||
         claim.name.startsWith('$') ||
         typeof claim.value !== 'string' ||
-        claim.value.length === 0
+        claim.value.length === 0 ||
+        !shouldIncludeDiscoveryClaim(claim, allowed_markdown_lines)
       ) {
         return [];
       }
@@ -123,6 +135,7 @@ export async function discoverFields(
           class_names: new Set(document_classes),
           document_id: claim.document_id,
           name: claim.name,
+          normalized_value: normalizeDiscoveryValue(claim.value),
           origin: claim.origin,
           value: claim.value,
         },
@@ -147,7 +160,9 @@ export async function discoverFields(
   const fields = [...field_buckets.values()]
     .map(buildFieldSuggestion)
     .filter(
-      (field_suggestion) => !defined_field_names.has(field_suggestion.name),
+      (field_suggestion) =>
+        !defined_field_names.has(field_suggestion.name) &&
+        isPlausibleFieldName(field_suggestion.name),
     )
     .sort((left_suggestion, right_suggestion) =>
       left_suggestion.confidence !== right_suggestion.confidence
@@ -182,7 +197,10 @@ function buildFieldSuggestion(field_bucket) {
   const conflicting_evidence = buildEvidenceReferences(
     field_bucket.observations.filter(
       (field_observation) =>
-        scoreFieldValue(field_observation.value, type_result.name) === 0,
+        scoreFieldValue(
+          field_observation.normalized_value,
+          type_result.name,
+        ) === 0,
     ),
   );
 
@@ -228,12 +246,13 @@ function buildEvidenceReferences(observations) {
 function inferFieldMultiplicity(observations) {
   /** @type {Map<string, Set<string>>} */
   const values_by_document = observations.reduce((values, observation) => {
+    const normalized_value = observation.normalized_value;
     const current_values = values.get(observation.document_id);
 
     if (current_values) {
-      current_values.add(observation.value);
+      current_values.add(normalized_value);
     } else {
-      values.set(observation.document_id, new Set([observation.value]));
+      values.set(observation.document_id, new Set([normalized_value]));
     }
 
     return values;
@@ -369,7 +388,7 @@ function scoreFieldType(observations, field_type_name) {
 
   const total_score = observations.reduce(
     (sum, observation) =>
-      sum + scoreFieldValue(observation.value, field_type_name),
+      sum + scoreFieldValue(observation.normalized_value, field_type_name),
     0,
   );
 
@@ -423,6 +442,7 @@ const FIELD_TYPE_SCORERS = {
  *   class_names: Set<string>,
  *   document_id: string,
  *   name: string,
+ *   normalized_value: string,
  *   origin: ClaimOrigin,
  *   value: string,
  * }} FieldObservation
@@ -434,3 +454,109 @@ const FIELD_TYPE_SCORERS = {
  *   observations: FieldObservation[],
  * }} FieldBucket
  */
+
+/**
+ * @param {PatramClaim} claim
+ * @param {Set<number> | null} allowed_markdown_lines
+ * @returns {boolean}
+ */
+function shouldIncludeDiscoveryClaim(claim, allowed_markdown_lines) {
+  if (claim.parser !== 'markdown') {
+    return true;
+  }
+
+  if (claim.markdown_style === 'front_matter') {
+    return true;
+  }
+
+  return allowed_markdown_lines?.has(claim.origin.line) ?? false;
+}
+
+/**
+ * @param {string} file_path
+ * @param {string} source_text
+ * @param {PatramClaim[]} claims
+ * @returns {Set<number> | null}
+ */
+function collectAllowedMarkdownDirectiveLines(file_path, source_text, claims) {
+  if (!file_path.endsWith('.md')) {
+    return null;
+  }
+
+  const title_claim = claims.find((claim) => claim.type === 'document.title');
+
+  if (!title_claim) {
+    return new Set();
+  }
+
+  const lines = source_text.split('\n');
+  /** @type {Set<number>} */
+  const allowed_lines = new Set();
+
+  for (
+    let line_index = title_claim.origin.line;
+    line_index < lines.length;
+    line_index += 1
+  ) {
+    const line = lines[line_index];
+
+    if (line.trim().length === 0) {
+      continue;
+    }
+
+    if (isMarkdownDiscoveryDirective(file_path, line, line_index + 1)) {
+      allowed_lines.add(line_index + 1);
+      continue;
+    }
+
+    break;
+  }
+
+  return allowed_lines;
+}
+
+/**
+ * @param {string} file_path
+ * @param {string} line
+ * @param {number} line_number
+ * @returns {boolean}
+ */
+function isMarkdownDiscoveryDirective(file_path, line, line_number) {
+  return (
+    matchVisibleDirectiveFields(file_path, line, line_number) !== null ||
+    matchHiddenDirectiveFields(file_path, line, line_number) !== null
+  );
+}
+
+/**
+ * @param {string} value
+ * @returns {string}
+ */
+function normalizeDiscoveryValue(value) {
+  const trimmed_value = value.trim();
+  const markdown_link_match = trimmed_value.match(
+    /^\[([^\]]+)\]\(([^)]+)\)$/du,
+  );
+
+  if (markdown_link_match && isPathLikeTarget(markdown_link_match[2])) {
+    return markdown_link_match[2];
+  }
+
+  const code_span_match = trimmed_value.match(/^`([^`]+)`[.,;:]?$/du);
+
+  if (code_span_match) {
+    return code_span_match[1];
+  }
+
+  return trimmed_value;
+}
+
+/**
+ * @param {string} field_name
+ * @returns {boolean}
+ */
+function isPlausibleFieldName(field_name) {
+  const field_name_tokens = field_name.split('_');
+
+  return field_name.length <= 32 && field_name_tokens.length <= 4;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "patram",
-  "version": "0.9.0",
+  "version": "0.11.0",
   "type": "module",
   "main": "./lib/patram.js",
   "types": "./lib/patram.d.ts",
@@ -67,6 +67,7 @@
     "ansis": "^4.2.0",
     "beautiful-mermaid": "^1.1.3",
     "globby": "^16.1.1",
+    "jsonc-parser": "^3.3.1",
     "md4x": "^0.0.25",
     "shiki": "^4.0.2",
     "string-width": "^8.2.0",