patram 0.8.0 → 0.10.0

package/lib/patram.d.ts

@@ -3,52 +3,53 @@ export {
   loadTaggedFencedBlocks,
   selectTaggedBlock,
   selectTaggedBlocks,
-} from './tagged-fenced-blocks.js';
+} from './parse/tagged-fenced/tagged-fenced-blocks.js';
 
-export { parseWhereClause } from './parse-where-clause.js';
-export { getQuerySemanticDiagnostics } from './query-inspection.js';
-export { loadProjectGraph } from './load-project-graph.js';
-export { overlayGraph } from './overlay-graph.js';
-export { queryGraph } from './query-graph.js';
+export { parseWhereClause } from './graph/query/parse.js';
+export { getQuerySemanticDiagnostics } from './graph/query/inspect.js';
+export { loadProjectGraph } from './graph/load-project-graph.js';
+export { queryGraph } from './graph/query/execute.js';
 
-export type PatramGraphNode = import('./build-graph.types.ts').GraphNode;
-export type PatramGraphEdge = import('./build-graph.types.ts').GraphEdge;
+export type PatramGraphNode =
+  import('./graph/build-graph.types.d.ts').GraphNode;
+export type PatramGraphEdge =
+  import('./graph/build-graph.types.d.ts').GraphEdge;
 export type PatramBuildGraphResult =
-  import('./build-graph.types.ts').BuildGraphResult;
+  import('./graph/build-graph.types.d.ts').BuildGraphResult;
 export type PatramDiagnostic =
-  import('./load-patram-config.types.ts').PatramDiagnostic;
+  import('./config/load-patram-config.types.d.ts').PatramDiagnostic;
 export type PatramRepoConfig =
-  import('./load-patram-config.types.ts').PatramRepoConfig;
+  import('./config/load-patram-config.types.d.ts').PatramRepoConfig;
 export type PatramParsedFieldName =
-  import('./parse-where-clause.types.ts').ParsedFieldName;
+  import('./graph/parse-where-clause.types.d.ts').ParsedFieldName;
 export type PatramParsedFieldTerm =
-  import('./parse-where-clause.types.ts').ParsedFieldTerm;
+  import('./graph/parse-where-clause.types.d.ts').ParsedFieldTerm;
 export type PatramParsedFieldSetTerm =
-  import('./parse-where-clause.types.ts').ParsedFieldSetTerm;
+  import('./graph/parse-where-clause.types.d.ts').ParsedFieldSetTerm;
 export type PatramParsedTraversalTerm =
-  import('./parse-where-clause.types.ts').ParsedTraversalTerm;
+  import('./graph/parse-where-clause.types.d.ts').ParsedTraversalTerm;
 export type PatramParsedRelationTerm =
-  import('./parse-where-clause.types.ts').ParsedRelationTerm;
+  import('./graph/parse-where-clause.types.d.ts').ParsedRelationTerm;
 export type PatramParsedRelationTargetTerm =
-  import('./parse-where-clause.types.ts').ParsedRelationTargetTerm;
+  import('./graph/parse-where-clause.types.d.ts').ParsedRelationTargetTerm;
 export type PatramParsedAggregateComparison =
-  import('./parse-where-clause.types.ts').ParsedAggregateComparison;
+  import('./graph/parse-where-clause.types.d.ts').ParsedAggregateComparison;
 export type PatramParsedAggregateName =
-  import('./parse-where-clause.types.ts').ParsedAggregateName;
+  import('./graph/parse-where-clause.types.d.ts').ParsedAggregateName;
 export type PatramParsedAggregateTerm =
-  import('./parse-where-clause.types.ts').ParsedAggregateTerm;
+  import('./graph/parse-where-clause.types.d.ts').ParsedAggregateTerm;
 export type PatramParsedTermExpression =
-  import('./parse-where-clause.types.ts').ParsedTermExpression;
+  import('./graph/parse-where-clause.types.d.ts').ParsedTermExpression;
 export type PatramParsedNotExpression =
-  import('./parse-where-clause.types.ts').ParsedNotExpression;
+  import('./graph/parse-where-clause.types.d.ts').ParsedNotExpression;
 export type PatramParsedBooleanExpression =
-  import('./parse-where-clause.types.ts').ParsedBooleanExpression;
+  import('./graph/parse-where-clause.types.d.ts').ParsedBooleanExpression;
 export type PatramParsedTerm =
-  import('./parse-where-clause.types.ts').ParsedTerm;
+  import('./graph/parse-where-clause.types.d.ts').ParsedTerm;
 export type PatramParsedExpression =
-  import('./parse-where-clause.types.ts').ParsedExpression;
+  import('./graph/parse-where-clause.types.d.ts').ParsedExpression;
 export type PatramParseWhereClauseResult =
-  import('./parse-where-clause.types.ts').ParseWhereClauseResult;
+  import('./graph/parse-where-clause.types.d.ts').ParseWhereClauseResult;
 export type PatramQuerySource =
   | {
       kind: 'ad_hoc';
@@ -59,8 +60,8 @@ export type PatramQuerySource =
     };
 
 export interface PatramProjectGraphResult {
-  claims: import('./parse-claims.types.ts').PatramClaim[];
-  config: import('./load-patram-config.types.ts').PatramRepoConfig;
+  claims: import('./parse/parse-claims.types.d.ts').PatramClaim[];
+  config: import('./config/load-patram-config.types.d.ts').PatramRepoConfig;
   diagnostics: PatramDiagnostic[];
   graph: PatramBuildGraphResult;
   source_file_paths: string[];

package/lib/patram.js

@@ -3,10 +3,9 @@ export {
   loadTaggedFencedBlocks,
   selectTaggedBlock,
   selectTaggedBlocks,
-} from './tagged-fenced-blocks.js';
+} from './parse/tagged-fenced/tagged-fenced-blocks.js';
 
-export { parseWhereClause } from './parse-where-clause.js';
-export { getQuerySemanticDiagnostics } from './query-inspection.js';
-export { loadProjectGraph } from './load-project-graph.js';
-export { overlayGraph } from './overlay-graph.js';
-export { queryGraph } from './query-graph.js';
+export { parseWhereClause } from './graph/query/parse.js';
+export { getQuerySemanticDiagnostics } from './graph/query/inspect.js';
+export { loadProjectGraph } from './graph/load-project-graph.js';
+export { queryGraph } from './graph/query/execute.js';

package/lib/{discover-fields.js → scan/discover-fields.js}

@@ -1,6 +1,6 @@
 /* eslint-disable max-lines, max-lines-per-function */
 /**
- * @import { ClaimOrigin, PatramClaim } from './parse-claims.types.ts';
+ * @import { ClaimOrigin, PatramClaim } from '../parse/parse-claims.types.ts';
  * @import {
  *   DiscoveredFieldMultiplicity,
  *   DiscoveredFieldTypeName,
@@ -17,9 +17,14 @@ import { readFile } from 'node:fs/promises';
 import process from 'node:process';
 import { resolve } from 'node:path';
 
-import { DEFAULT_INCLUDE_PATTERNS } from './source-file-defaults.js';
+import { DEFAULT_INCLUDE_PATTERNS } from '../config/source-file-defaults.js';
 import { listSourceFiles } from './list-source-files.js';
-import { parseSourceFile } from './parse-claims.js';
+import { parseSourceFile } from '../parse/parse-claims.js';
+import {
+  matchHiddenDirectiveFields,
+  matchVisibleDirectiveFields,
+} from '../parse/markdown/parse-markdown-directives.js';
+import { isPathLikeTarget } from '../parse/claim-helpers.js';
 
 /**
  * Field discovery from source claims.
@@ -29,10 +34,10 @@ import { parseSourceFile } from './parse-claims.js';
  *
  * Kind: discovery
  * Status: active
- * Tracked in: ../docs/plans/v1/field-model-redesign.md
- * Decided by: ../docs/decisions/field-discovery-workflow.md
+ * Tracked in: ../../docs/plans/v1/field-model-redesign.md
+ * Decided by: ../../docs/decisions/field-discovery-workflow.md
  * @patram
- * @see {@link ./render-field-discovery.js}
+ * @see {@link ../output/render-field-discovery.js}
  */
 
 const TYPE_NAME_ORDER = /** @type {const} */ ([
@@ -71,10 +76,9 @@ export async function discoverFields(
   options,
 ) {
   const defined_field_names = options?.defined_field_names ?? new Set();
-  const source_file_paths = await listSourceFiles(
-    DEFAULT_INCLUDE_PATTERNS,
-    project_directory,
-  );
+  const source_file_paths = (
+    await listSourceFiles(DEFAULT_INCLUDE_PATTERNS, project_directory)
+  ).filter((source_file_path) => source_file_path.includes('/'));
   const parse_results = await Promise.all(
     source_file_paths.map(async (source_file_path) => {
       const source_text = await readFile(
@@ -88,6 +92,7 @@ export async function discoverFields(
           source: source_text,
         }).claims,
         path: source_file_path,
+        source_text,
       };
     }),
   );
@@ -107,13 +112,20 @@ export async function discoverFields(
       }
     }
 
+    const allowed_markdown_lines = collectAllowedMarkdownDirectiveLines(
+      parse_result.path,
+      parse_result.source_text,
+      parse_result.claims,
+    );
+
     return parse_result.claims.flatMap((claim) => {
       if (
         claim.type !== 'directive' ||
         !claim.name ||
         claim.name.startsWith('$') ||
         typeof claim.value !== 'string' ||
-        claim.value.length === 0
+        claim.value.length === 0 ||
+        !shouldIncludeDiscoveryClaim(claim, allowed_markdown_lines)
       ) {
         return [];
       }
@@ -123,6 +135,7 @@ export async function discoverFields(
           class_names: new Set(document_classes),
           document_id: claim.document_id,
           name: claim.name,
+          normalized_value: normalizeDiscoveryValue(claim.value),
           origin: claim.origin,
           value: claim.value,
         },
@@ -132,6 +145,7 @@ export async function discoverFields(
   /** @type {Map<string, FieldBucket>} */
   const field_buckets = field_observations.reduce(
     (buckets, field_observation) => {
+      /** @type {FieldBucket} */
       const bucket = buckets.get(field_observation.name) ?? {
         name: field_observation.name,
         observations: [],
@@ -141,12 +155,14 @@ export async function discoverFields(
       buckets.set(field_observation.name, bucket);
       return buckets;
     },
-    new Map(),
+    /** @type {Map<string, FieldBucket>} */ (new Map()),
   );
   const fields = [...field_buckets.values()]
     .map(buildFieldSuggestion)
     .filter(
-      (field_suggestion) => !defined_field_names.has(field_suggestion.name),
+      (field_suggestion) =>
+        !defined_field_names.has(field_suggestion.name) &&
+        isPlausibleFieldName(field_suggestion.name),
     )
     .sort((left_suggestion, right_suggestion) =>
       left_suggestion.confidence !== right_suggestion.confidence
@@ -181,7 +197,10 @@ function buildFieldSuggestion(field_bucket) {
   const conflicting_evidence = buildEvidenceReferences(
     field_bucket.observations.filter(
       (field_observation) =>
-        scoreFieldValue(field_observation.value, type_result.name) === 0,
+        scoreFieldValue(
+          field_observation.normalized_value,
+          type_result.name,
+        ) === 0,
     ),
   );
 
@@ -227,16 +246,17 @@ function buildEvidenceReferences(observations) {
 function inferFieldMultiplicity(observations) {
   /** @type {Map<string, Set<string>>} */
   const values_by_document = observations.reduce((values, observation) => {
+    const normalized_value = observation.normalized_value;
     const current_values = values.get(observation.document_id);
 
     if (current_values) {
-      current_values.add(observation.value);
+      current_values.add(normalized_value);
     } else {
-      values.set(observation.document_id, new Set([observation.value]));
+      values.set(observation.document_id, new Set([normalized_value]));
     }
 
     return values;
-  }, new Map());
+  }, /** @type {Map<string, Set<string>>} */ (new Map()));
   const repeated_identical_documents = [...values_by_document.values()].reduce(
     (count, values) => {
       if (values.size > 1) {
@@ -368,7 +388,7 @@ function scoreFieldType(observations, field_type_name) {
 
   const total_score = observations.reduce(
     (sum, observation) =>
-      sum + scoreFieldValue(observation.value, field_type_name),
+      sum + scoreFieldValue(observation.normalized_value, field_type_name),
     0,
   );
 
@@ -422,6 +442,7 @@ const FIELD_TYPE_SCORERS = {
  *   class_names: Set<string>,
  *   document_id: string,
  *   name: string,
+ *   normalized_value: string,
  *   origin: ClaimOrigin,
  *   value: string,
  * }} FieldObservation
@@ -433,3 +454,109 @@ const FIELD_TYPE_SCORERS = {
  *   observations: FieldObservation[],
  * }} FieldBucket
  */
+
+/**
+ * @param {PatramClaim} claim
+ * @param {Set<number> | null} allowed_markdown_lines
+ * @returns {boolean}
+ */
+function shouldIncludeDiscoveryClaim(claim, allowed_markdown_lines) {
+  if (claim.parser !== 'markdown') {
+    return true;
+  }
+
+  if (claim.markdown_style === 'front_matter') {
+    return true;
+  }
+
+  return allowed_markdown_lines?.has(claim.origin.line) ?? false;
+}
+
+/**
+ * @param {string} file_path
+ * @param {string} source_text
+ * @param {PatramClaim[]} claims
+ * @returns {Set<number> | null}
+ */
+function collectAllowedMarkdownDirectiveLines(file_path, source_text, claims) {
+  if (!file_path.endsWith('.md')) {
+    return null;
+  }
+
+  const title_claim = claims.find((claim) => claim.type === 'document.title');
+
+  if (!title_claim) {
+    return new Set();
+  }
+
+  const lines = source_text.split('\n');
+  /** @type {Set<number>} */
+  const allowed_lines = new Set();
+
+  for (
+    let line_index = title_claim.origin.line;
+    line_index < lines.length;
+    line_index += 1
+  ) {
+    const line = lines[line_index];
+
+    if (line.trim().length === 0) {
+      continue;
+    }
+
+    if (isMarkdownDiscoveryDirective(file_path, line, line_index + 1)) {
+      allowed_lines.add(line_index + 1);
+      continue;
+    }
+
+    break;
+  }
+
+  return allowed_lines;
+}
+
+/**
+ * @param {string} file_path
+ * @param {string} line
+ * @param {number} line_number
+ * @returns {boolean}
+ */
+function isMarkdownDiscoveryDirective(file_path, line, line_number) {
+  return (
+    matchVisibleDirectiveFields(file_path, line, line_number) !== null ||
+    matchHiddenDirectiveFields(file_path, line, line_number) !== null
+  );
+}
+
+/**
+ * @param {string} value
+ * @returns {string}
+ */
+function normalizeDiscoveryValue(value) {
+  const trimmed_value = value.trim();
+  const markdown_link_match = trimmed_value.match(
+    /^\[([^\]]+)\]\(([^)]+)\)$/du,
+  );
+
+  if (markdown_link_match && isPathLikeTarget(markdown_link_match[2])) {
+    return markdown_link_match[2];
+  }
+
+  const code_span_match = trimmed_value.match(/^`([^`]+)`[.,;:]?$/du);
+
+  if (code_span_match) {
+    return code_span_match[1];
+  }
+
+  return trimmed_value;
+}
+
+/**
+ * @param {string} field_name
+ * @returns {boolean}
+ */
+function isPlausibleFieldName(field_name) {
+  const field_name_tokens = field_name.split('_');
+
+  return field_name.length <= 32 && field_name_tokens.length <= 4;
+}

package/lib/scan/list-repo-files.d.ts

@@ -0,0 +1,16 @@
+/**
+ * List repo files available for broken-link validation.
+ *
+ * @param {string} [project_directory]
+ * @returns {Promise<string[]>}
+ */
+export function listRepoFiles(project_directory?: string): Promise<string[]>;
+/**
+ * @param {string[]} include_patterns
+ * @param {string} project_directory
+ * @param {{ dot?: boolean }} [options]
+ * @returns {Promise<string[]>}
+ */
+export function listMatchingFiles(include_patterns: string[], project_directory: string, options?: {
+    dot?: boolean;
+}): Promise<string[]>;

package/lib/{list-source-files.js → scan/list-repo-files.js}

@@ -1,39 +1,6 @@
-import { globby } from 'globby';
 import process from 'node:process';
 
-/**
- * Source file scanning.
- *
- * Expands include globs into stable repo-relative file lists for indexing and
- * broken-link validation.
- *
- * Kind: scan
- * Status: active
- * Tracked in: ../docs/plans/v0/source-anchor-dogfooding.md
- * Decided by: ../docs/decisions/source-scan.md
- * @patram
- * @see {@link ./load-project-graph.js}
- * @see {@link ../docs/decisions/source-scan.md}
- */
-
-/**
- * List source files matched by Patram include globs.
- *
- * @param {string[]} include_patterns
- * @param {string} [project_directory]
- * @returns {Promise<string[]>}
- */
-export async function listSourceFiles(
-  include_patterns,
-  project_directory = process.cwd(),
-) {
-  const source_file_paths = await listMatchingFiles(
-    include_patterns,
-    project_directory,
-  );
-
-  return [...new Set(source_file_paths)].sort(comparePaths);
-}
+import { globby } from 'globby';
 
 /**
  * List repo files available for broken-link validation.
@@ -55,7 +22,7 @@ export async function listRepoFiles(project_directory = process.cwd()) {
  * @param {{ dot?: boolean }} [options]
  * @returns {Promise<string[]>}
  */
-async function listMatchingFiles(
+export async function listMatchingFiles(
   include_patterns,
   project_directory,
   options = {},

package/lib/{list-source-files.d.ts → scan/list-source-files.d.ts}

@@ -6,11 +6,11 @@
  *
  * Kind: scan
  * Status: active
- * Tracked in: ../docs/plans/v0/source-anchor-dogfooding.md
- * Decided by: ../docs/decisions/source-scan.md
+ * Tracked in: ../../docs/plans/v0/source-anchor-dogfooding.md
+ * Decided by: ../../docs/decisions/source-scan.md
  * @patram
- * @see {@link ./load-project-graph.js}
- * @see {@link ../docs/decisions/source-scan.md}
+ * @see {@link ../graph/load-project-graph.js}
+ * @see {@link ../../docs/decisions/source-scan.md}
  */
 /**
  * List source files matched by Patram include globs.
@@ -20,10 +20,3 @@
  * @returns {Promise<string[]>}
  */
 export function listSourceFiles(include_patterns: string[], project_directory?: string): Promise<string[]>;
-/**
- * List repo files available for broken-link validation.
- *
- * @param {string} [project_directory]
- * @returns {Promise<string[]>}
- */
-export function listRepoFiles(project_directory?: string): Promise<string[]>;

package/lib/scan/list-source-files.js

@@ -0,0 +1,45 @@
+import process from 'node:process';
+
+import { listMatchingFiles } from './list-repo-files.js';
+/**
+ * Source file scanning.
+ *
+ * Expands include globs into stable repo-relative file lists for indexing and
+ * broken-link validation.
+ *
+ * Kind: scan
+ * Status: active
+ * Tracked in: ../../docs/plans/v0/source-anchor-dogfooding.md
+ * Decided by: ../../docs/decisions/source-scan.md
+ * @patram
+ * @see {@link ../graph/load-project-graph.js}
+ * @see {@link ../../docs/decisions/source-scan.md}
+ */
+
+/**
+ * List source files matched by Patram include globs.
+ *
+ * @param {string[]} include_patterns
+ * @param {string} [project_directory]
+ * @returns {Promise<string[]>}
+ */
+export async function listSourceFiles(
+  include_patterns,
+  project_directory = process.cwd(),
+) {
+  const source_file_paths = await listMatchingFiles(
+    include_patterns,
+    project_directory,
+  );
+
+  return [...new Set(source_file_paths)].sort(comparePaths);
+}
+
+/**
+ * @param {string} left_path
+ * @param {string} right_path
+ * @returns {number}
+ */
+function comparePaths(left_path, right_path) {
+  return left_path.localeCompare(right_path, 'en');
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "patram",
-  "version": "0.8.0",
+  "version": "0.10.0",
   "type": "module",
   "main": "./lib/patram.js",
   "types": "./lib/patram.d.ts",
@@ -15,7 +15,6 @@
     "bin/patram.js",
     "lib/**/*.d.ts",
     "lib/**/*.js",
-    "lib/**/*.ts",
     "!bin/**/*.test.d.ts",
     "!bin/**/*.test-helpers.d.ts",
     "!bin/**/*.test.js",
@@ -38,9 +37,11 @@
   },
   "license": "MIT",
   "scripts": {
-    "all": "npm run check:lint && npm run check:format && npm run check:types && npm run check:patram && npm run test:coverage && npm run check:dupes",
+    "all": "npm run check:lint && npm run check:format && npm run check:types && npm run check:patram && npm run check:knip && npm run check:knip:production && npm run test:coverage && npm run check:dupes",
     "check:dupes": "jscpd --min-tokens 100 --min-lines 6 --mode mild --threshold 0 --reporters console --gitignore .",
     "check:format": "prettier --check .",
+    "check:knip": "knip",
+    "check:knip:production": "knip --production --include exports",
     "check:lint": "eslint .",
     "check:patram": "./bin/patram.js check",
     "check:staged": "lint-staged",
@@ -62,7 +63,6 @@
     ]
   },
   "dependencies": {
-    "@shikijs/cli": "^4.0.2",
     "@shikijs/vscode-textmate": "^10.0.2",
     "ansis": "^4.2.0",
     "beautiful-mermaid": "^1.1.3",
@@ -77,16 +77,17 @@
   "devDependencies": {
     "@eslint/js": "^10.0.1",
     "@types/node": "^24.12.0",
-    "@vitest/coverage-v8": "^4.1.1",
+    "@vitest/coverage-istanbul": "^4.1.2",
     "eslint": "^10.1.0",
     "eslint-plugin-jsdoc": "^62.8.0",
     "globals": "^17.4.0",
     "husky": "^9.1.7",
     "jscpd": "^4.0.8",
+    "knip": "^6.1.0",
     "lint-staged": "^16.2.6",
     "prettier": "^3.5.3",
-    "slice-ansi": "^8.0.0",
-    "typescript": "^6.0.2",
+    "typescript": "^5.9.3",
+    "typescript-eslint": "^8.57.2",
     "vitest": "^4.1.1"
   }
 }

package/lib/build-graph.types.ts

@@ -1,27 +0,0 @@
-import type { ClaimOrigin } from './parse-claims.types.ts';
-
-export interface GraphNode {
-  $class?: string;
-  $id?: string;
-  $path?: string;
-  id: string;
-  kind?: string;
-  key?: string;
-  path?: string;
-  title?: string;
-  [field: string]: string | string[] | undefined;
-}
-
-export interface GraphEdge {
-  from: string;
-  id: string;
-  origin: ClaimOrigin;
-  relation: string;
-  to: string;
-}
-
-export interface BuildGraphResult {
-  document_node_ids?: Record<string, string>;
-  edges: GraphEdge[];
-  nodes: Record<string, GraphNode>;
-}

package/lib/discover-fields.types.ts

@@ -1,52 +0,0 @@
-export type DiscoveredFieldTypeName =
-  | 'date'
-  | 'date_time'
-  | 'enum'
-  | 'glob'
-  | 'integer'
-  | 'path'
-  | 'string';
-
-export type DiscoveredFieldMultiplicity = 'multiple' | 'single';
-
-export interface FieldDiscoveryEvidenceReference {
-  column: number;
-  line: number;
-  path: string;
-  value: string;
-}
-
-export interface FieldDiscoveryClassUsage {
-  classes: string[];
-}
-
-export interface FieldDiscoveryTypeSuggestion {
-  confidence: number;
-  name: DiscoveredFieldTypeName;
-}
-
-export interface FieldDiscoveryMultiplicitySuggestion {
-  confidence: number;
-  name: DiscoveredFieldMultiplicity;
-}
-
-export interface FieldDiscoverySuggestion {
-  confidence: number;
-  conflicting_evidence: FieldDiscoveryEvidenceReference[];
-  evidence_references: FieldDiscoveryEvidenceReference[];
-  likely_class_usage: FieldDiscoveryClassUsage;
-  likely_multiplicity: FieldDiscoveryMultiplicitySuggestion;
-  likely_type: FieldDiscoveryTypeSuggestion;
-  name: string;
-}
-
-export interface FieldDiscoverySummary {
-  claim_count: number;
-  count: number;
-  source_file_count: number;
-}
-
-export interface FieldDiscoveryResult {
-  fields: FieldDiscoverySuggestion[];
-  summary: FieldDiscoverySummary;
-}