waymark-docs 0.1.0

package/dist/documents/filter.js

@@ -0,0 +1,212 @@
+export function filterDocuments({ documents, configuration, criteria, }) {
+    const kinds = parseIdentifierOptions({
+        optionName: "--kinds",
+        values: criteria.kinds,
+        declarations: configuration.kinds,
+        declarationName: "kind",
+    });
+    const tags = parseIdentifierOptions({
+        optionName: "--tags",
+        values: criteria.tags,
+        declarations: configuration.tags,
+        declarationName: "tag",
+    });
+    const requiredTags = parseIdentifierOptions({
+        optionName: "--require-tags",
+        values: criteria.requiredTags,
+        declarations: configuration.tags,
+        declarationName: "tag",
+    });
+    const matchesMetadataFilter = criteria.filter === undefined
+        ? undefined
+        : parseMetadataFilter({
+            expression: criteria.filter,
+            declaredKinds: new Set(configuration.kinds.keys()),
+            declaredTags: new Set(configuration.tags.keys()),
+        });
+    const normalizedQuery = criteria.query?.toLowerCase();
+    return documents
+        .filter((document) => (kinds.size === 0 || kinds.has(document.kind)) &&
+        (tags.size === 0 || document.tags.some((tag) => tags.has(tag))) &&
+        [...requiredTags].every((tag) => document.tags.includes(tag)) &&
+        (matchesMetadataFilter === undefined ||
+            matchesMetadataFilter(document)) &&
+        (normalizedQuery === undefined ||
+            document.body.toLowerCase().includes(normalizedQuery)))
+        .sort((left, right) => compareText(left.path, right.path));
+}
+export function parseMetadataFilter({ expression, declaredKinds, declaredTags, }) {
+    const tokens = tokenize(expression, declaredKinds, declaredTags);
+    let nextTokenIndex = 0;
+    function peek() {
+        return tokens[nextTokenIndex];
+    }
+    function consume() {
+        const token = tokens[nextTokenIndex];
+        if (!token) {
+            throw syntaxError(expression.length, 'Expected a kind: or tag: predicate, or "(".');
+        }
+        nextTokenIndex += 1;
+        return token;
+    }
+    function parsePrimary() {
+        const token = consume();
+        if (token.type === "kind" || token.type === "tag") {
+            return {
+                type: token.type,
+                identifier: token.identifier,
+            };
+        }
+        if (token.type === "left-parenthesis") {
+            const expressionNode = parseOr();
+            const closingToken = peek();
+            if (closingToken?.type !== "right-parenthesis") {
+                throw syntaxError(closingToken?.position ?? expression.length, 'Expected ")".');
+            }
+            consume();
+            return expressionNode;
+        }
+        throw syntaxError(token.position, 'Expected a kind: or tag: predicate, or "(".');
+    }
+    function parseNot() {
+        if (peek()?.type !== "not")
+            return parsePrimary();
+        consume();
+        return { type: "not", operand: parseNot() };
+    }
+    function parseAnd() {
+        let node = parseNot();
+        while (peek()?.type === "and") {
+            consume();
+            node = { type: "and", left: node, right: parseNot() };
+        }
+        return node;
+    }
+    function parseOr() {
+        let node = parseAnd();
+        while (peek()?.type === "or") {
+            consume();
+            node = { type: "or", left: node, right: parseAnd() };
+        }
+        return node;
+    }
+    if (tokens.length === 0) {
+        throw syntaxError(0, "Expression cannot be empty.");
+    }
+    const root = parseOr();
+    const remainingToken = peek();
+    if (remainingToken) {
+        const message = remainingToken.type === "kind" ||
+            remainingToken.type === "tag" ||
+            remainingToken.type === "not" ||
+            remainingToken.type === "left-parenthesis"
+            ? "Expected AND or OR before this token."
+            : `Unexpected ${describeToken(remainingToken)}.`;
+        throw syntaxError(remainingToken.position, message);
+    }
+    return (document) => evaluate(root, document);
+}
+function tokenize(expression, declaredKinds, declaredTags) {
+    const tokens = [];
+    let position = 0;
+    while (position < expression.length) {
+        if (/\s/.test(expression[position] ?? "")) {
+            position += 1;
+            continue;
+        }
+        if (expression[position] === "(") {
+            tokens.push({ type: "left-parenthesis", position });
+            position += 1;
+            continue;
+        }
+        if (expression[position] === ")") {
+            tokens.push({ type: "right-parenthesis", position });
+            position += 1;
+            continue;
+        }
+        const tokenPosition = position;
+        while (position < expression.length &&
+            !/\s|\(|\)/.test(expression[position] ?? "")) {
+            position += 1;
+        }
+        const value = expression.slice(tokenPosition, position);
+        const operator = value.toUpperCase();
+        if (operator === "NOT") {
+            tokens.push({ type: "not", position: tokenPosition });
+            continue;
+        }
+        if (operator === "AND") {
+            tokens.push({ type: "and", position: tokenPosition });
+            continue;
+        }
+        if (operator === "OR") {
+            tokens.push({ type: "or", position: tokenPosition });
+            continue;
+        }
+        const predicate = /^(kind|tag):([a-z0-9]+(?:-[a-z0-9]+)*)$/.exec(value);
+        if (!predicate) {
+            throw syntaxError(tokenPosition, `Unsupported token "${value}". Expected kind:<identifier>, tag:<identifier>, NOT, AND, OR, or parentheses.`);
+        }
+        const [, predicateType, identifier] = predicate;
+        if (predicateType !== "kind" && predicateType !== "tag") {
+            throw new Error("Metadata Filter parser invariant failed.");
+        }
+        if (!identifier)
+            throw new Error("Metadata Filter parser invariant failed.");
+        const declarations = predicateType === "kind" ? declaredKinds : declaredTags;
+        if (!declarations.has(identifier)) {
+            throw syntaxError(tokenPosition, `Undeclared ${predicateType} "${identifier}".`);
+        }
+        tokens.push({
+            type: predicateType,
+            identifier,
+            position: tokenPosition,
+        });
+    }
+    return tokens;
+}
+function evaluate(node, document) {
+    switch (node.type) {
+        case "kind":
+            return document.kind === node.identifier;
+        case "tag":
+            return document.tags.includes(node.identifier);
+        case "not":
+            return !evaluate(node.operand, document);
+        case "and":
+            return evaluate(node.left, document) && evaluate(node.right, document);
+        case "or":
+            return evaluate(node.left, document) || evaluate(node.right, document);
+    }
+}
+function syntaxError(position, message) {
+    return new Error(`Metadata Filter syntax error at position ${position + 1}: ${message}`);
+}
+function describeToken(token) {
+    if (token.type === "right-parenthesis")
+        return '")"';
+    if (token.type === "left-parenthesis")
+        return '"("';
+    return `"${token.type.toUpperCase()}"`;
+}
+function parseIdentifierOptions({ optionName, values, declarations, declarationName, }) {
+    const identifiers = new Set();
+    for (const value of values) {
+        for (const identifier of value.split(",")) {
+            if (identifier === "") {
+                throw new Error(`${optionName} contains an empty identifier.`);
+            }
+            if (identifiers.has(identifier)) {
+                throw new Error(`${optionName} contains duplicate identifier "${identifier}".`);
+            }
+            if (!declarations.has(identifier)) {
+                throw new Error(`${optionName} contains undeclared ${declarationName} "${identifier}".`);
+            }
+            identifiers.add(identifier);
+        }
+    }
+    return identifiers;
+}
+function compareText(left, right) {
+    return left < right ? -1 : left > right ? 1 : 0;
+}

package/dist/documents/index.d.ts

@@ -0,0 +1,3 @@
+export { filterDocuments } from "./filter.js";
+export { scanDocuments } from "./scan.js";
+export type { DocumentScanDiagnostic, DocumentScanResult, WaymarkDocument, } from "./scan.js";

package/dist/documents/index.js

@@ -0,0 +1,2 @@
+export { filterDocuments } from "./filter.js";
+export { scanDocuments } from "./scan.js";

package/dist/documents/scan.d.ts

@@ -0,0 +1,33 @@
+import { type Configuration } from "../configuration/index.js";
+export type WaymarkDocument = {
+    path: string;
+    kind: string;
+    description: string;
+    tags: string[];
+    body: string;
+};
+export type DocumentScanDiagnostic = {
+    path: string;
+    field: string;
+    message: string;
+};
+type DocumentScanScope = {
+    directoryPath: string;
+    recursive: boolean;
+};
+export type DocumentScanResult = {
+    kind: "valid";
+    documents: WaymarkDocument[];
+    unregisteredDocuments: string[];
+    kindUsageCounts: Map<string, number>;
+    tagUsageCounts: Map<string, number>;
+} | {
+    kind: "invalid";
+    diagnostics: DocumentScanDiagnostic[];
+};
+export declare function scanDocuments({ rootPath, configuration, scope, }: {
+    rootPath: string;
+    configuration: Configuration;
+    scope?: DocumentScanScope;
+}): Promise<DocumentScanResult>;
+export {};

package/dist/documents/scan.js

@@ -0,0 +1,134 @@
+import { lstat, readFile, realpath } from "node:fs/promises";
+import { isAbsolute, join, relative, sep } from "node:path";
+import { convertPathToPattern, globby } from "globby";
+import { configurationFileName, } from "../configuration/index.js";
+import { isErrorWithCode } from "../filesystem.js";
+import { classifyDocument } from "./classify.js";
+export async function scanDocuments({ rootPath, configuration, scope, }) {
+    const { rootPath: scanRootPath, scope: resolvedScope } = await resolveDocumentScanScope({ rootPath, scope });
+    const discoveredPaths = await globby(createCandidatePatterns(resolvedScope), {
+        cwd: scanRootPath,
+        gitignore: true,
+        ignore: ["**/.git", "**/.git/**", ...configuration.exclusions],
+        dot: true,
+        followSymbolicLinks: false,
+        braceExpansion: false,
+        extglob: false,
+    });
+    const documents = [];
+    const kindUsageCounts = createUsageCounts(configuration.kinds);
+    const tagUsageCounts = createUsageCounts(configuration.tags);
+    const diagnostics = [];
+    const unregisteredDocuments = [];
+    for (const path of discoveredPaths.sort(compareText)) {
+        if (path === configurationFileName)
+            continue;
+        if (path.endsWith(`/${configurationFileName}`)) {
+            diagnostics.push({
+                path,
+                field: "configuration",
+                message: "Nested Waymark configurations are not allowed.",
+            });
+            continue;
+        }
+        const source = await readFile(join(scanRootPath, path), "utf8");
+        const classification = classifyDocument({
+            source,
+            requireNamespace: configuration.requireNamespace,
+            declaredKinds: configuration.kinds,
+            declaredTags: configuration.tags,
+            path,
+        });
+        if (classification.kind === "unregistered") {
+            unregisteredDocuments.push(path);
+            continue;
+        }
+        if (classification.kind === "invalid") {
+            diagnostics.push(...classification.diagnostics);
+            continue;
+        }
+        const document = { path, ...classification.document };
+        documents.push(document);
+        incrementUsageCount(kindUsageCounts, document.kind);
+        for (const tag of document.tags)
+            incrementUsageCount(tagUsageCounts, tag);
+    }
+    if (diagnostics.length > 0) {
+        return {
+            kind: "invalid",
+            diagnostics: diagnostics.sort(compareDiagnostics),
+        };
+    }
+    return {
+        kind: "valid",
+        documents,
+        unregisteredDocuments,
+        kindUsageCounts,
+        tagUsageCounts,
+    };
+}
+async function resolveDocumentScanScope({ rootPath, scope, }) {
+    if (!scope)
+        return { rootPath, scope };
+    let directoryStats;
+    try {
+        directoryStats = await lstat(scope.directoryPath);
+    }
+    catch (error) {
+        if (isErrorWithCode(error, "ENOENT")) {
+            throw new Error(`Directory does not exist: ${scope.directoryPath}`);
+        }
+        throw error;
+    }
+    if (!directoryStats.isDirectory()) {
+        throw new Error(`Not a directory: ${scope.directoryPath}`);
+    }
+    const canonicalRootPath = await realpath(rootPath);
+    const canonicalDirectoryPath = await realpath(scope.directoryPath);
+    const relativeDirectoryPath = relative(canonicalRootPath, canonicalDirectoryPath);
+    if (relativeDirectoryPath !== "" &&
+        (relativeDirectoryPath === ".." ||
+            relativeDirectoryPath.startsWith(`..${sep}`) ||
+            isAbsolute(relativeDirectoryPath))) {
+        throw new Error(`Directory is outside the configuration root: ${scope.directoryPath}`);
+    }
+    return {
+        rootPath: canonicalRootPath,
+        scope: {
+            relativeDirectoryPath,
+            recursive: scope.recursive,
+        },
+    };
+}
+function createCandidatePatterns(scope) {
+    if (!scope) {
+        return ["**/*.md", "**/*.mdx", `**/${configurationFileName}`];
+    }
+    const directoryPattern = convertPathToPattern(scope.relativeDirectoryPath);
+    const directoryPrefix = directoryPattern === "" ? "" : `${directoryPattern}/`;
+    const candidatePrefix = scope.recursive
+        ? `${directoryPrefix}**/`
+        : directoryPrefix;
+    return [
+        `${candidatePrefix}*.md`,
+        `${candidatePrefix}*.mdx`,
+        `${candidatePrefix}${configurationFileName}`,
+    ];
+}
+function createUsageCounts(declarations) {
+    return new Map([...declarations.keys()].map((identifier) => [identifier, 0]));
+}
+function incrementUsageCount(usageCounts, identifier) {
+    const currentCount = usageCounts.get(identifier);
+    if (currentCount !== undefined) {
+        usageCounts.set(identifier, currentCount + 1);
+    }
+}
+function compareDiagnostics(left, right) {
+    return (compareText(left.path, right.path) ||
+        compareText(left.field, right.field) ||
+        compareText(left.message, right.message));
+}
+function compareText(left, right) {
+    return left < right ? -1 : left > right ? 1 : 0;
+}

package/dist/filesystem.d.ts

@@ -0,0 +1,2 @@
+export declare function pathExists(path: string): Promise<boolean>;
+export declare function isErrorWithCode(error: unknown, code: string): error is NodeJS.ErrnoException;

package/dist/filesystem.js

@@ -0,0 +1,15 @@
+import { lstat } from "node:fs/promises";
+export async function pathExists(path) {
+    try {
+        await lstat(path);
+        return true;
+    }
+    catch (error) {
+        if (isErrorWithCode(error, "ENOENT"))
+            return false;
+        throw error;
+    }
+}
+export function isErrorWithCode(error, code) {
+    return error instanceof Error && "code" in error && error.code === code;
+}

package/package.json

@@ -0,0 +1,55 @@
+{
+  "name": "waymark-docs",
+  "version": "0.1.0",
+  "description": "Find repository documentation relevant to coding agents",
+  "keywords": [
+    "ai-agents",
+    "cli",
+    "documentation",
+    "markdown",
+    "repository"
+  ],
+  "homepage": "https://github.com/ysfaran/waymark#readme",
+  "bugs": {
+    "url": "https://github.com/ysfaran/waymark/issues"
+  },
+  "license": "MIT",
+  "author": "Yusuf Aran",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ysfaran/waymark.git",
+    "directory": "cli"
+  },
+  "bin": {
+    "waymark": "dist/cli.js"
+  },
+  "files": [
+    "CHANGELOG.md",
+    "dist"
+  ],
+  "type": "module",
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "pnpm clean && tsc --project tsconfig.json && chmod +x dist/cli.js",
+    "clean": "rm -rf dist",
+    "dev": "tsx src/cli.ts",
+    "prepack": "cp ../README.md ../CHANGELOG.md ../LICENSE . && pnpm build",
+    "postpack": "rm -f README.md CHANGELOG.md LICENSE",
+    "typecheck": "tsc --project tsconfig.json --noEmit"
+  },
+  "dependencies": {
+    "commander": "14.0.3",
+    "globby": "16.2.2",
+    "yaml": "2.9.0"
+  },
+  "devDependencies": {
+    "@types/node": "24.10.10",
+    "tsx": "4.20.6",
+    "typescript": "7.0.2"
+  },
+  "engines": {
+    "node": ">=24"
+  }
+}