test-inventory 0.1.0 → 0.1.1

package/CHANGELOG.md

@@ -1,5 +1,23 @@
 # test-inventory
 
+## 0.1.1
+
+### Patch Changes
+
+- [#5](https://github.com/Huldoser/test-inventory/pull/5) [`c812450`](https://github.com/Huldoser/test-inventory/commit/c812450a0652339ccc713ce305a14882154dd6e6) - A custom test object imported from `'.'` or `'..'` is followed like other relative imports.
+
+- [#5](https://github.com/Huldoser/test-inventory/pull/5) [`c812450`](https://github.com/Huldoser/test-inventory/commit/c812450a0652339ccc713ce305a14882154dd6e6) - When a pattern names a folder, the `no-files-matched` message suggests a pattern for the test files in it, and the command line prints that message.
+
+- [#5](https://github.com/Huldoser/test-inventory/pull/5) [`c812450`](https://github.com/Huldoser/test-inventory/commit/c812450a0652339ccc713ce305a14882154dd6e6) - A pattern whose folder doesn't exist now gives only the `no-files-matched` warning, not a `read-error`.
+
+- [#5](https://github.com/Huldoser/test-inventory/pull/5) [`c812450`](https://github.com/Huldoser/test-inventory/commit/c812450a0652339ccc713ce305a14882154dd6e6) - Messages for problems that make Playwright refuse a file now say that Playwright runs no tests at all, in any file.
+
+- [#5](https://github.com/Huldoser/test-inventory/pull/5) [`c812450`](https://github.com/Huldoser/test-inventory/commit/c812450a0652339ccc713ce305a14882154dd6e6) - A new `test-in-function` warning marks a test declared in a function that is called more than once, from another describe, never, or from other files. The runner registers such a test in the describe around each call, while the record stays where the test is written.
+
+- [#5](https://github.com/Huldoser/test-inventory/pull/5) [`c812450`](https://github.com/Huldoser/test-inventory/commit/c812450a0652339ccc713ce305a14882154dd6e6) - A version range that starts below the oldest supported version, such as `^1.40.0`, now gets the `approximate-framework-version` warning, since the newest rules of its major apply.
+
+- [#5](https://github.com/Huldoser/test-inventory/pull/5) [`c812450`](https://github.com/Huldoser/test-inventory/commit/c812450a0652339ccc713ce305a14882154dd6e6) - Every test in a Vitest file that fails to load is now `notLoaded`, as Vitest runs none of them: a chain Vitest doesn't have, `sequential` or `bench` from Vitest 5, or options as the third argument from Vitest 4. A call to `bench` is reported where it is made, not where it is imported.
+
 ## 0.1.0
 
 ### Minor Changes

package/README.md

@@ -1,10 +1,10 @@
-# test-inventory
+# <picture><source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/Huldoser/test-inventory/main/docs/public/logo-dark.svg"><img src="https://raw.githubusercontent.com/Huldoser/test-inventory/main/docs/public/logo-light.svg" alt="test-inventory" height="48"></picture>
 
 Test metadata from source code, not from test results.
 
 test-inventory reads Playwright and Vitest test files and lists every test declaration as JSON: its title, file and
 lines, the describes around it, its tags and annotations, and whether it is active, skipped, fixme, todo, expected
-to fail or in a file Playwright refuses to load, with the condition when the state only applies sometimes. It parses
+to fail or in a file the runner refuses to load, with the condition when the state only applies sometimes. It parses
 the code and runs nothing, so it needs no browsers, no build and no runner configuration.
 
 Documentation: [huldoser.github.io/test-inventory](https://huldoser.github.io/test-inventory/)

package/dist/cli.js

@@ -130,8 +130,9 @@ export async function run(argv, io) {
         io.stderr.write(`test-inventory: unexpected error: ${error instanceof Error ? String(error.stack) : String(error)}\n`);
         return FAILED;
     }
-    if (inventory.diagnostics.some((diagnostic) => diagnostic.code === 'no-files-matched')) {
-        io.stderr.write('test-inventory: no test files match the patterns.\n');
+    const unmatched = inventory.diagnostics.find((diagnostic) => diagnostic.code === 'no-files-matched');
+    if (unmatched) {
+        io.stderr.write(`test-inventory: ${unmatched.message[0].toLowerCase()}${unmatched.message.slice(1)}\n`);
     }
     // A reader that stops early, as `| head` does, closes the pipe: an EPIPE error, which ends the output quietly.
     const quiet = () => undefined;

package/dist/diagnostics.d.ts

@@ -3,7 +3,10 @@ import type { SourceText } from './source.ts';
 import type { Diagnostic, DiagnosticCode, DiagnosticLevel, Framework } from './types.ts';
 /** `error` when tests may be missing from the output; `warning` when a value is missing or approximate. */
 export declare const LEVELS: Record<DiagnosticCode, DiagnosticLevel>;
-/** Problems for which the runner refuses to load the whole file, so that none of its tests run. */
+/**
+ * Problems for which the runner refuses to load the whole file, so that none of its tests run. In Vitest, these are
+ * calls that throw while the file is collected.
+ */
 export declare const REJECTS_FILE: Record<Framework, ReadonlySet<DiagnosticCode>>;
 export declare function toDiagnostic(problem: Problem, relativeFilePath: string, source: SourceText | null, testId?: string | null): Diagnostic;
 export declare function fileDiagnostic(code: DiagnosticCode, relativeFilePath: string, message: string): Diagnostic;

package/dist/diagnostics.js

@@ -34,11 +34,15 @@ export const LEVELS = {
     'no-files-matched': 'warning',
     'unsupported-file': 'warning',
     'approximate-framework-version': 'warning',
+    'test-in-function': 'warning',
 };
-/** Problems for which the runner refuses to load the whole file, so that none of its tests run. */
+/**
+ * Problems for which the runner refuses to load the whole file, so that none of its tests run. In Vitest, these are
+ * calls that throw while the file is collected.
+ */
 export const REJECTS_FILE = {
     playwright: new Set(['test-without-body', 'tag-without-at', 'invalid-chain', 'duplicate-title']),
-    vitest: new Set(),
+    vitest: new Set(['invalid-chain', 'removed-api']),
 };
 export function toDiagnostic(problem, relativeFilePath, source, testId = null) {
     const position = problem.node && source ? source.position(problem.node.start) : null;

package/dist/extract.js

@@ -3,6 +3,14 @@ import { exportName } from "./frameworks/bindings.js";
 import { problem } from "./frameworks/common.js";
 import { snippet, splitLines } from "./source.js";
 import { isScopeNode } from "./values.js";
+/** Whether a function is called where it is written, so that its tests are registered there. */
+function calledWhereWritten(call, written) {
+    return (call.scope === written.scope &&
+        call.helper === written.helper &&
+        call.guards === written.guards &&
+        call.isInLoop === written.isInLoop &&
+        !call.inBody);
+}
 const LOOP_METHODS = new Set(['forEach', 'map']);
 /** Joins conditions with `&&`, adding parentheses where a part could otherwise be read differently. */
 function joinConditions(parts) {
@@ -30,6 +38,12 @@ class Walker {
     values;
     effects = new Map();
     helperCalls = [];
+    /** Where each function with a name is written, outside test and hook bodies. */
+    written = new Map();
+    /** Every call by name to a function declared in the file. */
+    callSites = new Map();
+    /** The function with a name each test is declared in, which registers the test when it is called. */
+    declaredIn = new Map();
     constructor(parsed, adapter, values) {
         this.parsed = parsed;
         this.adapter = adapter;
@@ -55,8 +69,11 @@ class Walker {
             isInCondition: false,
             isInFunction: false,
             helper: null,
+            declaredIn: null,
         });
-        this.applyHelpers();
+        const exported = this.exportedFunctions(this.parsed.program);
+        this.applyHelpers(exported);
+        this.checkFunctionTests(exported);
         return {
             suites: this.suites,
             tests: this.tests,
@@ -134,17 +151,36 @@ class Walker {
             case 'ArrowFunctionExpression': {
                 // A function outside any test or hook runs when it is called; what it sets applies where it is called.
                 const helper = context.test || context.hookScope ? null : node;
+                const named = helper !== null && this.isNamed(node);
+                if (named)
+                    this.written.set(node, { ...this.placement(context), declaredIn: context.declaredIn });
                 this.visitChildren(node, {
                     ...context,
                     isInFunction: true,
                     helper: helper ?? context.helper,
                     guards: helper ? [] : context.guards,
+                    declaredIn: named ? node : context.declaredIn,
                 });
                 return;
             }
         }
         this.visitChildren(node, context);
     }
+    /** Whether a function has a name it can be called by: `function name() {}` or `const name = () => {}`. */
+    isNamed(fn) {
+        const parent = this.parents.at(-2);
+        return ((fn.type === 'FunctionDeclaration' && fn.id !== null) ||
+            (parent?.type === 'VariableDeclarator' && parent.id.type === 'Identifier' && parent.init === fn));
+    }
+    placement(context) {
+        return {
+            scope: context.scope,
+            helper: context.helper,
+            guards: joinConditions(context.guards),
+            isInLoop: context.isInLoop,
+            inBody: context.test !== null || context.hookScope !== null,
+        };
+    }
     guarded(context, guard) {
         return { ...context, isInCondition: true, guards: guard === null ? context.guards : [...context.guards, guard] };
     }
@@ -266,6 +302,9 @@ class Walker {
         const fn = callee.type === 'Identifier' ? this.functionNamed(callee.name, this.scopes) : null;
         if (!fn)
             return;
+        const sites = this.callSites.get(fn) ?? [];
+        sites.push(this.placement(context));
+        this.callSites.set(fn, sites);
         if (context.helper) {
             this.effectsOf(context.helper).push({ kind: 'call', fn, guards: context.guards });
         }
@@ -307,8 +346,7 @@ class Walker {
      * Applies what helper functions set to the tests, hooks and describes that call them, through nested helpers too,
      * with the conditions around each call. A helper no test or hook reaches, or that the file exports, is reported.
      */
-    applyHelpers() {
-        const exported = this.exportedFunctions(this.parsed.program);
+    applyHelpers(exported) {
         const reached = new Set();
         const expand = (fn, guards, path) => {
             if (path.has(fn) || exported.has(fn))
@@ -340,6 +378,35 @@ class Walker {
             this.problems.push(problem('state-in-helper', own.call, `${callee}() is in a function that ${why}; it applies to the tests that call the function, so it is not applied here.`, callee));
         }
     }
+    /** Why calling a function with a name doesn't register its tests where they are written; null when it does. */
+    misplaced(fn, exported) {
+        if (exported.has(fn))
+            return 'the file exports the function';
+        const sites = this.callSites.get(fn) ?? [];
+        if (sites.length === 0)
+            return 'nothing in the file calls the function by name';
+        if (sites.length > 1)
+            return `the function is called ${sites.length} times`;
+        const written = this.written.get(fn);
+        return calledWhereWritten(sites[0], written)
+            ? null
+            : 'the function is called in another place than where it is written';
+    }
+    /**
+     * Warns about tests declared in a function with a name, or in one around it, whose calls register them somewhere
+     * else than where they are written: the runner adds a test to the describe around each call.
+     */
+    checkFunctionTests(exported) {
+        for (const [test, innermost] of this.declaredIn) {
+            let why = null;
+            for (let fn = innermost; fn && !why; fn = this.written.get(fn).declaredIn) {
+                why = this.misplaced(fn, exported);
+            }
+            if (!why)
+                continue;
+            test.problems.push(problem('test-in-function', test.call, `The test is declared in a function, and ${why}. The runner registers the test at each call of the function, in the describe around the call; it is listed once, where it is written.`));
+        }
+    }
     nested(call, context) {
         if (!context.test && !context.hookScope)
             return false;
@@ -363,6 +430,8 @@ class Walker {
             problems: [...declaration.problems],
         };
         this.tests.push(test);
+        if (context.declaredIn)
+            this.declaredIn.set(test, context.declaredIn);
         context.scope.suite?.cuts.push(statement);
         if (declaration.fn) {
             this.visitCallback(declaration.fn, {

package/dist/files.js

@@ -59,6 +59,9 @@ export async function findFiles(root, patterns, ignore, framework) {
     for (const failure of failures) {
         const where = relative(failure.path);
         if (failure.kind === 'folder') {
+            // A folder a pattern names that isn't there holds no tests; `no-files-matched` reports a scan that found none.
+            if (failure.error.code === 'ENOENT' || failure.error.code === 'ENOTDIR')
+                continue;
             const message = `The folder could not be read: ${readFailure(failure.error)}; test files in it are missing.`;
             diagnostics.push(fileDiagnostic('read-error', `${where}/`, message));
         }

package/dist/frameworks/playwright.js

@@ -345,7 +345,7 @@ export class PlaywrightAdapter {
             const callee = snippet(text(this.file, call.callee));
             return {
                 kind: 'invalid',
-                problem: problem('test-without-body', call, `${callee}() has no test body, so Playwright fails to load the file.`),
+                problem: problem('test-without-body', call, `${callee}() has no test body, so Playwright fails to load the file and runs no tests at all.`),
             };
         }
         const state = STATES.get(key) ?? null;
@@ -468,7 +468,7 @@ export class PlaywrightAdapter {
                     details.tags.push(value);
                 }
                 else {
-                    problems.push(problem('tag-without-at', tag.node, `Tag "${value}" does not start with "@", so Playwright fails to load the file.`, value));
+                    problems.push(problem('tag-without-at', tag.node, `Tag "${value}" does not start with "@", so Playwright fails to load the file and runs no tests at all.`, value));
                 }
             }
             for (const { node: unresolved, importedFrom } of tags.unresolved) {

package/dist/frameworks/vitest.d.ts

@@ -19,6 +19,8 @@ export declare class VitestAdapter implements Adapter {
     private warnedAboutTags;
     constructor(parsed: ParsedFile, file: FileContext, readModule: ModuleReader | null);
     classify(call: CallExpression, context: CallContext): ApiCall | null;
+    /** A call to `bench` imported from Vitest 5, which is undefined there, so the call throws. */
+    private benchCall;
     private chainCall;
     /**
      * The function a chain calls and the links after it, or null when it isn't one of Vitest's. An import that can't be

package/dist/frameworks/vitest.js

@@ -33,6 +33,8 @@ class VitestBindings {
     kinds = new Map();
     hooks = new Map();
     namespaces = new Set();
+    /** Names `bench` is imported as from Vitest 5, which no longer exports it. */
+    benches = new Set();
     /** The import specifiers and declarators that bind those names, to tell them from local names that shadow them. */
     declarations = new Set();
     problems = [];
@@ -55,7 +57,7 @@ class VitestBindings {
                     this.problems.push(problem('invalid-import', specifier, "Vitest has no default export; import { test, describe } from 'vitest' instead."));
                 }
                 else {
-                    this.bindImported(exportName(specifier.imported), specifier.local.name, specifier, version);
+                    this.bindImported(exportName(specifier.imported), specifier.local.name, version);
                 }
             }
         }
@@ -100,7 +102,7 @@ class VitestBindings {
         const node = unwrap(expression);
         return node.type === 'Identifier' && this.namespaces.has(node.name);
     }
-    bindImported(imported, local, node, version) {
+    bindImported(imported, local, version) {
         const kind = KINDS.get(imported);
         if (kind) {
             this.kinds.set(local, kind);
@@ -109,7 +111,7 @@ class VitestBindings {
             this.hooks.set(local, imported);
         }
         else if (imported === 'bench' && atLeast(version, 5, 0)) {
-            this.problems.push(problem('removed-api', node, "Vitest 5 no longer exports bench from 'vitest'; benchmarks in this file are not tests."));
+            this.benches.add(local);
         }
     }
     bind(declarator) {
@@ -166,7 +168,24 @@ export class VitestAdapter {
         }
     }
     classify(call, context) {
-        return this.hookCall(call, context) ?? this.contextSkip(call, context) ?? this.chainCall(call, context);
+        return (this.benchCall(call, context) ??
+            this.hookCall(call, context) ??
+            this.contextSkip(call, context) ??
+            this.chainCall(call, context));
+    }
+    /** A call to `bench` imported from Vitest 5, which is undefined there, so the call throws. */
+    benchCall(call, context) {
+        const base = readChain(call.callee)?.base;
+        if (base?.type !== 'Identifier' || !this.bindings.benches.has(base.name))
+            return null;
+        const declaration = this.file.values.declarationOf(base.name, context.scopes);
+        if (declaration === null || !this.bindings.declarations.has(declaration))
+            return null;
+        const callee = snippet(text(this.file, call.callee));
+        return {
+            kind: 'invalid',
+            problem: problem('removed-api', call, `Vitest 5 no longer exports bench from 'vitest'; ${callee}(...) fails. Benchmarks are not tests.`, callee),
+        };
     }
     chainCall(call, context) {
         const chain = readChain(call.callee);

package/dist/records.js

@@ -231,7 +231,7 @@ class RecordBuilder {
             seen.set(group, keys);
             if (keys.has(key)) {
                 const message = playwright
-                    ? 'Another test in the file has the same title path, so Playwright fails to load the file.'
+                    ? 'Another test in the file has the same title path, so Playwright fails to load the file and runs no tests at all.'
                     : 'Another test in the same describe has the same title.';
                 const item = problem('duplicate-title', test.call, message, test.declaration.title.text);
                 found.push({ index, problem: playwright ? { ...item, level: 'error' } : item });

package/dist/resolve.js

@@ -179,7 +179,8 @@ export function moduleResolver(root) {
     return (fromFile) => (specifier) => {
         const directory = path.dirname(fromFile);
         let candidates;
-        if (specifier.startsWith('./') || specifier.startsWith('../')) {
+        // `.` and `..` are paths too: the index of the file's folder, or of the folder above.
+        if (/^\.\.?(?:\/|$)/.test(specifier)) {
             candidates = [path.resolve(directory, specifier)];
         }
         else if (specifier.startsWith('#')) {

package/dist/scan.js

@@ -2,7 +2,7 @@ import { existsSync, statSync } from 'node:fs';
 import { readFile } from 'node:fs/promises';
 import path from 'node:path';
 import { fileDiagnostic } from "./diagnostics.js";
-import { findFiles, readFailure } from "./files.js";
+import { findFiles, normalizePattern, readFailure } from "./files.js";
 import { detectFrameworkVersion, OLDEST } from "./framework-version.js";
 import { createInventory } from "./inventory.js";
 import { checkComments, checkFramework, checkObject, checkStrings, checkVersion, invalid, majorMinor, } from "./options.js";
@@ -18,7 +18,9 @@ function versionDiagnostics(framework, version, source, range) {
     }
     const [major, minor] = majorMinor(version);
     const [oldestMajor, oldestMinor] = OLDEST[framework];
-    if (major < oldestMajor || (major === oldestMajor && minor < oldestMinor)) {
+    // A range gets the newest rules of its major, so only its major has to be supported.
+    const tooOld = major < oldestMajor || (major === oldestMajor && minor < oldestMinor && range === null);
+    if (tooOld) {
         return [
             fileDiagnostic('unsupported-framework-version', source, `${name} ${version} is older than ${oldestMajor}.${oldestMinor}, the oldest supported version; the results follow ${oldestMajor}.${oldestMinor}.`),
         ];
@@ -29,6 +31,17 @@ function versionDiagnostics(framework, version, source, range) {
         fileDiagnostic('approximate-framework-version', source, `Only the range "${range}" was found for ${name}, so the rules of the newest ${major}.x release apply.`),
     ];
 }
+/** For a pattern that names a folder, which matches no file, a pattern that matches the test files in it. */
+function folderHint(root, patterns, framework) {
+    const folder = patterns.find((pattern) => {
+        const target = path.resolve(root, normalizePattern(pattern));
+        return existsSync(target) && statSync(target).isDirectory();
+    });
+    if (folder === undefined)
+        return '';
+    const example = `${folder.replace(/[\\/]+$/, '')}/**/*.${framework === 'playwright' ? 'spec' : 'test'}.ts`;
+    return ` "${folder}" is a folder; to scan the test files in it, use a pattern such as "${example}".`;
+}
 /**
  * Scans the test files that match `patterns` under `root`, one file at a time.
  *
@@ -60,7 +73,8 @@ export async function scan(options) {
     projectDiagnostics.push(...found.diagnostics);
     if (found.files.length === 0) {
         const list = patterns.map((pattern) => `"${pattern}"`).join(', ');
-        projectDiagnostics.push(fileDiagnostic('no-files-matched', '.', `No test files match ${list} in the root.`));
+        const message = `No test files match ${list} in the root.${folderHint(root, patterns, framework)}`;
+        projectDiagnostics.push(fileDiagnostic('no-files-matched', '.', message));
     }
     const readerFor = moduleResolver(root);
     const results = [];

package/dist/types.d.ts

@@ -20,7 +20,7 @@ export type TestState = 'active' | 'skip' | 'fixme' | 'todo' | 'expectedToFail'
 export type StateSource = 'test' | 'suite' | 'file';
 /** `error`: tests may be missing from the output. `warning`: a value is missing or approximate. */
 export type DiagnosticLevel = 'error' | 'warning';
-export type DiagnosticCode = 'unresolved-test-import' | 'parse-error' | 'read-error' | 'framework-mismatch' | 'unresolved-tag' | 'unresolved-annotation' | 'unresolved-cases' | 'test-in-loop' | 'dynamic-title' | 'duplicate-title' | 'duplicate-id' | 'internal-error' | 'invalid-chain' | 'skip-without-body' | 'test-without-body' | 'describe-not-called' | 'async-describe' | 'nested-test' | 'tag-without-at' | 'removed-api' | 'only-in-skipped-describe' | 'config-may-change-state' | 'symlink-ignored' | 'invalid-import' | 'globals-not-enabled' | 'unknown-framework-version' | 'unsupported-framework-version' | 'local-test-object' | 'state-in-helper' | 'unresolved-option' | 'dynamic-reason' | 'no-files-matched' | 'unsupported-file' | 'approximate-framework-version';
+export type DiagnosticCode = 'unresolved-test-import' | 'parse-error' | 'read-error' | 'framework-mismatch' | 'unresolved-tag' | 'unresolved-annotation' | 'unresolved-cases' | 'test-in-loop' | 'dynamic-title' | 'duplicate-title' | 'duplicate-id' | 'internal-error' | 'invalid-chain' | 'skip-without-body' | 'test-without-body' | 'describe-not-called' | 'async-describe' | 'nested-test' | 'tag-without-at' | 'removed-api' | 'only-in-skipped-describe' | 'config-may-change-state' | 'symlink-ignored' | 'invalid-import' | 'globals-not-enabled' | 'unknown-framework-version' | 'unsupported-framework-version' | 'local-test-object' | 'state-in-helper' | 'unresolved-option' | 'dynamic-reason' | 'no-files-matched' | 'unsupported-file' | 'approximate-framework-version' | 'test-in-function';
 export type MetaValue = string | number | boolean | null | MetaValue[] | {
     [key: string]: MetaValue;
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "test-inventory",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "Test metadata from source code, not from test results.",
   "keywords": [
     "testing",
@@ -65,6 +65,7 @@
     "licenses": "node scripts/check-licenses.ts",
     "compare": "node scripts/compare-runners.ts",
     "corpus": "node scripts/scan-corpus.ts",
+    "release": "node scripts/release.ts",
     "lint": "eslint .",
     "lint:fix": "eslint . --fix",
     "format": "prettier --write . --ignore-path .gitignore --ignore-path .prettierignore --ignore-path .git/info/exclude",

package/schema/v1.json

@@ -880,7 +880,8 @@
             "dynamic-reason",
             "no-files-matched",
             "unsupported-file",
-            "approximate-framework-version"
+            "approximate-framework-version",
+            "test-in-function"
           ]
         },
         {