ucn 5.5.3 → 5.6.1

package/core/python-modules.js

@@ -141,6 +141,12 @@ function pythonModuleScope(index, file, receiver, site, memo = new Map()) {
                 for (const child of current.namedChildren) stack.push(child);
             }
         }
+        // The name token of `module.Name`: the module root is read where
+        // the whole attribute is.
+        const { sameNode } = require('../languages/utils');
+        if (node?.parent?.type === 'attribute' && sameNode(node.parent.childForFieldName('attribute'), node)) {
+            node = node.parent;
+        }
         if (node) {
             const { referenceScope } = require('../languages/lexical-scope');
             const scopesKey = 'py-module-scopes';

package/core/reflection.js

@@ -24,8 +24,9 @@
  * memoized on the index and reset with it.
  */
 
-const { langTraits } = require('../languages');
+const { langTraits, detectLanguage } = require('../languages');
 const { sameNode } = require('../languages/utils');
+const { pythonPossibleIdentities, pythonSourceName } = require('./python-binding-identity');
 
 const WILD = '\u0000*';
 const MODULE_RECEIVER = Object.freeze({ kind: 'module' });
@@ -153,7 +154,13 @@ function _stringLiteral(node) {
 function stringAlternatives(node, ctx, depth = 0) {
     node = _unwrap(node);
     if (!node || depth > 12) return null;
-    if (STRING_NODES.has(node.type)) return _stringLiteral(node);
+    if (STRING_NODES.has(node.type)) {
+        const literal = ctx?.language === 'python'
+            ? require('./runtime-entrypoints').pythonStringLiteral(node)
+            : ['javascript', 'typescript', 'tsx', 'html'].includes(ctx?.language)
+                ? require('../languages/static-js').jsString(node) : null;
+        return literal != null ? [[literal]] : _stringLiteral(node);
+    }
     switch (node.type) {
         case 'concatenated_string': {
             let acc = [[]];
@@ -730,8 +737,31 @@ const MEMBER_CALLEE_FIELDS = {
     member_expression: ['property', 'object'], // JS/TS
     member_access_expression: ['name', 'expression'], // C#
     attribute: ['attribute', 'object'], // Python
+    field_access: ['field', 'object'], // Java qualified namespace
 };
 
+/** Parentheses change neither a callee nor a namespace qualifier. */
+function _calleeNode(node) {
+    while (node?.type === 'parenthesized_expression') {
+        node = node.namedChildren.find(child => child.type !== 'comment');
+    }
+    return node;
+}
+
+/** A written identifier/member path, from fields rather than intervening text. */
+function _calleeName(node) {
+    node = _calleeNode(node);
+    if (!node) return '';
+    if (['identifier', 'property_identifier', 'field_identifier', 'package_identifier'].includes(node.type)) {
+        return node.text;
+    }
+    const fields = MEMBER_CALLEE_FIELDS[node.type];
+    if (!fields) return '';
+    const owner = _calleeName(node.childForFieldName(fields[1]));
+    const member = _calleeName(node.childForFieldName(fields[0]));
+    return owner && member ? `${owner}.${member}` : '';
+}
+
 /** Callee text, member name and receiver of a call node (no arguments). */
 function _calleeParts(node, wantMembers, wantQualified) {
     if (node.type === 'method_invocation') {
@@ -739,7 +769,7 @@ function _calleeParts(node, wantMembers, wantQualified) {
         const name = node.childForFieldName('name')?.text || '';
         return { calleeText: name, member: name, object: node.childForFieldName('object') };
     }
-    const fn = node.childForFieldName('function') || node.namedChild(0);
+    const fn = _calleeNode(node.childForFieldName('function') || node.namedChild(0));
     if (!fn) return null;
     if (fn.type === 'identifier') {
         return { calleeText: fn.text || '', member: fn.text, object: null, bare: true };
@@ -762,18 +792,58 @@ function _callArgs(node) {
     return argsNode ? argsNode.namedChildren.filter(c => c.type !== 'comment') : null;
 }
 
+// Construction nodes: `new Function(..)` builds a callable from a string.
+const NEW_NODE_TYPES = ['new_expression', 'object_creation_expression'];
+
+/**
+ * Local names a file's imports bind to the modules of `{ package }` rules:
+ * `reflect` for `import "reflect"`, `r` for `import r "reflect"`. An import
+ * that binds no qualifier (Go `.` and `_`) names none.
+ * @returns {Map<string, string>} local name -> module
+ */
+function _packageQualifiers(rules, imports) {
+    const modules = new Set(rules.filter(rule => rule.package).map(rule => rule.package));
+    const qualifiers = new Map();
+    if (modules.size === 0) return qualifiers;
+    for (const item of imports || []) {
+        if (!item || !modules.has(item.module)) continue;
+        for (const name of item.names || []) {
+            // The parser already supplies valid import names, including
+            // Unicode identifiers. Dot and blank imports bind no qualifier.
+            if (name && name !== '_' && name !== '.') qualifiers.set(name, item.module);
+        }
+    }
+    return qualifiers;
+}
+
 /**
- * Reflection sites of one parsed file.
+ * Reflection facts of one parsed file: { sites, reflectionUses, evalUses }.
+ *
  * Each site: { kind, line, expression, dynamic, name?, patterns?, receiver?,
- *   membersOnly? }.
+ *   membersOnly? }: a reflective access that names its member by a string.
+ *
+ * The two counts are how often the file uses the language's reflection and
+ * code-execution vocabulary, whether or not a name can be read from the use.
+ * `reflectionUses`: the call-shaped sites (a `functions` or `members` call
+ * that passes its name argument) plus calls of the `unnamed` rules.
+ * `evalUses`: bare calls of `namespaceLookups.calls` that pass an argument,
+ * plus calls and constructions of the `codeExecution` rules. They count call
+ * and construction nodes of the tree, so a spelling inside a comment or a
+ * string is no use, and neither is a method or a definition that shares a
+ * name with a builtin (`self.exec(..)`, `def eval(..)`), nor a same-named
+ * method called without the name (`request.getMethod()`).
+ *
+ * `options.imports` are the file's import records: a `{ package }` rule
+ * matches a call qualified by a local name they bind to that module.
  */
-function reflectionSitesInTree(tree, language, code = null) {
+function reflectionFactsInTree(tree, language, code = null, options = {}) {
+    const facts = { sites: [], reflectionUses: 0, evalUses: 0 };
     const traits = langTraits(language) || {};
     const api = traits.reflectionApi;
-    if (!api || !tree) return [];
+    if (!api || !tree) return facts;
     const resolve = makeResolver();
-    const ctx = { formatCalls: api.formatCalls || null, resolve };
-    const sites = [];
+    const ctx = { formatCalls: api.formatCalls || null, resolve, language };
+    const sites = facts.sites;
     const push = (node, kind, nameNode, receiverNode, receiverFact) => {
         const cls = classifyNameExpression(nameNode, ctx);
         const expression = String(node.text || '');
@@ -796,22 +866,40 @@ function reflectionSitesInTree(tree, language, code = null) {
         });
     };
     const ns = api.namespaceLookups;
+    // Vocabulary that names no member: counted, never a site.
+    const unnamed = api.unnamed || [];
+    const codeExecution = api.codeExecution || [];
+    const constructs = codeExecution.filter(rule => rule.construct);
+    const qualifiers = _packageQualifiers([...unnamed, ...codeExecution], options.imports);
+    const scopeMemo = new Map();
     // Candidate nodes are located natively: reflective calls from the
     // vocabulary's spelling positions (descendantForIndex, walking up to the
     // enclosing call), computed member keys by node type. No JS-side walk of
     // the whole tree.
-    const hintPattern = api.hintPattern || null;
-    const useHints = code != null && hintPattern;
+    const hintSources = [];
+    if (api.hintPattern) hintSources.push(api.hintPattern.source);
+    // A package's qualifier is the file's own choice (`r "reflect"`).
+    if (qualifiers.size > 0) {
+        const { literalNameRegex } = require('./shared');
+        hintSources.push(...[...qualifiers.keys()].map(name => literalNameRegex(name).source));
+    }
+    // Aliases can call a reflective API without spelling that API at the call.
+    // Resolve the Python call ASTs whenever its binding vocabulary is present.
+    if (api.pythonBindingIdentity && code != null && !api.hintPattern.test(code.normalize('NFKC'))) return facts;
+    const useHints = !api.pythonBindingIdentity && code != null && hintSources.length > 0;
+    const callTypes = constructs.length > 0 ? [...CALL_NODE_TYPES, ...NEW_NODE_TYPES] : CALL_NODE_TYPES;
     const types = [];
     if (!useHints) {
-        types.push(...CALL_NODE_TYPES);
+        types.push(...callTypes);
         if (ns?.subscriptOf) types.push('subscript');
     }
     if (api.computedMembers) types.push('subscript_expression');
     const wantMembers = (api.members || []).length > 0;
-    // Qualified function callees (`Reflect.get`) need the callee text of
-    // member-shaped calls; bare vocabularies (`getattr`) never do.
-    const wantQualified = (api.functions || []).some(rule => rule.qualified);
+    // Qualified callees (`Reflect.get`, a package's functions) need the
+    // callee text of member-shaped calls; bare vocabularies (`getattr`)
+    // never do.
+    const wantQualified = api.pythonBindingIdentity || qualifiers.size > 0 ||
+        [...(api.functions || []), ...unnamed, ...codeExecution].some(rule => rule.qualified);
     const visitNode = node => {
         if (node.type === 'subscript_expression') {
             // obj["on" + evt] / this[`handle${t}`] (JS/TS computed member
@@ -819,6 +907,28 @@ function reflectionSitesInTree(tree, language, code = null) {
             // counts; opaque keys stay the computed-dispatch disclosure.
             const index = node.childForFieldName('index');
             const inner = _unwrap(index);
+            const { jsStaticKey, jsConstValue } = require('../languages/static-js');
+            const staticName = jsStaticKey(inner);
+            if (staticName != null) {
+                // This is normal member syntax. The call extractor carries
+                // the exact receiver; reflection keeps value-position access live.
+                if (node.parent?.type !== 'call_expression' || !sameNode(node.parent.childForFieldName('function'), node)) {
+                    sites.push({ kind: 'computed', name: staticName, patterns: [staticName], dynamic: false,
+                        line: node.startPosition.row + 1, expression: node.text.slice(0, EXPRESSION_CAP),
+                        receiver: classifyReceiver(node.childForFieldName('object'), language), membersOnly: true });
+                }
+                return;
+            }
+            const object = node.childForFieldName('object');
+            const construction = object?.type === 'new_expression' ? object : jsConstValue(object, true);
+            const type = construction?.type === 'new_expression' ? construction.childForFieldName('constructor') : null;
+            if (type?.type === 'identifier' && node.parent?.type === 'call_expression' &&
+                sameNode(node.parent.childForFieldName('function'), node)) {
+                sites.push({ kind: 'computed', dynamic: true, dynamicMember: true,
+                    receiver: { kind: 'type', name: type.text }, membersOnly: true,
+                    line: node.startPosition.row + 1, expression: node.text.slice(0, EXPRESSION_CAP) });
+                return;
+            }
             // `registry[type](..)` with `type` a parameter of the enclosing
             // callable dispatches by the name its callers pass (fix #428).
             if (inner?.type === 'identifier' && node.parent?.type === 'call_expression' &&
@@ -843,16 +953,70 @@ function reflectionSitesInTree(tree, language, code = null) {
             }
             return;
         }
+        if (NEW_NODE_TYPES.includes(node.type)) {
+            // `new Function(code)`: the constructed class decides.
+            const constructed = node.childForFieldName('constructor') || node.childForFieldName('type');
+            const text = _calleeName(constructed);
+            if (constructs.some(rule => rule.construct.test(text))) facts.evalUses++;
+            return;
+        }
         const call = _calleeParts(node, wantMembers, wantQualified);
         if (!call) return;
+        if (api.pythonBindingIdentity) {
+            const fn = _calleeNode(node.childForFieldName('function'));
+            const identities = pythonPossibleIdentities(fn, scopeMemo);
+            const args = _callArgs(node) || [];
+            const keyword = name => args.find(arg => arg.type === 'keyword_argument' &&
+                pythonSourceName(arg.childForFieldName('name')?.text) === name)?.childForFieldName('value');
+            for (const identity of identities) {
+                const isPatch = identity === 'unittest.mock.patch';
+                const isObjectPatch = identity === 'unittest.mock.patch.object';
+                const isMonkeyPatch = identity === 'pytest.MonkeyPatch.instance.setattr';
+                if (isPatch || isObjectPatch || isMonkeyPatch) {
+                    const positional = args.filter(arg => arg.type !== 'keyword_argument');
+                    const nameNode = positional[0] || keyword('target');
+                    const optional = isPatch || isObjectPatch ? keyword('create')?.type === 'true'
+                        : keyword('raising')?.type === 'false';
+                    const memberNode = positional[1] || keyword(isObjectPatch ? 'attribute' : 'name');
+                    const member = classifyNameExpression(memberNode, ctx);
+                    if (!optional && (isObjectPatch || (isMonkeyPatch && !stringAlternatives(nameNode, ctx))) && member.name) {
+                        facts.reflectionUses++;
+                        push(node, 'runtime-member', memberNode, nameNode);
+                        continue;
+                    }
+                    const alternatives = stringAlternatives(nameNode, ctx);
+                    const target = alternatives?.length === 1 && !alternatives[0].includes(WILD)
+                        ? alternatives[0].join('') : null;
+                    if (!optional && target?.includes('.')) {
+                        const name = target.slice(target.lastIndexOf('.') + 1);
+                        facts.reflectionUses++;
+                        const spellings = new Set([name]);
+                        // A private source member is spelled _Owner__member by
+                        // runtime APIs. The final owner-aware matcher decides it;
+                        // this index must first make its source spelling reachable.
+                        for (let offset = 1; offset < name.length - 1; offset++) {
+                            if (name[offset] === '_' && name[offset + 1] === '_') spellings.add(name.slice(offset));
+                        }
+                        sites.push({ kind: 'runtime-target', name, patterns: [...spellings], dynamic: false,
+                            runtimeTarget: target, line: node.startPosition.row + 1,
+                            expression: String(node.text).slice(0, EXPRESSION_CAP) });
+                    }
+                    continue;
+                }
+            }
+            call.possibleCallees = identities.filter(identity => identity.startsWith('builtins.')).map(identity => identity.slice(9));
+            call.calleeText = call.possibleCallees[0] || '';
+        }
         // Match on the callee spelling first; arguments are read only for a
         // vocabulary hit (most calls in a hinted file are unrelated).
-        const calleeText = () => (call.calleeText ??= call.fn.text || '');
+        const calleeText = () => (call.calleeText ??= _calleeName(call.fn));
         for (const rule of api.functions || []) {
-            if (!rule.callee.test(calleeText())) continue;
+            const matched = (call.possibleCallees || [calleeText()]).find(name => rule.callee.test(name));
+            if (!matched) continue;
             const args = _callArgs(node);
             if (args && args[rule.nameArg]) {
-                push(node, rule.kind || calleeText().split('.').pop(),
+                facts.reflectionUses++;
+                push(node, rule.kind || matched.split('.').pop(),
                     args[rule.nameArg], args[rule.receiverArg]);
             }
             return;
@@ -861,7 +1025,10 @@ function reflectionSitesInTree(tree, language, code = null) {
             for (const rule of api.members || []) {
                 if (!rule.member.test(call.member)) continue;
                 const args = _callArgs(node);
-                if (args && args[rule.nameArg]) push(node, call.member, args[rule.nameArg], call.object);
+                if (args && args[rule.nameArg]) {
+                    facts.reflectionUses++;
+                    push(node, call.member, args[rule.nameArg], call.object);
+                }
                 return;
             }
         }
@@ -869,25 +1036,58 @@ function reflectionSitesInTree(tree, language, code = null) {
         // so even a fully dynamic name is bounded to it.
         if (ns?.calls && call.bare && ns.calls.test(call.calleeText)) {
             const args = _callArgs(node);
-            if (args && args[0]) push(node, call.calleeText, args[0], null, MODULE_RECEIVER);
+            if (args && args[0]) {
+                facts.evalUses++;
+                push(node, call.calleeText, args[0], null, MODULE_RECEIVER);
+            }
+            return;
         }
+        if (unnamed.length === 0 && codeExecution.length === 0) return;
+        // The callee as written, receiver included (`Class.forName`,
+        // `reflect.ValueOf`); a bare callee is its own text.
+        let written = null;
+        const writtenCallee = () => (written ??= (node.type === 'method_invocation'
+            ? (call.object ? `${_calleeName(call.object)}.${call.member}` : call.member || '')
+            : calleeText()));
+        const matches = rule => {
+            if (rule.construct) return false;
+            if (rule.callee) return rule.callee.test(writtenCallee());
+            if (rule.package) {
+                const qualifier = _calleeNode(call.object);
+                if (qualifier?.type !== 'identifier' || qualifiers.get(qualifier.text) !== rule.package) return false;
+                // The file import owns the spelling only outside scopes
+                // that bind another value to it (parameters, locals, etc.).
+                return require('../languages/lexical-scope').referenceScope(
+                    qualifier, language, scopeMemo) === 'module';
+            }
+            return false;
+        };
+        if (unnamed.some(matches)) facts.reflectionUses++;
+        else if (codeExecution.some(matches)) facts.evalUses++;
     };
     try {
         const candidates = [];
         if (useHints) {
+            // One candidate per node: nested calls can begin at one offset
+            // (`a.getMethod(x).getField(y)`), so the end is part of the key.
             const seen = new Set();
-            const re = new RegExp(hintPattern.source, 'g');
+            const once = (kind, node) => {
+                const key = `${kind}${node.startIndex}:${node.endIndex}`;
+                if (seen.has(key)) return false;
+                seen.add(key);
+                return true;
+            };
+            const re = new RegExp(hintSources.join('|'), 'gu');
             let m;
             while ((m = re.exec(code)) !== null) {
+                if (m[0] === '') { re.lastIndex++; continue; }
                 let node = tree.rootNode.descendantForIndex(m.index);
-                for (let i = 0; node && i < 6; i++, node = node.parent) {
-                    if (!CALL_NODE_TYPES.includes(node.type)) continue;
-                    if (!seen.has(node.startIndex)) { seen.add(node.startIndex); candidates.push(node); }
+                for (; node; node = node.parent) {
+                    if (!callTypes.includes(node.type)) continue;
+                    if (once('c', node)) candidates.push(node);
                     // `globals()[name]`: the lookup is the enclosing subscript.
                     const parent = node.parent;
-                    if (ns?.subscriptOf && parent?.type === 'subscript' &&
-                        !seen.has(-parent.startIndex - 1)) {
-                        seen.add(-parent.startIndex - 1);
+                    if (ns?.subscriptOf && parent?.type === 'subscript' && once('s', parent)) {
                         candidates.push(parent);
                     }
                     break;
@@ -898,7 +1098,12 @@ function reflectionSitesInTree(tree, language, code = null) {
         candidates.sort((a, b) => a.startIndex - b.startIndex || (a.endIndex - b.endIndex));
         for (const node of candidates) visitNode(node);
     } catch (_) { /* partial inventory is disclosed as-is */ }
-    return sites;
+    return facts;
+}
+
+/** The reflection sites of one parsed file (see reflectionFactsInTree). */
+function reflectionSitesInTree(tree, language, code = null) {
+    return reflectionFactsInTree(tree, language, code).sites;
 }
 
 // ---------------------------------------------------------------------------
@@ -954,10 +1159,16 @@ function reflectionPatternIndex(index) {
     let entryCount = 0;
     const moduleScoped = new Map();
     const keyParameterSites = [];
+    const dynamicMembers = [];
     for (const [file, sites] of byFile) {
         const fe = index.files.get(file);
         if (!fe) continue;
         for (const site of sites) {
+            if (site.dynamicMember) {
+                dynamicMembers.push({ pattern: '*', file, relativePath: fe.relativePath,
+                    language: fe.language, family: _family(fe.language), site });
+                continue;
+            }
             if (site.receiver?.kind === 'module') {
                 // Bounded to this file's top-level names: a fully dynamic
                 // name reaches every one of them.
@@ -987,7 +1198,7 @@ function reflectionPatternIndex(index) {
         }
     }
     const result = { byFragment, lengths: [...lengths].sort((a, b) => a - b), entryCount, moduleScoped,
-        keyParameterSites, quoted: new Map(), keyNames: new Map() };
+        keyParameterSites, dynamicMembers, quoted: new Map(), keyNames: new Map() };
     index._reflectionPatternIndex = result;
     return result;
 }
@@ -1003,6 +1214,9 @@ function patternEntriesFor(index, name, language, file = null) {
         }
     }
     const family = language ? _family(language) : null;
+    for (const entry of pidx.dynamicMembers) {
+        if (!family || entry.family === family) out.push(entry);
+    }
     for (const entry of pidx.keyParameterSites) {
         if (family && entry.family !== family) continue;
         if (_keyParameterPasses(index, pidx, entry, name)) out.push({ ...entry, pattern: name });
@@ -1063,6 +1277,7 @@ function _reachesKey(index, fromRef, targetKey, targetName) {
  */
 function reflectionEntryReaches(index, entry, def) {
     const site = entry.site;
+    if (site.runtimeTarget) return require('./runtime-entrypoints').runtimeTargetReaches(index, def, site.runtimeTarget);
     if (site.membersOnly && !def.className && !def.receiver) return false;
     const recv = site.receiver;
     if (!recv) return true;
@@ -1101,8 +1316,9 @@ function reflectionEntryReaches(index, entry, def) {
 
 /** Reaching pattern entries for one definition (sorted file, line). */
 function reflectionPatternReferences(index, def) {
-    const lang = index.files.get(def.file)?.language;
-    const entries = patternEntriesFor(index, def.name, lang, def.file)
+    const lang = index.files.get(def.file)?.language || detectLanguage(def.file, index.root);
+    const runtimeName = langTraits(lang)?.reflectionApi?.pythonBindingIdentity ? pythonSourceName(def.name) : def.name;
+    const entries = patternEntriesFor(index, runtimeName, lang, def.file)
         .filter(entry => reflectionEntryReaches(index, entry, def));
     entries.sort((a, b) => (a.relativePath < b.relativePath ? -1 : a.relativePath > b.relativePath ? 1 : 0) ||
         a.site.line - b.site.line || (a.pattern < b.pattern ? -1 : a.pattern > b.pattern ? 1 : 0));
@@ -1132,6 +1348,7 @@ module.exports = {
     stringAlternatives,
     classifyNameExpression,
     classifyReceiver,
+    reflectionFactsInTree,
     reflectionSitesInTree,
     globMatches,
     patternEntriesFor,