slotdiff 0.2.0-alpha.1 → 0.3.0

package/src/erc7201.ts

@@ -11,6 +11,9 @@
  */
 
 import type { SolcOutput, StorageLayout } from "@openzeppelin/upgrades-core";
+import { styleText } from "node:util";
+import { LayoutIntegrityError, LayoutUnavailableError } from "./errors.js";
+import { namespaceLines, plain } from "./report.js";
 // Not exported from the package index; the package has no exports map, and
 // all exist since 1.46.0, the lowest version slotdiff accepts.
 import { getStorageLocationAnnotation } from "@openzeppelin/upgrades-core/dist/storage/namespace.js";
@@ -31,9 +34,10 @@ export interface NamespaceSlot {
   /**
    * `match`: the code uses the annotated slot. `mismatch`: it uses `actual`
    * instead. `conflict`: it assigns different slots in different places.
-   * `unread`: no assignment was found, or one could not be evaluated.
+   * `unused`: no code assigns its slot (a namespace kept reserved).
+   * `unread`: an assignment could not be evaluated.
    */
-  status: "match" | "mismatch" | "conflict" | "unread";
+  status: "match" | "mismatch" | "conflict" | "unused" | "unread";
 }
 
 interface AstNode {
@@ -80,14 +84,24 @@ export function namespaceSlots(
       }
     });
   }
+  // An overridden function can still run (through `super` or `Base.f()`), so
+  // its assembly is set aside: it never gives a slot, but it keeps a namespace
+  // it assigns from passing as unused.
   const assemblies: AstNode[] = [];
+  const setAside: AstNode[] = [];
   for (const source of Object.values(output.sources ?? {})) {
     walk(source.ast as unknown as AstNode, (node) => {
       const sibling =
         node.nodeType === "ContractDefinition" &&
         node.contractKind !== "library" &&
         !bases.has(node.id as number);
-      if (sibling || overridden.has(node.id as number)) return false;
+      if (sibling) return false;
+      if (overridden.has(node.id as number)) {
+        walk(node, (inner) => {
+          if (inner.nodeType === "InlineAssembly") setAside.push(inner);
+        });
+        return false;
+      }
       if (node.nodeType === "InlineAssembly") assemblies.push(node);
     });
   }
@@ -95,6 +109,20 @@ export function namespaceSlots(
 
   // Struct id → every slot value assigned to a pointer of that struct (undefined: unreadable).
   const assigned = new Map<number, Array<string | undefined>>();
+  // Structs assigned a slot only in set-aside code, and whether any pointer
+  // that is not a struct (a mapping or an array) is assigned one.
+  const hidden = new Set<number>();
+  let untyped = false;
+  const unannotated = (struct: number) => {
+    const node = byId.get(struct);
+    return node === undefined || getStorageLocationAnnotation(node as never) === undefined;
+  };
+  for (const assembly of setAside) {
+    forEachSlotAssignment(assembly, byId, (struct) => {
+      if (struct === undefined) untyped = true;
+      else hidden.add(struct);
+    });
+  }
   for (const assembly of assemblies) {
     const refs = (assembly.externalReferences ?? []) as Array<{
       declaration: number;
@@ -112,7 +140,10 @@ export function namespaceSlots(
         }
         const pointer = byId.get(declarationAt(target.src));
         const struct = (pointer?.typeName as AstNode | undefined)?.referencedDeclaration;
-        if (typeof struct !== "number") continue;
+        if (typeof struct !== "number") {
+          untyped = true;
+          continue;
+        }
         const value = node.value as AstNode;
         const slot =
           value.nodeType === "YulLiteral"
@@ -125,19 +156,47 @@ export function namespaceSlots(
     });
   }
 
+  const structsOf = (namespace: string) =>
+    [...byId.values()].filter(
+      (n) =>
+        n.nodeType === "StructDefinition" &&
+        bases.has(n.scope as number) &&
+        getStorageLocationAnnotation(n as never) === namespace,
+    );
+  // Storage placed through any other struct (a library's Layout, annotated or
+  // not, or a struct with no annotation) is in no layout OpenZeppelin builds,
+  // so a change to it would pass unseen. A slot passed in (StorageSlot) is not known.
+  const namespaced = new Set(namespaces.flatMap((n) => structsOf(n).map((s) => s.id!)));
+  for (const [id, slots] of assigned) {
+    if (namespaced.has(id) || slots.every((s) => s === undefined)) continue;
+    const struct = byId.get(id);
+    const scope = byId.get(struct?.scope as number)?.name;
+    throw new LayoutUnavailableError(
+      `${contract} keeps storage at a fixed slot through ${[scope, struct?.name].filter(Boolean).join(".")}, ` +
+        "which is not an ERC-7201 namespace of the contract or its bases (a @custom:storage-location " +
+        "struct declared there): its fields are in no layout slotdiff compares, so it cannot check them.",
+    );
+  }
+
   return namespaces.map((namespace) => {
     const expected = calculateERC7201StorageLocation(namespace.slice("erc7201:".length));
-    const slots = [...byId.values()]
-      .filter(
-        (n) =>
-          n.nodeType === "StructDefinition" &&
-          bases.has(n.scope as number) &&
-          getStorageLocationAnnotation(n as never) === namespace,
-      )
-      .flatMap((n) => assigned.get(n.id!) ?? []);
+    const structs = structsOf(namespace);
+    const slots = structs.flatMap((n) => assigned.get(n.id!) ?? []);
     const known = [...new Set(slots.filter((s) => s !== undefined))];
+    if (slots.length === 0) {
+      // Unused only when nothing could be placing it: no assignment in
+      // set-aside code, none to a struct nested in it, none to an untyped pointer.
+      const nested = new Set(structs.flatMap((n) => [...nestedStructs(n, byId)]));
+      const own = new Set(structs.map((n) => n.id!));
+      const placed =
+        untyped ||
+        [...hidden].some(unannotated) ||
+        [...own, ...nested].some((id) => hidden.has(id)) ||
+        [...nested].some((id) => !own.has(id) && assigned.has(id));
+      return { namespace, expected, status: placed ? "unread" : "unused" };
+    }
     if (known.length > 1) return { namespace, expected, status: "conflict" };
-    if (known.length === 0 || slots.includes(undefined)) {
+    if (slots.includes(undefined)) {
       return { namespace, expected, status: "unread" };
     }
     const actual = known[0];
@@ -152,11 +211,12 @@ export interface NamespacePair {
   next?: NamespaceSlot;
   /**
    * Why this pair fails the check: `<new>` assigns several slots (`conflict`),
-   * leaves the old data behind (`moved`), or keeps the data in place under an
-   * annotation that does not match (`mismatch`). `unread`: a slot could not be
-   * read, so the pair could not be checked.
+   * leaves the old data behind (`moved`), keeps the data in place under an
+   * annotation that does not match (`mismatch`), or stops assigning a namespace
+   * whose deployed annotation does not match its data (`dropped`). `unread`: a
+   * slot could not be read, so the pair could not be checked.
    */
-  problem?: "conflict" | "moved" | "mismatch" | "unread";
+  problem?: "conflict" | "moved" | "mismatch" | "dropped" | "unread";
 }
 
 export interface NamespaceCheck {
@@ -169,8 +229,13 @@ export interface NamespaceCheck {
   layout: StorageLayout;
   /** Every namespace of either side, old and new paired when they are the same storage. */
   pairs: NamespacePair[];
-  /** `fail` when any pair has a problem other than `unread`; `unread` when a slot could not be read. */
-  result: "pass" | "fail" | "unread";
+  /**
+   * `fail` when any pair has a problem other than `unread`; `unchecked` when a
+   * slot could not be read (a pair's problem is `unread`), as in `UpgradeCheck`.
+   */
+  result: "pass" | "fail" | "unchecked";
+  /** The CLI's namespace grid, with what each problem means; empty when nothing is worth showing. */
+  explain(color?: boolean): string;
 }
 
 /**
@@ -178,15 +243,16 @@ export interface NamespaceCheck {
  * OpenZeppelin goes by. An old and a new namespace are the same storage when
  * the code uses the same slot (under the same annotation first), failing that
  * when they share an annotation, and failing that when the new annotation
- * names the slot where the old data is.
+ * names the slot where the old data is. A namespace of either layout missing
+ * from that side's `namespaces` (or all of them, when it is absent) is unread.
  * Pass `layout` to `diffLayouts` in place of `old.layout`.
  */
 export function checkNamespaces(
-  old: { layout: StorageLayout; namespaces: NamespaceSlot[] },
-  next: { namespaces: NamespaceSlot[] },
+  old: { layout: StorageLayout; namespaces?: NamespaceSlot[] },
+  next: { layout: StorageLayout; namespaces?: NamespaceSlot[] },
 ): NamespaceCheck {
-  const unpaired = new Set(old.namespaces);
-  const pairs: NamespacePair[] = next.namespaces.map((n) => ({ next: n }));
+  const unpaired = new Set(slotsOf(old, "old"));
+  const pairs: NamespacePair[] = slotsOf(next, "new").map((n) => ({ next: n }));
   const sameSlot = (o: NamespaceSlot, n: NamespaceSlot) =>
     o.actual !== undefined && n.actual !== undefined && same(o.actual, n.actual);
   const rules: Array<(o: NamespaceSlot, n: NamespaceSlot) => boolean> = [
@@ -232,23 +298,148 @@ export function checkNamespaces(
     result: problems.some((p) => p !== undefined && p !== "unread")
       ? "fail"
       : problems.includes("unread")
-        ? "unread"
+        ? "unchecked"
         : "pass",
+    // Colored by default, as OpenZeppelin's reports are.
+    explain: (color = true) =>
+      namespaceLines(
+        pairs,
+        color ? (format, text) => styleText(format, text, { validateStream: false }) : plain,
+      ).join("\n"),
   };
 }
 
+/**
+ * One slot per namespace of `side.layout`. A namespace with no slot (a record
+ * from before slots were recorded) is unread, never left to OpenZeppelin's
+ * annotation-only match.
+ */
+function slotsOf(
+  side: { layout: StorageLayout; namespaces?: NamespaceSlot[] },
+  which: "old" | "new",
+): NamespaceSlot[] {
+  const misfit = slotsMisfit(side.layout, side.namespaces);
+  if (misfit !== undefined) {
+    throw new LayoutIntegrityError(
+      `The ${which} namespace slots do not fit its layout: ${misfit}.`,
+    );
+  }
+  const given = side.namespaces ?? [];
+  const seen = new Set(given.map((n) => n.namespace));
+  return [
+    ...given,
+    ...Object.keys(side.layout.namespaces ?? {})
+      .filter((namespace) => !seen.has(namespace))
+      .map((namespace): NamespaceSlot => ({
+        namespace,
+        expected: calculateERC7201StorageLocation(namespace.slice("erc7201:".length)),
+        status: "unread",
+      })),
+  ];
+}
+
+// `<formula>:<id>`, as OpenZeppelin keys namespaces. Never a name like
+// `__proto__`, which a plain object lookup would resolve on its prototype.
+const NAMESPACE_KEY = /^[\w.-]+:\S/;
+const SLOT = /^0x[0-9a-f]{64}$/i;
+
+/**
+ * Why `namespaces` cannot be the slots of `layout`'s namespaces, or undefined
+ * when they can (or are absent). A slot for a namespace the layout lacks, or a
+ * second one for the same namespace, could take another's place when
+ * namespaces are paired, so either is refused. Internal, shared with records.
+ */
+export function slotsMisfit(layout: StorageLayout, namespaces: unknown): string | undefined {
+  const names = Object.keys(layout.namespaces ?? {});
+  const odd = names.find((name) => !NAMESPACE_KEY.test(name));
+  if (odd !== undefined) return `${JSON.stringify(odd)} is not a storage location`;
+  if (namespaces === undefined) return undefined;
+  if (!Array.isArray(namespaces)) return "they are not a list";
+  const seen = new Set<string>();
+  for (const slot of namespaces) {
+    if (!isSlot(slot)) return "a slot is malformed";
+    if (!names.includes(slot.namespace)) return `${slot.namespace} is not in the layout`;
+    if (seen.has(slot.namespace)) return `${slot.namespace} appears twice`;
+    seen.add(slot.namespace);
+  }
+  return undefined;
+}
+
+// Well formed, and consistent with its status.
+function isSlot(value: unknown): value is NamespaceSlot {
+  const n = value as Partial<NamespaceSlot> | null;
+  if (
+    typeof n !== "object" ||
+    n === null ||
+    typeof n.namespace !== "string" ||
+    typeof n.expected !== "string" ||
+    !SLOT.test(n.expected) ||
+    (n.actual !== undefined && (typeof n.actual !== "string" || !SLOT.test(n.actual)))
+  ) {
+    return false;
+  }
+  const read = n.actual !== undefined;
+  const same = read && BigInt(n.actual!) === BigInt(n.expected);
+  return n.status === "match"
+    ? same
+    : n.status === "mismatch"
+      ? read && !same
+      : (n.status === "unread" || n.status === "conflict" || n.status === "unused") && !read;
+}
+
 function problemOf({ old, next }: NamespacePair): NamespacePair["problem"] {
   if (next?.status === "conflict") return "conflict";
   if (old?.actual !== undefined && next?.actual !== undefined && !same(old.actual, next.actual)) {
     return "moved";
   }
   if (next?.status === "mismatch") return "mismatch";
-  if ((old !== undefined && old.actual === undefined) || next?.status === "unread") {
+  // Only the deployed code knows where the data really is; once no code assigns
+  // the namespace, a later version that follows the wrong annotation would pass.
+  if (old?.status === "mismatch" && next?.status === "unused") return "dropped";
+  // An unused namespace has no data of that version to move: compared by annotation, as OpenZeppelin does.
+  const unknown = (s?: NamespaceSlot) =>
+    s !== undefined && s.actual === undefined && s.status !== "unused";
+  if (unknown(old) || next?.status === "unread") {
     return "unread";
   }
   return undefined;
 }
 
+// Calls `found` with the struct id of each pointer assigned a `.slot` in
+// `assembly`, or undefined for a pointer that is not a struct.
+function forEachSlotAssignment(
+  assembly: AstNode,
+  byId: Map<number, AstNode>,
+  found: (struct: number | undefined) => void,
+): void {
+  const refs = (assembly.externalReferences ?? []) as Array<{ declaration: number; src: string }>;
+  walk(assembly.AST as AstNode, (node) => {
+    if (node.nodeType !== "YulAssignment") return;
+    for (const target of node.variableNames as AstNode[]) {
+      if (typeof target.name !== "string" || !target.name.endsWith(".slot")) continue;
+      const pointer = byId.get(refs.find((r) => r.src === target.src)?.declaration ?? -1);
+      const struct = (pointer?.typeName as AstNode | undefined)?.referencedDeclaration;
+      found(typeof struct === "number" ? struct : undefined);
+    }
+  });
+}
+
+// Every struct that `struct` holds, at any depth (in members, mappings and arrays).
+function nestedStructs(struct: AstNode, byId: Map<number, AstNode>): Set<number> {
+  const found = new Set<number>();
+  const visit = (node: AstNode) =>
+    walk(node, (n) => {
+      if (n.nodeType !== "UserDefinedTypeName") return;
+      const target = byId.get(n.referencedDeclaration as number);
+      if (target?.nodeType === "StructDefinition" && !found.has(target.id!)) {
+        found.add(target.id!);
+        visit(target);
+      }
+    });
+  visit(struct);
+  return found;
+}
+
 function same(a: string, b: string): boolean {
   return BigInt(a) === BigInt(b);
 }

package/src/errors.ts

@@ -23,6 +23,28 @@ export class LayoutIntegrityError extends Error {
   }
 }
 
+/**
+ * A build is missing output the analysis needs (an AST, metadata, a storage
+ * layout or bytecode). Yours to fix, by compiling again with them selected
+ * (`hardhat compile --force` once the outputs are configured, say).
+ */
+export class IncompleteBuildError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "IncompleteBuildError";
+  }
+}
+
+/**
+ * Text from outside (an explorer, an RPC endpoint, solc quoting verified
+ * source) without control characters, so it cannot move the cursor or recolor
+ * a terminal. Tabs and line breaks stay.
+ */
+export function printable(text: string): string {
+  // eslint-disable-next-line no-control-regex
+  return text.replace(/[\u0000-\u0008\u000b-\u001f\u007f-\u009f]/g, " ");
+}
+
 /**
  * A network error's code (ECONNREFUSED, ENOTFOUND), never its message: fetch
  * quotes the URL in some messages, and URLs carry API keys.

package/src/explorer.ts

@@ -8,7 +8,7 @@
  */
 
 import type { SolcInput } from "@openzeppelin/upgrades-core";
-import { errorCode, LayoutUnavailableError } from "./errors.js";
+import { errorCode, LayoutUnavailableError, printable } from "./errors.js";
 
 export const ETHERSCAN_V2_API_URL = "https://api.etherscan.io/v2/api";
 
@@ -115,8 +115,12 @@ async function requestWithRetry(url: string, fetchImpl: FetchLike): Promise<Sour
       return entry as SourceCodeEntry;
     }
 
-    const detail =
-      typeof body.result === "string" ? body.result : (body.message ?? "unknown error");
+    // One line: a line break could draw a verdict of its own under the error.
+    const detail = printable(
+      String(typeof body.result === "string" ? body.result : (body.message ?? "unknown error")),
+    )
+      .replace(/\s+/g, " ")
+      .slice(0, 300);
     // Before the key check: Etherscan's rate-limit message mentions the API key.
     if (RATE_LIMIT.test(detail)) {
       if (retry) {
@@ -134,6 +138,10 @@ function backoff(attempt: number): Promise<void> {
   return new Promise((r) => setTimeout(r, 1000 * attempt));
 }
 
+// Control characters other than tabs and line breaks (\v, \f and \r included).
+// eslint-disable-next-line no-control-regex
+const CONTROL = /[\u0000-\u0008\u000e-\u001f\u007f-\u009f]/;
+
 /** Exported for tests. */
 export function parseSourceCodeEntry(entry: SourceCodeEntry, address: string): VerifiedSource {
   const raw = entry.SourceCode ?? "";
@@ -144,7 +152,8 @@ export function parseSourceCodeEntry(entry: SourceCodeEntry, address: string): V
   const compilerVersion = entry.CompilerVersion ?? "";
   if (!compilerVersion.startsWith("v0.")) {
     throw new LayoutUnavailableError(
-      `Contract ${address} was verified with "${compilerVersion}"; only Solidity is supported.`,
+      `Contract ${address} was verified with "${printable(compilerVersion).slice(0, 80)}"; ` +
+        "only Solidity is supported.",
     );
   }
 
@@ -154,7 +163,8 @@ export function parseSourceCodeEntry(entry: SourceCodeEntry, address: string): V
     )
   ) {
     throw new LayoutUnavailableError(
-      `Contract ${address} reports an unrecognized compiler version "${compilerVersion}".`,
+      `Contract ${address} reports an unrecognized compiler version ` +
+        `"${printable(compilerVersion).slice(0, 80)}".`,
     );
   }
 
@@ -166,11 +176,30 @@ export function parseSourceCodeEntry(entry: SourceCodeEntry, address: string): V
     );
   }
 
-  return {
-    input,
-    solcLongVersion: compilerVersion.slice(1),
-    contractName: entry.ContractName ?? "",
-  };
+  // Names, paths and source text (annotations, quoted by solc) reach the
+  // screen; real source holds no control characters but tabs and line breaks.
+  const contractName = entry.ContractName ?? "";
+  const sources = Object.values(input.sources) as Array<{ content?: unknown }>;
+  // Every source inline: solc would fetch `urls` itself.
+  if (sources.some((s) => typeof s?.content !== "string")) {
+    throw new LayoutUnavailableError(
+      `Contract ${address} is verified with sources that are not inline (urls); only content is read.`,
+    );
+  }
+  const contents = sources.map((s) => s.content as string);
+  if (
+    // Names and paths are printed on one line: no whitespace but a space.
+    [contractName, ...Object.keys(input.sources)].some(
+      (s) => printable(s) !== s || /[^\S ]/.test(s),
+    ) ||
+    contents.some((s) => CONTROL.test(s))
+  ) {
+    throw new LayoutUnavailableError(
+      `Contract ${address} is verified with control characters in its name, paths or source.`,
+    );
+  }
+
+  return { input, solcLongVersion: compilerVersion.slice(1), contractName };
 }
 
 // Etherscan wraps standard-json in an extra pair of braces (`{{...}}`); some

package/src/index.ts

@@ -32,8 +32,16 @@ export {
   type NamespaceSlot,
 } from "./erc7201.js";
 export {
-  layoutFromSource,
+  checkUpgrade,
+  type UpgradeCheck,
+  type UpgradeOptions,
+  type UpgradeSide,
+} from "./upgrade.js";
+export {
   analyzeSource,
+  analyzeBuild,
+  type BuildAnalysis,
+  type BuildCompiler,
   SAFETY_ERROR_KINDS,
   type SourceAnalysis,
   type SafetyOptions,
@@ -49,5 +57,5 @@ export {
   type ProvingMatch,
   type ImmutableReferences,
 } from "./bytecode.js";
-export { LayoutIntegrityError, LayoutUnavailableError } from "./errors.js";
-export type { SolcInput, StorageLayout } from "@openzeppelin/upgrades-core";
+export { IncompleteBuildError, LayoutIntegrityError, LayoutUnavailableError } from "./errors.js";
+export type { SolcInput, SolcOutput, StorageLayout } from "@openzeppelin/upgrades-core";