skir 0.0.1

package/src/language_server.ts

@@ -0,0 +1,301 @@
+import * as yaml from "yaml";
+import { fromZodError } from "zod-validation-error";
+import { SkirConfig } from "./config.js";
+import { ModuleParser, ModuleSet } from "./module_set.js";
+import { parseModule } from "./parser.js";
+import { tokenizeModule } from "./tokenizer.js";
+import type {
+  Module,
+  MutableModule,
+  RecordKey,
+  RecordLocation,
+  Result,
+  SkirError,
+} from "./types.js";
+
+export class LanguageServerModuleSet {
+  constructor(private readonly rootPath: string) {}
+
+  setFileContent(uri: string, content: string): void {
+    this.deleteFile(uri);
+    const fileType = getFileType(uri);
+    switch (fileType) {
+      case "skir.yml": {
+        const workspace = this.parseSkirConfig(content, uri);
+        if (workspace) {
+          this.workspaces.set(uri, workspace);
+          this.reassignModulesToWorkspaces();
+        }
+        break;
+      }
+      case "*.skir": {
+        const moduleWorkspace = this.findModuleWorkspace(uri);
+        const moduleBundle = this.parseSkirModule(content, uri);
+        this.moduleBundles.set(uri, moduleBundle);
+        if (moduleWorkspace) {
+          Workspace.addModule(moduleBundle, moduleWorkspace);
+        }
+        break;
+      }
+      default: {
+        const _: null = fileType;
+      }
+    }
+  }
+
+  deleteFile(uri: string): void {
+    const fileType = getFileType(uri);
+    switch (fileType) {
+      case "skir.yml": {
+        if (this.workspaces.delete(uri)) {
+          this.reassignModulesToWorkspaces();
+        }
+        break;
+      }
+      case "*.skir": {
+        const moduleBundle = this.moduleBundles.get(uri);
+        if (moduleBundle) {
+          Workspace.removeModule(moduleBundle);
+          this.moduleBundles.delete(uri);
+        }
+        break;
+      }
+      default: {
+        const _: null = fileType;
+      }
+    }
+  }
+
+  private reassignModulesToWorkspaces(): void {
+    if (this.reassigneModulesTimeout) {
+      // Already scheduled, do nothing.
+      return;
+    }
+    this.reassigneModulesTimeout = setTimeout(() => {
+      for (const [moduleUri, moduleBundle] of this.moduleBundles.entries()) {
+        Workspace.removeModule(moduleBundle);
+        const newWorkspace = this.findModuleWorkspace(moduleUri);
+        if (newWorkspace) {
+          Workspace.addModule(moduleBundle, newWorkspace);
+        }
+      }
+      for (const workspace of this.workspaces.values()) {
+        workspace.scheduleResolution();
+      }
+      this.reassigneModulesTimeout = undefined;
+    });
+  }
+
+  private parseSkirConfig(content: string, uri: string): Workspace | null {
+    let skirConfig: SkirConfig;
+    {
+      // `yaml.parse` fail with a helpful error message, no need to add context.
+      const parseResult = SkirConfig.safeParse(yaml.parse(content));
+      if (parseResult.success) {
+        skirConfig = parseResult.data;
+      } else {
+        const validationError = fromZodError(parseResult.error);
+        console.error(
+          `Error parsing skir.yml at ${uri}:`,
+          validationError.message,
+        );
+        return null;
+      }
+    }
+
+    let rootUri = new URL(skirConfig.srcDir || ".", uri).href;
+    if (!rootUri.endsWith("/")) {
+      rootUri += "/";
+    }
+    return new Workspace(rootUri);
+  }
+
+  private parseSkirModule(content: string, uri: string): ModuleBundle {
+    let astTree: Result<Module | null>;
+    {
+      const tokens = tokenizeModule(content, uri);
+      if (tokens.errors.length !== 0) {
+        astTree = {
+          result: null,
+          errors: tokens.errors,
+        };
+      } else {
+        astTree = parseModule(tokens.result, uri, content);
+      }
+    }
+    return {
+      astTree,
+      errors: [],
+    };
+  }
+
+  /** Finds the workspace which contains the given module URI. */
+  private findModuleWorkspace(moduleUri: string): ModuleWorkspace | undefined {
+    let match: Workspace | undefined;
+    const leftIsBetter = (
+      left: Workspace,
+      right: Workspace | undefined,
+    ): boolean => {
+      if (right === undefined || left.rootUri.length < right.rootUri.length) {
+        return true;
+      }
+      if (left.rootUri.length === right.rootUri.length) {
+        // Completely arbitrary, just to have a consistent order.
+        return left.rootUri < right.rootUri;
+      }
+      return false;
+    };
+    for (const workspace of this.workspaces.values()) {
+      const { rootUri } = workspace;
+      if (moduleUri.startsWith(rootUri) && leftIsBetter(workspace, match)) {
+        match = workspace;
+      }
+    }
+    if (!match) {
+      return undefined;
+    }
+    return {
+      workspace: match,
+      modulePath: moduleUri.substring(match.rootUri.length),
+    };
+  }
+
+  private reassigneModulesTimeout?: NodeJS.Timeout;
+  private readonly moduleBundles = new Map<string, ModuleBundle>(); // key: file URI
+  private readonly workspaces = new Map<string, Workspace>(); // key: file URI
+}
+
+function errorToDiagnostic(error: SkirError): Diagnostic {
+  const { token, message, expected } = error;
+  return {
+    range: {
+      start: token.position,
+      end: token.position + token.text.length,
+    },
+    message: message ? message : `expected: ${expected}`,
+  };
+}
+
+function getFileType(uri: string): "skir.yml" | "*.skir" | null {
+  if (uri.endsWith("/skir.yml")) {
+    return "skir.yml";
+  } else if (uri.endsWith(".skir")) {
+    return "*.skir";
+  }
+  return null;
+}
+
+interface Diagnostic {
+  readonly range?: {
+    readonly start: number;
+    readonly end: number;
+  };
+  readonly message: string;
+}
+
+interface ModuleWorkspace {
+  readonly workspace: Workspace;
+  readonly modulePath: string;
+}
+
+interface ModuleBundle {
+  readonly astTree: Result<Module | null>;
+  moduleWorkspace?: ModuleWorkspace;
+  errors: Diagnostic[];
+}
+
+class Workspace implements ModuleParser {
+  constructor(readonly rootUri: string) {}
+
+  private readonly mutableRecordMap = new Map<RecordKey, RecordLocation>();
+  // key: module path
+  private readonly modules = new Map<string, ModuleBundle>();
+  private scheduledResolution?: {
+    timeout: NodeJS.Timeout;
+    promise: Promise<void>;
+    callback: () => void;
+  };
+
+  static addModule(
+    moduleBundle: ModuleBundle,
+    moduleWorkspace: ModuleWorkspace,
+  ): void {
+    // If the module was already in a workspace, remove it from the old workspace.
+    Workspace.removeModule(moduleBundle);
+    const { workspace } = moduleWorkspace;
+    moduleBundle.moduleWorkspace = moduleWorkspace;
+    workspace.modules.set(moduleWorkspace.modulePath, moduleBundle);
+    for (const record of moduleBundle.astTree.result?.records || []) {
+      workspace.mutableRecordMap.set(record.record.key, record);
+    }
+    workspace.scheduleResolution();
+  }
+
+  static removeModule(moduleBundle: ModuleBundle): void {
+    const { moduleWorkspace } = moduleBundle;
+    if (!moduleWorkspace) {
+      return;
+    }
+    const { workspace } = moduleWorkspace;
+    workspace.modules.delete(moduleWorkspace.modulePath);
+    for (const record of moduleBundle.astTree.result?.records || []) {
+      workspace.mutableRecordMap.delete(record.record.key);
+    }
+    moduleBundle.moduleWorkspace = undefined;
+    workspace.scheduleResolution();
+  }
+
+  parseModule(modulePath: string): Result<MutableModule | null> {
+    const moduleBundle = this.modules.get(modulePath);
+    if (!moduleBundle) {
+      return {
+        result: null,
+        errors: [],
+      };
+    }
+    return moduleBundle.astTree;
+  }
+
+  scheduleResolution(): void {
+    if (this.scheduledResolution) {
+      clearTimeout(this.scheduledResolution.timeout);
+    }
+    const delayMilliseconds = 500;
+    const timeout = setTimeout(() => {
+      this.scheduledResolution = undefined;
+      this.resolve();
+    }, delayMilliseconds);
+    const scheduledResolution = {
+      timeout,
+      promise: new Promise<void>((resolve) => {
+        scheduledResolution.callback = resolve;
+      }),
+      callback: (() => {
+        throw new Error("callback not set");
+      }) as () => void,
+    };
+    this.scheduledResolution = scheduledResolution;
+  }
+
+  get resolutionDone(): Promise<void> {
+    if (this.scheduledResolution) {
+      return this.scheduledResolution.promise;
+    }
+    return Promise.resolve();
+  }
+
+  /**
+   * Synchronously performs type resolution (and validation).
+   * Stores the errors in every module bundle.
+   */
+  private resolve(): void {
+    const moduleSet = new ModuleSet(this);
+    for (const [modulePath, moduleBundle] of this.modules.entries()) {
+      const parseResult = moduleSet.parseAndResolve(modulePath);
+      moduleBundle.errors = parseResult.errors.map(errorToDiagnostic);
+    }
+    if (this.scheduledResolution) {
+      this.scheduledResolution.callback();
+    }
+  }
+}

package/src/literals.ts

@@ -0,0 +1,120 @@
+import type { DenseJson, Primitive } from "./types.js";
+
+export function unquoteAndUnescape(stringLiteral: string): string {
+  const unquoted = stringLiteral.slice(1, -1);
+  const parts = unquoted.split("\\\\");
+  for (let i = 0; i < parts.length; ++i) {
+    const part = parts[i]!.replace(/\\(\r\n|\n|\r)/g, "\\n")
+      // Escape unescaped double quotes which can appear in a single-quoted
+      // string literal.
+      .replace(/(?<=^|[^\\])"/g, '\\"');
+    parts[i] = JSON.parse(`"${part}"`);
+  }
+  return parts.join("\\");
+}
+
+export function valueHasPrimitiveType(
+  token: string,
+  expectedType: Primitive,
+): boolean {
+  switch (expectedType) {
+    case "bool":
+      return token === "false" || token === "true";
+    case "bytes":
+      return (
+        isStringLiteral(token) &&
+        /^hex:([0-9A-Fa-f]{2})*$/.test(unquoteAndUnescape(token))
+      );
+    case "timestamp": {
+      if (!isStringLiteral(token)) {
+        return false;
+      }
+      const dateTime = unquoteAndUnescape(token);
+      const timestamp = Date.parse(dateTime).valueOf();
+      return (
+        // A timezone is required.
+        /(Z|[+-]\d\d:\d\d)$/.test(dateTime) &&
+        -8640000000000000 <= timestamp &&
+        timestamp <= 8640000000000000 &&
+        timestamp === Math.round(timestamp)
+      );
+    }
+    case "int32":
+      return isIntLiteral(token, BigInt(-2147483648), BigInt(2147483647));
+    case "int64":
+      return isIntLiteral(
+        token,
+        BigInt("-9223372036854775808"),
+        BigInt("9223372036854775807"),
+      );
+    case "uint64":
+      return isIntLiteral(token, BigInt(0), BigInt("18446744073709551615"));
+    case "float32":
+    case "float64": {
+      if (isStringLiteral(token)) {
+        const stringLiteral = unquoteAndUnescape(token);
+        return (
+          stringLiteral === "NaN" ||
+          stringLiteral === "Infinity" ||
+          stringLiteral === "-Infinity"
+        );
+      } else {
+        return /^[-0-9]/.test(token);
+      }
+    }
+    case "string":
+      return isStringLiteral(token);
+  }
+}
+
+export function isStringLiteral(token: string): boolean {
+  return /^['"]/.test(token);
+}
+
+function isIntLiteral(token: string, min: bigint, max: bigint): boolean {
+  if (!/^-?[0-9]+$/.test(token)) {
+    return false;
+  }
+  const value = BigInt(token);
+  return min <= value && value <= max;
+}
+
+export function literalValueToDenseJson(
+  token: string,
+  type: Primitive,
+): DenseJson {
+  switch (type) {
+    case "bool":
+      return token === "true" ? 1 : 0;
+    case "bytes": {
+      const string = unquoteAndUnescape(token);
+      const buffer = Buffer.from(string.substring(4), "hex");
+      return buffer.toString("base64");
+    }
+    case "timestamp": {
+      const dateTime = unquoteAndUnescape(token);
+      return Math.round(new Date(dateTime).valueOf());
+    }
+    case "int32":
+    case "float32":
+    case "float64": {
+      return isStringLiteral(token) ? unquoteAndUnescape(token) : Number(token);
+    }
+    case "int64":
+    case "uint64":
+      return String(BigInt(token));
+    case "string":
+      return unquoteAndUnescape(token);
+  }
+}
+
+/**
+ * Assuming `token` is a literal value of primitive type, returns a string which
+ * uniquely identifies the value within the primitive type. The behavior is
+ * undefined if `token` does not match `type`.
+ * Use this function to check if two literal values are actually equal.
+ */
+export function literalValueToIdentity(token: string, type: Primitive): string {
+  const denseJson = literalValueToDenseJson(token, type);
+  return typeof denseJson === "string" ? denseJson : JSON.stringify(denseJson);
+}

package/src/module_collector.ts

@@ -0,0 +1,22 @@
+import { glob } from "glob";
+import * as paths from "path";
+import { REAL_FILE_SYSTEM } from "./io.js";
+import { ModuleSet } from "./module_set.js";
+
+export async function collectModules(srcDir: string): Promise<ModuleSet> {
+  const modules = ModuleSet.create(REAL_FILE_SYSTEM, srcDir);
+  const skirFiles = await glob(paths.join(srcDir, "**/*.skir"), {
+    stat: true,
+    withFileTypes: true,
+  });
+  for await (const skirFile of skirFiles) {
+    if (!skirFile.isFile) {
+      continue;
+    }
+    const relativePath = paths
+      .relative(srcDir, skirFile.fullpath())
+      .replace(/\\/g, "/");
+    modules.parseAndResolve(relativePath);
+  }
+  return modules;
+}