piral-cli 1.4.0-beta.6243 → 1.4.0-beta.6250

package/src/common/website.ts

@@ -0,0 +1,69 @@
+import { join, resolve } from 'path';
+import { createPiralStubIndexIfNotExists } from './template';
+import { packageJson } from './constants';
+import { ForceOverwrite } from './enums';
+import { createDirectory, writeBinary } from './io';
+import { writeJson } from './io';
+import { progress } from './log';
+import { axios } from '../external';
+import { EmulatorWebsiteManifestFiles, EmulatorWebsiteManifest } from '../types';
+
+async function downloadEmulatorFiles(manifestUrl: string, target: string, files: EmulatorWebsiteManifestFiles) {
+  const requiredFiles = [files.typings, files.main, files.app];
+
+  await Promise.all(
+    requiredFiles.map(async (file) => {
+      const url = new URL(file, manifestUrl);
+      const res = await axios.default.get(url.href, { responseType: 'arraybuffer' });
+      const data: Buffer = res.data;
+      await writeBinary(target, file, data);
+    }),
+  );
+}
+
+export async function scaffoldFromEmulatorWebsite(rootDir: string, manifestUrl: string) {
+  progress(`Downloading emulator from %s ...`, manifestUrl);
+  const response = await axios.default.get(manifestUrl);
+  const emulatorJson: EmulatorWebsiteManifest = response.data;
+
+  const targetDir = resolve(rootDir, 'node_modules', emulatorJson.name);
+  const appDirName = 'app';
+  const mainFile = 'index.js';
+  const appDir = resolve(targetDir, appDirName);
+  await createDirectory(appDir);
+
+  await writeJson(
+    targetDir,
+    packageJson,
+    {
+      name: emulatorJson.name,
+      description: emulatorJson.description,
+      version: emulatorJson.version,
+      importmap: emulatorJson.importmap,
+      pilets: emulatorJson.scaffolding.pilets,
+      piralCLI: {
+        version: emulatorJson.scaffolding.cli,
+        timestamp: emulatorJson.timestamp,
+        source: manifestUrl,
+        generated: true,
+      },
+      files: emulatorJson.files.assets,
+      main: `./${join(appDirName, mainFile)}`,
+      typings: `./${join(appDirName, emulatorJson.files.typings)}`,
+      app: `./${join(appDirName, emulatorJson.files.app)}`,
+      peerDependencies: {},
+      optionalDependencies: emulatorJson.dependencies.optional,
+      devDependencies: emulatorJson.dependencies.included,
+    },
+    true,
+  );
+
+  // actually including this one hints that the app shell should have been included - which is forbidden
+  await createPiralStubIndexIfNotExists(appDir, mainFile, ForceOverwrite.yes, {
+    name: emulatorJson.name,
+    outFile: emulatorJson.files.main,
+  });
+
+  await downloadEmulatorFiles(manifestUrl, appDir, emulatorJson.files);
+  return emulatorJson.name;
+}

package/src/injectors/pilet-injector.test.ts

@@ -40,7 +40,7 @@ describe('Piral-CLI pilet injector', () => {
     expect(injector.active).toBeFalsy();
   });
 
-  it('PiletInjector does not send empty content', () => {
+  it('PiletInjector does not send empty content', async () => {
     // Arrange
     const core = new EventEmitter();
     const injector = new PiletInjector(optionsMock, configMock, core);
@@ -51,7 +51,7 @@ describe('Piral-CLI pilet injector', () => {
 
     // Act
     try {
-      injector.sendFile(target, url); // this file does not exist
+      await injector.sendFile(target, url); // this file does not exist
     } catch {
       hasFailed = true;
     }
@@ -113,7 +113,7 @@ describe('Piral-CLI pilet injector', () => {
     expect(res).toBeUndefined();
   });
 
-  it('PiletInjector wont crash on request with no target', () => {
+  it('PiletInjector wont crash on request with no target', async () => {
     // Arrange
     const optionsMock = {
       pilets: [],
@@ -136,7 +136,7 @@ describe('Piral-CLI pilet injector', () => {
     } as KrasRequest;
 
     // Act
-    const res = injector.handle(request);
+    const res = await injector.handle(request);
 
     // Assert
     expect(res).toBeUndefined();

package/src/injectors/pilet-injector.ts

@@ -1,20 +1,14 @@
 import { URL } from 'url';
 import { join } from 'path';
 import { EventEmitter } from 'events';
-import { readFileSync, existsSync, statSync } from 'fs';
-import { KrasInjector, KrasResponse, KrasRequest, KrasInjectorConfig, KrasConfiguration, KrasResult } from 'kras';
+import { readFile, stat, writeFile } from 'fs/promises';
+import { KrasInjector, KrasRequest, KrasInjectorConfig, KrasConfiguration, KrasResult } from 'kras';
 import { log } from '../common/log';
 import { getPiletSpecMeta } from '../common/spec';
 import { config as commonConfig } from '../common/config';
 import { axios, mime, jju } from '../external';
 import { Bundler } from '../types';
 
-interface Pilet {
-  bundler: Bundler;
-  root: string;
-  getMeta(basePath: string): PiletMetadata;
-}
-
 export interface PiletInjectorConfig extends KrasInjectorConfig {
   /**
    * The pilets to serve.
@@ -54,31 +48,43 @@ export interface PiletInjectorConfig extends KrasInjectorConfig {
   headers?: Record<string, string>;
 }
 
+interface Pilet {
+  bundler: Bundler;
+  root: string;
+  getMeta(basePath: string): PiletMetadata;
+}
+
 interface PiletMetadata {
   name?: string;
   config?: Record<string, any>;
   [key: string]: unknown;
 }
 
-function getMetaOverride(root: string, metaFile: string) {
+async function getMetaOverride(root: string, metaFile: string) {
   if (metaFile) {
     const metaPath = join(root, metaFile);
+    const exists = await stat(metaPath).then(
+      () => true,
+      () => false,
+    );
 
-    if (existsSync(metaPath)) {
-      return jju.parse(readFileSync(metaPath, 'utf8'));
+    if (exists) {
+      const metaContent = await readFile(metaPath, 'utf8');
+      return jju.parse(metaContent);
     }
   }
 
   return undefined;
 }
 
-function fillPiletMeta(pilet: Pilet, metaFile: string, subPath: string) {
+async function fillPiletMeta(pilet: Pilet, metaFile: string, subPath: string) {
   const { root, bundler } = pilet;
   const packagePath = join(root, 'package.json');
-  const def = jju.parse(readFileSync(packagePath, 'utf8'));
+  const jsonContent = await readFile(packagePath, 'utf8');
+  const def = jju.parse(jsonContent);
   const file = bundler.bundle.name.replace(/^[\/\\]/, '');
   const target = join(bundler.bundle.dir, file);
-  const metaOverride = getMetaOverride(root, metaFile);
+  const metaOverride = await getMetaOverride(root, metaFile);
 
   pilet.getMeta = (parentPath) => {
     const basePath = `${parentPath}${subPath}`;
@@ -96,11 +102,11 @@ function fillPiletMeta(pilet: Pilet, metaFile: string, subPath: string) {
   };
 }
 
+type FeedResponse = { items?: Array<PiletMetadata> } | Array<PiletMetadata> | PiletMetadata;
+
 async function loadFeed(feed: string) {
   try {
-    const response = await axios.default.get<{ items?: Array<PiletMetadata> } | Array<PiletMetadata> | PiletMetadata>(
-      feed,
-    );
+    const response = await axios.default.get<FeedResponse>(feed);
 
     if (Array.isArray(response.data)) {
       return response.data;
@@ -118,16 +124,33 @@ export default class PiletInjector implements KrasInjector {
   public config: PiletInjectorConfig;
   private serverConfig: KrasConfiguration;
   private indexPath: string;
+  private proxyInfo?: {
+    source: string;
+    files: Array<string>;
+    date: Date;
+  };
 
   constructor(config: PiletInjectorConfig, serverConfig: KrasConfiguration, core: EventEmitter) {
     this.config = config;
     this.serverConfig = serverConfig;
 
     if (this.config.active) {
-      const { pilets, api, publicUrl, assetUrl } = config;
+      const { pilets, api, app, publicUrl, assetUrl } = config;
       this.indexPath = `${publicUrl}index.html`;
       const cbs = {};
 
+      if (app.endsWith('/app')) {
+        const packageJson = require(`${app}/../package.json`);
+
+        if (typeof packageJson.piralCLI.source === 'string') {
+          this.proxyInfo = {
+            source: packageJson.piralCLI.source,
+            files: packageJson.files,
+            date: new Date(packageJson.piralCLI.timestamp),
+          };
+        }
+      }
+
       core.on('user-connected', (e) => {
         const baseUrl = assetUrl || e.req.headers.origin;
 
@@ -144,8 +167,8 @@ export default class PiletInjector implements KrasInjector {
       });
 
       pilets.forEach((p, i) =>
-        p.bundler.on(() => {
-          fillPiletMeta(p, config.meta, `/${i}/`);
+        p.bundler.on(async () => {
+          await fillPiletMeta(p, config.meta, `/${i}/`);
 
           for (const id of Object.keys(cbs)) {
             const { baseUrl, notify } = cbs[id];
@@ -251,7 +274,7 @@ export default class PiletInjector implements KrasInjector {
     return merged;
   }
 
-  sendContent(content: Buffer | string, type: string, url: string): KrasResponse {
+  sendContent(content: Buffer | string, type: string, url: string): KrasResult {
     const { headers } = this.config;
 
     return {
@@ -269,8 +292,8 @@ export default class PiletInjector implements KrasInjector {
     };
   }
 
-  sendFile(target: string, url: string): KrasResponse {
-    const content = readFileSync(target);
+  async sendFile(target: string, url: string): Promise<KrasResult> {
+    const content = await readFile(target);
     const type = mime.getType(target) ?? 'application/octet-stream';
     return this.sendContent(content, type, url);
   }
@@ -281,23 +304,25 @@ export default class PiletInjector implements KrasInjector {
     const pilet = pilets[+index];
     const bundler = pilet?.bundler;
 
+    await bundler?.ready();
+
     if (!path) {
-      await bundler?.ready();
       const content = await this.getIndexMeta(baseUrl);
       return this.sendContent(content, 'application/json', url);
-    } else {
-      return bundler?.ready().then(() => {
-        const target = join(bundler.bundle.dir, rest.join('/'));
+    } else if (bundler?.bundle) {
+      const target = join(bundler.bundle.dir, rest.join('/'));
+      const info = await stat(target).catch(() => undefined);
 
-        if (existsSync(target) && statSync(target).isFile()) {
-          return this.sendFile(target, url);
-        }
-      });
+      if (info && info.isFile()) {
+        return await this.sendFile(target, url);
+      }
     }
+
+    return undefined;
   }
 
-  sendIndexFile(target: string, url: string, baseUrl: string): KrasResponse {
-    const indexHtml = readFileSync(target, 'utf8');
+  async sendIndexFile(target: string, url: string, baseUrl: string): Promise<KrasResult> {
+    const indexHtml = await readFile(target, 'utf8');
 
     // mechanism to inject server side debug piletApi config into piral emulator
     const windowInjectionScript = `window['dbg:pilet-api'] = '${this.getPiletApi(baseUrl)}';`;
@@ -308,29 +333,50 @@ export default class PiletInjector implements KrasInjector {
     return this.sendContent(content, mime.getType(target), url);
   }
 
-  handle(req: KrasRequest): KrasResponse {
+  private async shouldLoad(target: string, path: string) {
+    if (this.proxyInfo) {
+      if (!this.proxyInfo.files.includes(path)) {
+        return false;
+      }
+
+      const fileInfo = await stat(target).catch(() => undefined);
+
+      if (!fileInfo || fileInfo.mtime < this.proxyInfo.date) {
+        const url = new URL(path, this.proxyInfo.source);
+        const response = await axios.default.get(url.href, { responseType: 'arraybuffer' });
+        await writeFile(target, response.data);
+      }
+
+      return true;
+    } else {
+      const fileInfo = await stat(target).catch(() => undefined);
+      return fileInfo && fileInfo.isFile();
+    }
+  }
+
+  async handle(req: KrasRequest): Promise<KrasResult> {
     const { app, api, publicUrl, assetUrl } = this.config;
     const baseUrl =
       assetUrl || (req.headers.host ? `${req.encrypted ? 'https' : 'http'}://${req.headers.host}` : undefined);
 
     if (!req.target) {
       if (req.url.startsWith(publicUrl)) {
-        const path = req.url.substring(publicUrl.length).split('?')[0];
+        const path = req.url.substring(publicUrl.length).split('?').shift();
 
         if (app) {
           const target = join(app, path);
 
-          if (existsSync(target) && statSync(target).isFile()) {
+          if (await this.shouldLoad(target, path)) {
             if (req.url === this.indexPath) {
-              return this.sendIndexFile(target, req.url, baseUrl);
+              return await this.sendIndexFile(target, req.url, baseUrl);
             }
 
-            return this.sendFile(target, req.url);
+            return await this.sendFile(target, req.url);
           }
         }
 
         if (req.url !== this.indexPath) {
-          return this.handle({
+          return await this.handle({
             ...req,
             url: this.indexPath,
           });
@@ -339,8 +385,8 @@ export default class PiletInjector implements KrasInjector {
 
       return undefined;
     } else if (req.target === api) {
-      const path = req.url.substring(1).split('?')[0];
-      return this.sendResponse(path, req.url, baseUrl);
+      const path = req.url.substring(1).split('?').shift();
+      return await this.sendResponse(path, req.url, baseUrl);
     }
   }
 }

package/src/injectors/piral-injector.test.ts

@@ -64,7 +64,7 @@ describe('Piral-CLI piral injector', () => {
     expect(injector.active).toBeFalsy();
   });
 
-  it('PiralInjector can send reponse and fails with invalid path', () => {
+  it('PiralInjector can send reponse and fails with invalid path', async () => {
     // Arrange
     const config = {
       bundler: bundlerMock,
@@ -75,7 +75,7 @@ describe('Piral-CLI piral injector', () => {
     const injector = new PiralInjector(config, undefined as any, new EventEmitter());
 
     // Act
-    const res = injector.sendResponse('some/nice/invalid/path', 'sometarget.file', 'someDir', 'localhost:1234');
+    const res = await injector.sendResponse('some/nice/invalid/path', 'sometarget.file', 'someDir', 'localhost:1234');
 
     // Assert
     expect(res).toBeUndefined();

package/src/injectors/piral-injector.ts

@@ -1,15 +1,10 @@
 import { join } from 'path';
 import { EventEmitter } from 'events';
-import { readFileSync, existsSync, statSync } from 'fs';
-import { KrasInjector, KrasResponse, KrasRequest, KrasInjectorConfig, KrasConfiguration } from 'kras';
+import { readFile, stat } from 'fs/promises';
+import { KrasInjector, KrasRequest, KrasInjectorConfig, KrasConfiguration, KrasResult } from 'kras';
 import { mime } from '../external';
 import { Bundler } from '../types';
 
-/**
- * The maximum amount of retries when sending a response
- */
-const maxRetrySendResponse = 4;
-
 export interface PiralInjectorConfig extends KrasInjectorConfig {
   bundler: Bundler;
   publicUrl: string;
@@ -17,6 +12,20 @@ export interface PiralInjectorConfig extends KrasInjectorConfig {
   headers?: Record<string, string>;
 }
 
+/**
+ * The maximum amount of retries when sending a response
+ */
+const maxRetrySendResponse = 4;
+
+async function isNoFile(target: string) {
+  try {
+    const info = await stat(target);
+    return !info.isFile();
+  } catch {
+    return true;
+  }
+}
+
 export default class PiralInjector implements KrasInjector {
   public config: PiralInjectorConfig;
 
@@ -63,7 +72,7 @@ export default class PiralInjector implements KrasInjector {
 
   setOptions() {}
 
-  sendContent(content: Buffer | string, type: string, url: string): KrasResponse {
+  sendContent(content: Buffer | string, type: string, url: string): KrasResult {
     const { headers } = this.config;
     return {
       injector: { name: this.name },
@@ -80,8 +89,8 @@ export default class PiralInjector implements KrasInjector {
     };
   }
 
-  sendIndexFile(target: string, url: string): KrasResponse {
-    const indexHtml = readFileSync(target, 'utf8');
+  async sendIndexFile(target: string, url: string): Promise<KrasResult> {
+    const indexHtml = await readFile(target, 'utf8');
     const { feed } = this.config;
 
     if (feed) {
@@ -96,7 +105,7 @@ export default class PiralInjector implements KrasInjector {
     return this.sendContent(indexHtml, mime.getType(target), url);
   }
 
-  sendResponse(path: string, target: string, dir: string, url: string, recursionDepth = 0): KrasResponse {
+  async sendResponse(path: string, target: string, dir: string, url: string, recursionDepth = 0): Promise<KrasResult> {
     if (recursionDepth > maxRetrySendResponse) {
       return undefined;
     }
@@ -104,17 +113,18 @@ export default class PiralInjector implements KrasInjector {
     const { bundler } = this.config;
     const newTarget = join(bundler.bundle.dir, bundler.bundle.name);
 
-    if (!path || !existsSync(target) || !statSync(target).isFile()) {
-      return this.sendResponse(bundler.bundle.name, newTarget, dir, url, recursionDepth + 1);
+    if (!path || (await isNoFile(target))) {
+      return await this.sendResponse(bundler.bundle.name, newTarget, dir, url, recursionDepth + 1);
     } else if (target === newTarget) {
-      return this.sendIndexFile(target, url);
+      return await this.sendIndexFile(target, url);
+    } else {
+      const type = mime.getType(target) ?? 'application/octet-stream';
+      const content = await readFile(target);
+      return this.sendContent(content, type, url);
     }
-
-    const type = mime.getType(target) ?? 'application/octet-stream';
-    return this.sendContent(readFileSync(target), type, url);
   }
 
-  handle(req: KrasRequest): KrasResponse {
+  async handle(req: KrasRequest): Promise<KrasResult> {
     if (!req.target) {
       const { bundler, publicUrl } = this.config;
 
@@ -122,8 +132,9 @@ export default class PiralInjector implements KrasInjector {
         const pathLength = publicUrl.length || 1;
         const path = req.url.substring(pathLength);
         const dir = bundler.bundle.dir;
-        const target = join(dir, path.split('?')[0]);
-        return bundler.ready().then(() => this.sendResponse(path, target, dir, req.url));
+        const target = join(dir, path.split('?').shift());
+        await bundler.ready();
+        return await this.sendResponse(path, target, dir, req.url);
       }
     }
   }

package/src/types/common.ts

@@ -4,6 +4,30 @@ export interface Importmap {
   exclude?: Array<string>;
 }
 
+export interface EmulatorWebsiteManifestFiles {
+  typings: string;
+  main: string;
+  app: string;
+  assets: Array<string>;
+}
+
+export interface EmulatorWebsiteManifest {
+  name: string;
+  description: string;
+  version: string;
+  timestamp: string;
+  scaffolding: {
+    pilets: PiletsInfo;
+    cli: string;
+  };
+  files: EmulatorWebsiteManifestFiles;
+  importmap: Importmap;
+  dependencies: {
+    optional: Record<string, string>;
+    included: Record<string, string>;
+  };
+}
+
 export interface PackageData {
   name: string;
   version: string;

package/src/types/internal.ts

@@ -1,19 +1,20 @@
 import type { LogLevels } from './common';
 import type { ImportmapVersions, PiletSchemaVersion } from './public';
 
+export interface PiralInstanceDetails {
+  selected?: boolean;
+  port?: number;
+  path?: string;
+  url?: string;
+}
+
 /**
  * Shape of the pilet.json
  */
 export interface PiletDefinition {
   schemaVersion?: PiletSchemaVersion;
   importmapVersions?: ImportmapVersions;
-  piralInstances?: Record<
-    string,
-    {
-      selected?: boolean;
-      port?: number;
-    }
-  >;
+  piralInstances?: Record<string, PiralInstanceDetails>;
 }
 
 export interface PackageFiles {

package/src/types/public.ts

@@ -136,6 +136,7 @@ export interface BundlerPrepareArgs<T> {
 
 export interface BaseBundlerDefinition<T> {
   path: string;
+  exec?: string;
   prepare?: BundlerPrepareArgs<T>;
 }
 
@@ -235,7 +236,7 @@ export type PiralBuildType = 'all' | 'release' | 'emulator' | 'emulator-sources'
 
 export type PiletBuildType = 'default' | 'standalone' | 'manifest';
 
-export type PackageType = 'registry' | 'file' | 'git';
+export type PackageType = 'registry' | 'file' | 'git' | 'remote';
 
 export type NpmClientType = 'npm' | 'yarn' | 'pnp' | 'pnpm' | 'lerna' | 'rush' | 'bun';